Compare commits

..

1 Commits

Author SHA1 Message Date
Jakob Borg
1f09488a0f lib/model, lib/versioner: Prevent symlink attack via versioning (fixes #4286)
Prior to this, the following is possible:

- Create a symlink "foo -> /somewhere", it gets synced
- Delete "foo", it gets versioned
- Create "foo/bar", it gets synced
- Delete "foo/bar", it gets versioned in "/somewhere/bar"

With this change, versioners should never version symlinks.
2017-08-08 08:02:33 +02:00
2319 changed files with 1128479 additions and 77655 deletions

View File

@@ -1,5 +0,0 @@
coverage:
range: "40...100"
ignore:
- "**.pb.go"

2
.github/CODEOWNERS vendored
View File

@@ -1,2 +0,0 @@
/AUTHORS @calmh
/*.md @calmh

View File

@@ -1,42 +0,0 @@
### DO NOT REPORT SECURITY ISSUES IN THIS ISSUE TRACKER
Instead, contact security@syncthing.net directly - see
https://syncthing.net/security.html for more information.
### DO NOT POST SUPPORT REQUESTS OR GENERAL QUESTIONS IN THIS ISSUE TRACKER
Please use the forum at https://forum.syncthing.net/ where a large number of
helpful people hang out. This issue tracker is for reporting bugs or feature
requests directly to the developers. Worst case you might get a short
"that's a bug, please report it on GitHub" response on the forum, in which
case we thank you for your patience and following our advice. :)
### Please use the correct issue tracker
If your problem relates to a Syncthing wrapper or [sub-project](https://github.com/syncthing) such as [Syncthing for Android](https://github.com/syncthing/syncthing-android/issues), [SyncTrayzor](https://github.com/canton7/synctrayzor) or the [documentation](https://github.com/syncthing/docs/issues), please use their respective issue trackers.
### Does your log mention database corruption?
If your Syncthing log reports panics because of database corruption it is most likely a fault with your system's storage or memory. Affected log entries will contain lines starting with `panic: leveldb`. You will need to delete the index database to clear this, by running `syncthing -reset-database`.
### Please do post actual bug reports and feature requests.
If your issue is a bug report, replace this boilerplate with a description
of the problem, being sure to include at least:
- what happened,
- what you expected to happen instead, and
- any steps to reproduce the problem.
Also fill out the version information below and add log output or
screenshots as appropriate.
If your issue is a feature request, simply replace this template text in
its entirety.
### Version Information
Syncthing Version: v0.x.y
OS Version: Windows 7 / Ubuntu 14.04 / ...
Browser Version: (if applicable, for GUI issues)

7
.gitignore vendored
View File

@@ -17,10 +17,3 @@ RELEASE
deb
lib/auto/gui.files.go
snapcraft.yaml
prime/
snap/
parts/
stage/
*.snap
*.bz2
/repos

View File

@@ -1,5 +0,0 @@
service:
golangci-lint-version: 1.13.x
prepare:
- go run build.go assets
- GO111MODULE=on go mod vendor

1
.mailmap Symbolic link
View File

@@ -0,0 +1 @@
NICKS

92
AUTHORS
View File

@@ -1,60 +1,39 @@
# This is the official list of Syncthing authors for copyright purposes.
#
# THIS FILE IS MOSTLY AUTO GENERATED. IF YOU'VE MADE A COMMIT TO THE
# REPOSITORY YOU WILL BE ADDED HERE AUTOMATICALLY WITHOUT THE NEED FOR
# ANY MANUAL ACTION.
#
# That said, you are welcome to correct your name or add a nickname / GitHub
# user name as appropriate. The format is:
# The format is:
#
# Name Name Name (nickname) <email1@example.com> <email2@example.com>
#
# The in-GUI authors list is periodically automatically updated from the
# contents of this file.
#
# The NICKS list is auto generated from this file.
Aaron Bieber (qbit) <qbit@deftly.net>
Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com> <adam@proactiveservices.co.uk>
Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com>
Adel Qalieh (adelq) <aqalieh95@gmail.com> <adelq@users.noreply.github.com>
Alessandro G. (alessandro.g89) <alessandro.g89@gmail.com>
Alexander Graf (alex2108) <register-github@alex-graf.de>
Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
andresvia <andres.via@gmail.com>
Andrew Dunham (andrew-d) <andrew@du.nham.ca>
Andrew Rabert (nvllsvm) <ar@nullsum.net> <6550543+nvllsvm@users.noreply.github.com>
Andrey D (scienmind) <scintertech@cryptolab.net> <scienmind@users.noreply.github.com>
andyleap <andyleap@gmail.com>
Andrey D (scienmind) <scintertech@cryptolab.net>
Antoine Lamielle (0x010C) <antoine.lamielle@0x010c.fr> <gh@0x010c.fr>
Antony Male (canton7) <antony.male@gmail.com>
Aranjedeath <Aranjedeath@users.noreply.github.com>
Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com>
BAHADIR YILMAZ <bahadiryilmaz32@gmail.com>
Bart De Vries (mogwa1) <devriesb@gmail.com>
Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
Ben Shepherd (benshep) <bjashepherd@gmail.com>
Ben Sidhom (bsidhom) <bsidhom@gmail.com>
Benedikt Heine (bebehei) <bebe@bebehei.de>
Benedikt Morbach <benedikt.morbach@googlemail.com>
Benno Fünfstück <benno.fuenfstueck@gmail.com>
Benny Ng (tpng) <benny.tpng@gmail.com>
Boris Rybalkin <ribalkin@gmail.com>
Brandon Philips (philips) <brandon@ifup.org>
Brendan Long (brendanlong) <self@brendanlong.com>
Brian R. Becker (brbecker) <brbecker@gmail.com>
Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
Carsten Hagemann (Moter8) <moter8@gmail.com>
Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
Cedric Staniewski (xduugu) <cedric@gmx.ca>
Chris Howie (cdhowie) <me@chrishowie.com>
Chris Joel (cdata) <chris@scriptolo.gy>
Chris Tonkinson <chris@masterbran.ch>
chucic <chucic@seznam.cz>
Colin Kennedy (moshen) <moshen.colin@gmail.com>
Cromefire_ <tim.l@nghorst.net>
Dale Visser <dale.visser@live.com>
Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
Daniel Martí (mvdan) <mvdan@mvdan.cc>
@@ -62,58 +41,35 @@ Darshil Chanpura (dtchanpura) <dtchanpura@gmail.com> <dcprime314@gmail.com>
David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
Denis A. (dva) <denisva@gmail.com>
Dennis Wilson (snnd) <dw@risu.io>
derekriemer <derek.riemer@colorado.edu>
desbma <desbma@users.noreply.github.com>
Dmitry Saveliev (dsaveliev) <d.e.saveliev@gmail.com>
Dominik Heidler (asdil12) <dominik@heidler.eu>
Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
Elliot Huffman <thelich2@gmail.com>
Emil Hessman (ceh) <emil@hessman.se>
Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
Evgeny Kuznetsov <evgeny@kuznetsov.md>
Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
Felix Ableitner (Nutomic) <me@nutomic.com>
Felix Unterpaintner (bigbear2nd) <bigbear2nd@gmail.com>
Francois-Xavier Gsell (zukoo) <fxgsell@gmail.com>
Frank Isemann (fti7) <frank@isemann.name>
Gilli Sigurdsson (gillisig) <gilli@vx.is>
Graham Miln (grahammiln) <graham.miln@dssw.co.uk> <graham.miln@miln.eu>
Han Boetes <han@boetes.org>
Harrison Jones (harrisonhjones) <harrisonhjones@users.noreply.github.com>
Heiko Zuerker (Smiley73) <heiko@zuerker.org>
Hugo Locurcio <hugo.locurcio@hugo.pro>
Iain Barnett <iainspeed@gmail.com>
Ian Johnson (anonymouse64) <ian.johnson@canonical.com> <person.uwsome@gmail.com>
Iskander Sharipov (Alex) <quasilyte@gmail.com>
Jaakko Hannikainen (jgke) <jgke@jgke.fi>
Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
Jake Peterson (acogdev) <jake@acogdev.com>
Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net>
James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
janost <janost@tuta.io>
Jaroslav Malec (dzarda) <dzardacz@gmail.com>
jaseg <githubaccount@jaseg.net>
Jaya Chithra (jayachithra) <s.k.jayachithra@gmail.com>
Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
Jerry Jacobs (xor-gate) <jerry.jacobs@xor-gate.org> <xor-gate@users.noreply.github.com>
Jochen Voss (seehuhn) <voss@seehuhn.de>
Johan Andersson <j@i19.se>
Johan Vromans (sciurius) <jvromans@squirrel.nl>
John Rinehart (fuzzybear3965) <johnrichardrinehart@gmail.com>
Jonas Thelemann <e-mail@jonas-thelemann.de>
Jonathan Cross <jcross@gmail.com>
Jose Manuel Delicado (jmdaweb) <jmdaweb@hotmail.com> <jmdaweb@users.noreply.github.com>
Jörg Thalheim <Mic92@users.noreply.github.com>
Karol Różycki (krozycki) <rozycki.karol@gmail.com>
Keith Turner <kturner@apache.org>
Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
Kevin Allen (ironmig) <kma1660@gmail.com>
Kevin White, Jr. (kwhite17) <kevinwhite1710@gmail.com>
klemens <ka7@github.com>
Kurt Fitzner (Kudalufi) <kurt@va1der.ca> <kurt.fitzner@gmail.com>
Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
Laurent Arnoud <laurent@spkdev.net>
Laurent Etiemble (letiemble) <laurent.etiemble@gmail.com> <laurent.etiemble@monobjc.net>
Leo Arias (elopio) <yo@elopio.net>
Liu Siyuan (liusy182) <liusy182@gmail.com> <liusy182@hotmail.com>
@@ -123,78 +79,42 @@ Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
Marc Pujol (kilburn) <kilburn@la3.org>
Marcin Dziadus (marcindziadus) <dziadus.marcin@gmail.com>
marco-m <marco.molteni@laposte.net>
Mark Pulford (mpx) <mark@kyne.com.au>
Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
Matic Potočnik <hairyfotr@gmail.com>
Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
Matt Robenolt <matt@ydekproductions.com>
Matteo Ruina <matteo.ruina@gmail.com>
Maurizio Tomasi <ziotom78@gmail.com>
Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
MaximAL <almaximal@ya.ru>
Maxime Thirouin <m@moox.io>
Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
Michael Ploujnikov (plouj) <ploujj@gmail.com>
Michael Tilli (pyfisch) <pyfisch@gmail.com>
Mike Boone <mike@boonedocks.net>
MikeLund <MikeLund@users.noreply.github.com>
Nate Morrison (nrm21) <natemorrison@gmail.com>
Nicholas Rishel (PrototypeNM1) <rishel.nick@gmail.com> <PrototypeNM1@users.noreply.github.com>
Nico Stapelbroek <3368018+nstapelbroek@users.noreply.github.com>
Nicolas Braud-Santoni <nicolas@braud-santoni.eu>
Niels Peter Roest (Niller303) <nielsproest@hotmail.com> <seje.niels@hotmail.com>
Nils Jakobi (thunderstorm99) <jakobi.nils@gmail.com>
NoLooseEnds <jon.koslung@gmail.com>
otbutz <tbutz@optitool.de>
Oyebanji Jacob Mayowa <oyebanji05@gmail.com>
Pascal Jungblut (pascalj) <github@pascalj.com> <mail@pascal-jungblut.com>
Pawel Palenica (qepasa) <pawelpalenica11@gmail.com>
Paweł Rozlach <vespian@users.noreply.github.com>
perewa <cavalcante.ten@gmail.com>
Peter Badida <KeyWeeUsr@users.noreply.github.com>
Peter Dave Hello <hsu@peterdavehello.org>
Peter Hoeg (peterhoeg) <peter@speartail.com>
Peter Marquardt (wwwutz) <wwwutz@gmail.com> <wwwutz@googlemail.com>
Phil Davis <phil.davis@inf.org>
Philippe Schommers (filoozoom) <philippe@schommers.be>
Phill Luby (pluby) <phill.luby@newredo.com>
Pier Paolo Ramon <ramonpierre@gmail.com>
Piotr Bejda (piobpl) <piotrb10@gmail.com>
Pramodh KP (pramodhkp) <pramodh.p@directi.com> <1507241+pramodhkp@users.noreply.github.com>
Richard Hartmann <RichiH@users.noreply.github.com>
Robert Carosi (nov1n) <robert@carosi.nl>
Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
Ross Smith II (rasa) <ross@smithii.com>
rubenbe <github-com-00ff86@vandamme.email>
Ryan Sullivan (KayoticSully) <kayoticsully@gmail.com>
Sacheendra Talluri (sacheendra) <sacheendra.t@gmail.com>
Scott Klupfel (kluppy) <kluppy@going2blue.com>
Sergey Mishin (ralder) <ralder@yandex.ru>
Simon Frei (imsodin) <freisim93@gmail.com>
Sly_tom_cat <slytomcat@mail.ru>
Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
Suhas Gundimeda (snugghash) <suhas.gundimeda@gmail.com> <snugghash@gmail.com>
Taylor Khan (nelsonkhan) <nelsonkhan@gmail.com>
Thomas Hipp <thomashipp@gmail.com>
Tim Abell (timabell) <tim@timwise.co.uk>
Tim Howes (timhowes) <timhowes@berkeley.edu>
Tobias Nygren (tnn2) <tnn@nygren.pp.se>
Tobias Tom (tobiastom) <t.tom@succont.de>
Tomas Cerveny (kozec) <kozec@kozec.com>
Tommy Thorn <tommy-github-email@thorn.ws>
Tully Robinson (tojrobinson) <tully@tojr.org>
Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
Unrud (Unrud) <unrud@openaliasbox.org> <Unrud@users.noreply.github.com>
Veeti Paananen (veeti) <veeti.paananen@rojekti.fi>
Victor Buinsky (buinsky) <vix_booja@tut.by>
Vil Brekin (Vilbrekin) <vilbrekin@gmail.com>
Vladimir Rusinov <vrusinov@google.com>
wangguoliang <liangcszzu@163.com>
William A. Kennington III (wkennington) <william@wkennington.com>
Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de> <wulf@weich-kr.de>
Xavier O. (damajor) <damajor@gmail.com>
xjtdy888 (xjtdy888) <xjtdy888@163.com>
Yannic A. (eipiminus1) <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>
佛跳墙 <daoquan@qq.com>

View File

@@ -1,73 +1,92 @@
# Contributor Covenant Code of Conduct
## Conduct
## Our Pledge
* We are committed to providing a friendly, safe and welcoming
environment for all, regardless of gender, sexual orientation,
disability, ethnicity, religion, or similar personal characteristic.
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to making participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, gender identity and expression, level of experience,
education, socio-economic status, nationality, personal appearance, race,
religion, or sexual identity and orientation.
* On IRC, please avoid using overtly sexual nicknames or other nicknames
that might detract from a friendly, safe and welcoming environment for
all.
## Our Standards
* Please be kind and courteous. There's no need to be mean or rude.
Examples of behavior that contributes to creating a positive environment
include:
* Respect that people have differences of opinion and that every design
or implementation choice carries a trade-off and numerous costs. There
is seldom a right answer.
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
* Please keep unstructured critique to a minimum. If you have solid
ideas you want to experiment with, make a fork and see how it works.
Examples of unacceptable behavior by participants include:
* We will exclude you from interaction if you insult, demean or harass
anyone. That is not welcome behaviour. We interpret the term
"harassment" as including the definition in the <a
href="http://citizencodeofconduct.org/">Citizen Code of Conduct</a>;
if you have any lack of clarity about what might be included in that
concept, please read their definition. In particular, we don't
tolerate behavior that excludes people in socially marginalized
groups.
* The use of sexualized language or imagery and unwelcome sexual attention or
advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic
address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
* Private harassment is also unacceptable. No matter who you are, if you
feel you have been or are being harassed or made uncomfortable by a
community member, please contact one of the channel ops or any of the
Syncthing core team immediately. Whether you're a regular contributor
or a newcomer, we care about making this community a safe place for
you and we've got your back.
## Our Responsibilities
* Likewise any spamming, trolling, flaming, baiting or other
attention-stealing behaviour is not welcome.
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior.
## Moderation
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful.
These are the policies for upholding our community's standards of
conduct in our communication channels, most notably in Syncthing-related
IRC channels and on the web forum.
## Scope
1. Remarks that violate the Syncthing standards of conduct, including
hateful, hurtful, oppressive, or exclusionary remarks, are not
allowed. (Cursing is allowed, but never targeting another user, and
never in a hateful manner.)
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project e-mail
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
2. Remarks that moderators find inappropriate, whether listed in the
code of conduct or not, are also not allowed.
## Enforcement
3. Moderators will first respond to such remarks with a warning.
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at security@syncthing.net. All
complaints will be reviewed and investigated and will result in a response that
is deemed necessary and appropriate to the circumstances. The project team is
obligated to maintain confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
4. If the warning is unheeded, the user will be "kicked," i.e., kicked
out of the communication channel to cool off.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
5. If the user comes back and continues to make trouble, they will be
banned, i.e., indefinitely excluded.
## Attribution
6. Moderators may choose at their discretion to un-ban the user if it
was a first offense and they offer the offended party a genuine
apology.
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
7. If a moderator bans someone and you think it was unjustified, please
take it up with that moderator, or with a different moderator, **in
private**. Complaints about bans in-channel are not allowed.
[homepage]: https://www.contributor-covenant.org
8. Moderators are held to a higher standard than other community
members. If a moderator creates an inappropriate situation, they
should expect less leeway than others.
In the Syncthing community we strive to go the extra step to look out
for each other. Don't just aim to be technically unimpeachable, try to
be your best self. In particular, avoid flirting with offensive or
sensitive issues, particularly if they're off-topic; this all too
often leads to unnecessary fights, hurt feelings, and damaged trust;
worse, it can drive people away from the community entirely.
And if someone takes issue with something you said or did, resist the
urge to be defensive. Just stop doing what it was they complained about
and apologize. Even if you feel you were misinterpreted or unfairly
accused, chances are good there was something you could've communicated
better — remember that it's your responsibility to make your fellow
community members comfortable. Everyone wants to get along and we are
all here first and foremost because we want to talk about cool
technology. You will find that people will be eager to assume good
intent and forgive as long as you earn their trust.
*Adapted from the [Rust Code of Conduct](https://github.com/rust-lang/rust/wiki/Note-development-policy#conduct)*
*Adapted from the [Node.js Policy on Trolling](http://blog.izs.me/post/30036893703/policy-on-trolling)*

View File

@@ -1,32 +0,0 @@
FROM golang:1.11 AS builder
WORKDIR /src
COPY . .
ENV CGO_ENABLED=0
ENV BUILD_HOST=syncthing.net
ENV BUILD_USER=docker
RUN rm -f syncthing && go run build.go -no-upgrade build syncthing
FROM alpine
EXPOSE 8384 22000 21027/udp
VOLUME ["/var/syncthing"]
RUN apk add --no-cache ca-certificates su-exec
COPY --from=builder /src/syncthing /bin/syncthing
ENV PUID=1000 PGID=1000
HEALTHCHECK --interval=1m --timeout=10s \
CMD nc -z localhost 8384 || exit 1
ENTRYPOINT \
chown "${PUID}:${PGID}" /var/syncthing \
&& su-exec "${PUID}:${PGID}" \
env HOME=/var/syncthing \
/bin/syncthing \
-home /var/syncthing/config \
-gui-address 0.0.0.0:8384

View File

@@ -36,7 +36,7 @@ or modification by unauthorized parties.
Syncthing should be approachable, understandable and inclusive.
> Complex concepts and maths form the base of Syncthing's functionality.
> Complex concepts and maths form the base of Synchting's functionality.
> This should nonetheless be abstracted or hidden to a degree where
> Syncthing is usable by the general public.

28
ISSUE_TEMPLATE.md Normal file
View File

@@ -0,0 +1,28 @@
Do not report security issues in this bug tracker. Instead, contact
security@syncthing.net directly - see https://syncthing.net/security.html
for more information.
If your issue is a support request ("How do I get my devices to connect?"
or similar), please use the support forum at https://forum.syncthing.net/
where a large number of helpful people hang out. This issue tracker is for
reporting bugs or feature requests directly to the developers.
If your issue is a bug report, replace this boilerplate with a description
of the problem, being sure to include at least:
- what happened,
- what you expected to happen instead, and
- any steps to reproduce the problem.
Also fill out the version information below and add log output or
screenshots as appropriate.
If your issue is a feature request, simply replace this template text in
its entirety.
### Version Information
Syncthing Version: v0.x.y
OS Version: Windows 7 / Ubuntu 14.04 / ...
Browser Version: (if applicable, for GUI issues)

149
NICKS Normal file
View File

@@ -0,0 +1,149 @@
# This file maps email addresses used in commits to nicks used the changelog.
# It is auto generated from the AUTHORS file by script/authors.go.
0x010C <antoine.lamielle@0x010c.fr>
0x010C <gh@0x010c.fr>
acogdev <jake@acogdev.com>
adelq <aqalieh95@gmail.com>
adelq <adelq@users.noreply.github.com>
alessandro.g89 <alessandro.g89@gmail.com>
alex2108 <register-github@alex-graf.de>
andersonvom <andersonvom@gmail.com>
andrew-d <andrew@du.nham.ca>
asdil12 <dominik@heidler.eu>
AudriusButkevicius <audrius.butkevicius@gmail.com>
aviau <alexandre@alexandreviau.net>
aviau <aviau@debian.org>
bencurthoys <ben@bencurthoys.com>
benshep <bjashepherd@gmail.com>
bigbear2nd <bigbear2nd@gmail.com>
brbecker <brbecker@gmail.com>
brendanlong <self@brendanlong.com>
brgmnn <dan.arne.bergmann@gmail.com>
brgmnn <brgmnn@users.noreply.github.com>
bsidhom <bsidhom@gmail.com>
buinsky <vix_booja@tut.by>
burkemw3 <mburke@amplify.com>
burkemw3 <burkemw3@gmail.com>
calmh <jakob@nym.se>
calmh <jakob@kastelo.net>
canton7 <antony.male@gmail.com>
Cathryne <cathryne.linenweaver@gmail.com>
Cathryne <Cathryne@users.noreply.github.com>
cdata <chris@scriptolo.gy>
cdhowie <me@chrishowie.com>
ceh <emil@hessman.se>
cqcallaw <enlightened.despot@gmail.com>
damajor <damajor@gmail.com>
dinosore <dinosore@dbrsoftware.co.uk>
dtchanpura <dtchanpura@gmail.com>
dtchanpura <dcprime314@gmail.com>
dva <denisva@gmail.com>
dzarda <dzardacz@gmail.com>
eipiminus1 <eipiminusone+github@gmail.com>
eipiminus1 <eipiminus1@users.noreply.github.com>
elopio <yo@elopio.net>
facastagnini <federico.castagnini@gmail.com>
filoozoom <philippe@schommers.be>
frioux <frew@afoolishmanifesto.com>
frioux <frioux@gmail.com>
fti7 <frank@isemann.name>
gillisig <gilli@vx.is>
hadogenes <szafar@linux.pl>
imsodin <freisim93@gmail.com>
ironmig <kma1660@gmail.com>
jarlebring <jarlebring@gmail.com>
jayachithra <s.k.jayachithra@gmail.com>
jedie <github.com@jensdiemer.de>
jedie <git@jensdiemer.de>
jgke <jgke@jgke.fi>
jmdaweb <jmdaweb@hotmail.com>
jmdaweb <jmdaweb@users.noreply.github.com>
jpjp <jamespatterson@operamail.com>
jpjp <jpjp@users.noreply.github.com>
kamadak <kamada@nanohz.org>
KayoticSully <kayoticsully@gmail.com>
kc1212 <kc04bc@gmx.com>
kc1212 <kc1212@users.noreply.github.com>
kilburn <kilburn@la3.org>
kluppy <kluppy@going2blue.com>
kozec <kozec@kozec.com>
kralo <max.schulze@online.de>
kralo <kralo@users.noreply.github.com>
krozycki <rozycki.karol@gmail.com>
Kudalufi <kurt@va1der.ca>
Kudalufi <kurt.fitzner@gmail.com>
kwhite17 <kevinwhite1710@gmail.com>
letiemble <laurent.etiemble@gmail.com>
letiemble <laurent.etiemble@monobjc.net>
liusy182 <liusy182@gmail.com>
liusy182 <liusy182@hotmail.com>
lkwg82 <lkwg82@gmx.de>
LordLandon <lordlandon@gmail.com>
majedev <majed.alhajry@gmail.com>
marcindziadus <dziadus.marcin@gmail.com>
marclaporte <marc@marclaporte.com>
marclaporte <marc@laporte.name>
mateon1 <matin1111@wp.pl>
mogwa1 <devriesb@gmail.com>
moshen <moshen.colin@gmail.com>
Moter8 <moter8@gmail.com>
mpx <mark@kyne.com.au>
mvdan <mvdan@mvdan.cc>
Niller303 <nielsproest@hotmail.com>
Niller303 <seje.niels@hotmail.com>
norgeous <daniel@harte.me>
norgeous <daniel@danielharte.co.uk>
norgeous <norgeous@users.noreply.github.com>
nov1n <robert@carosi.nl>
nrm21 <natemorrison@gmail.com>
Nutomic <me@nutomic.com>
pascalj <github@pascalj.com>
pascalj <mail@pascal-jungblut.com>
peterhoeg <peter@speartail.com>
philips <brandon@ifup.org>
piobpl <piotrb10@gmail.com>
plouj <ploujj@gmail.com>
pluby <phill.luby@newredo.com>
ProactiveServices <aD@simplypeachy.co.uk>
ProactiveServices <simplypeachy@users.noreply.github.com>
ProactiveServices <ProactiveServices@users.noreply.github.com>
pyfisch <pyfisch@gmail.com>
qbit <qbit@deftly.net>
ralder <ralder@yandex.ru>
rasa <ross@smithii.com>
Rewt0r <rewt0r@gmx.com>
Rewt0r <Rewt0r@users.noreply.github.com>
rumpelsepp <stefan@sevenbyte.org>
rumpelsepp <rumpelsepp@sevenbyte.org>
sacheendra <sacheendra.t@gmail.com>
scienmind <scintertech@cryptolab.net>
sciurius <jvromans@squirrel.nl>
seehuhn <voss@seehuhn.de>
Smiley73 <heiko@zuerker.org>
snnd <dw@risu.io>
snugghash <suhas.gundimeda@gmail.com>
snugghash <snugghash@gmail.com>
Stefan-Code <stefan.github@gmail.com>
Stefan-Code <Stefan.github@gmail.com>
timabell <tim@timwise.co.uk>
timhowes <timhowes@berkeley.edu>
tnn2 <tnn@nygren.pp.se>
tojrobinson <tully@tojr.org>
tpng <benny.tpng@gmail.com>
tylerbrazier <tyler@tylerbrazier.com>
Unrud <unrud@openaliasbox.org>
Unrud <Unrud@users.noreply.github.com>
uok <ueomkail@gmail.com>
uok <uok@users.noreply.github.com>
veeti <veeti.paananen@rojekti.fi>
Vilbrekin <vilbrekin@gmail.com>
wkennington <william@wkennington.com>
WSGCSysadmin <e.meitner@willystreet.coop>
wweich <wweich@users.noreply.github.com>
wweich <wweich@gmx.de>
wweich <wulf@weich-kr.de>
xduugu <cedric@gmx.ca>
zaynetro <romanznet@gmail.com>
Zillode <zillode@zillode.be>
zukoo <fxgsell@gmail.com>

View File

@@ -20,8 +20,13 @@ If this is a user visible change (including API and protocol changes), add a lin
to the corresponding pull request on https://github.com/syncthing/docs or describe
the documentation changes necessary.
## Authorship
Your name and email will be added automatically to the AUTHORS file
based on the commit metadata.
### Authorship
Every author of a code contribution (Go, Javascript, HTML, CSS etc, with the
possible exception of minor typo corrections and similar) is recorded in the
AUTHORS and NICKS files and the in-GUI credits. If this is your first
contribution, a maintainer will add you properly before accepting the
contribution. You need not do so yourself or worry about the fact that the
"authors" automated test fails. However, if your name (such as you want it
presented in the credits) is not visible on your Github profile or in your
commit messages, please assist by providing it here.

View File

@@ -1,34 +0,0 @@
# Docker Container for Syncthing
Use the Dockerfile in this repo, or pull the `syncthing/syncthing` image
from Docker Hub.
Use the `/var/syncthing` volume to have the synchronized files available on the
host. You can add more folders and map them as you prefer.
Note that Syncthing runs as UID 1000 and GID 1000 by default. These may be
altered with the ``PUID`` and ``PGID`` environment variables.
## Example Usage
```
$ docker pull syncthing/syncthing
$ docker run -p 8384:8384 -p 22000:22000 \
-v /wherever/st-sync:/var/syncthing \
syncthing/syncthing:latest
```
Note that local device discovery will not work with the above command, resulting in poor local transfer rates if local device addresses are not manually configured.
To allow local discovery, the docker host network can be used instead:
```
$ docker pull syncthing/syncthing
$ docker run --network=host \
-v /wherever/st-sync:/var/syncthing \
syncthing/syncthing:latest
```
Be aware that syncthing alone is now in control of what interfaces and ports it
listens on. You can edit the syncthing configuration to change the defaults if
there are conflicts.

View File

@@ -2,10 +2,11 @@
---
[![Latest Downloads](https://img.shields.io/badge/latest-downloads-brightgreen.svg?style=flat-square)](https://build.syncthing.net/latest/)
[![Latest Linux & Cross Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildLinuxCross.svg?style=flat-square&label=linux+%26+cross+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildLinuxCross&guest=1)
[![Latest Windows Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildWindows.svg?style=flat-square&label=windows+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildWindows&guest=1)
[![Latest Mac Build](https://img.shields.io/teamcity/https/build.syncthing.net/s/Syncthing_BuildMac.svg?style=flat-square&label=mac+build)](https://build.syncthing.net/viewType.html?buildTypeId=Syncthing_BuildMac&guest=1)
[![Latest Linux & Cross Build](https://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing.svg?style=flat-square&label=linux+%26+cross)](https://build.syncthing.net/job/syncthing/lastBuild/)
[![Latest Windows Build](https://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing-windows.svg?style=flat-square&label=windows)](https://build.syncthing.net/job/syncthing/lastBuild/)
[![Latest Mac Build](https://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing-mac.svg?style=flat-square&label=mac)](https://build.syncthing.net/job/syncthing/lastBuild/)
[![Latest Solaris Build](https://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing-solaris.svg?style=flat-square&label=solaris)](https://build.syncthing.net/job/syncthing/lastBuild/)
[![API Documentation](https://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](https://godoc.org/github.com/syncthing/syncthing)
[![MPLv2 License](https://img.shields.io/badge/license-MPLv2-blue.svg?style=flat-square)](https://www.mozilla.org/MPL/2.0/)
[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/88/badge)](https://bestpractices.coreinfrastructure.org/projects/88)
[![Go Report Card](https://goreportcard.com/badge/github.com/syncthing/syncthing)](https://goreportcard.com/report/github.com/syncthing/syncthing)
@@ -20,36 +21,36 @@ commentary, see the full [Goals document][13].
Syncthing should be:
1. **Safe From Data Loss**
1. Safe From Data Loss
Protecting the user's data is paramount. We take every reasonable
precaution to avoid corrupting the user's files.
2. **Secure Against Attackers**
2. Secure Against Attackers
Again, protecting the user's data is paramount. Regardless of our other
goals we must never allow the user's data to be susceptible to
eavesdropping or modification by unauthorized parties.
3. **Easy to Use**
3. Easy to Use
Syncthing should be approachable, understandable and inclusive.
4. **Automatic**
4. Automatic
User interaction should be required only when absolutely necessary.
5. **Universally Available**
5. Universally Available
Syncthing should run on every common computer. We are mindful that the
latest technology is not always available to any given individual.
6. **For Individuals**
6. For Individuals
Syncthing is primarily about empowering the individual user with safe,
secure and easy to use file synchronization.
7. **Everything Else**
7. Everything Else
There are many things we care about that don't make it on to the list. It
is fine to optimize for these values, as long as they are not in conflict
@@ -87,8 +88,8 @@ D26E6ED000654A3E, available from https://syncthing.net/security.html and
most key servers.
There is also a built in automatic upgrade mechanism (disabled in some
distribution channels) which uses a compiled in ECDSA signature. macOS
binaries are also properly code signed.
distribution channels) which uses a compiled in ECDSA signature. Mac OS
X binaries are also properly code signed.
## Documentation
@@ -110,5 +111,4 @@ All code is licensed under the [MPLv2 License][7].
[12]: https://www.bountysource.com/teams/syncthing/issues
[13]: https://github.com/syncthing/syncthing/blob/master/GOALS.md
[14]: assets/logo-text-128.png
[15]: https://syncthing.net/
[15]: https://syncthing.net/

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 160 KiB

484
build.go
View File

@@ -34,23 +34,17 @@ import (
)
var (
versionRe = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
goarch string
goos string
noupgrade bool
version string
goCmd string
goVersion float64
race bool
debug = os.Getenv("BUILDDEBUG") != ""
extraTags string
installSuffix string
pkgdir string
cc string
debugBinary bool
coverage bool
timeout = "120s"
gogoProtoVersion = "v1.2.0"
versionRe = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
goarch string
goos string
noupgrade bool
version string
goVersion float64
race bool
debug = os.Getenv("BUILDDEBUG") != ""
noBuildGopath bool
extraTags string
installSuffix string
)
type target struct {
@@ -114,14 +108,6 @@ var targets = map[string]target{
{src: "etc/linux-systemd/system/syncthing-resume.service", dst: "deb/lib/systemd/system/syncthing-resume.service", perm: 0644},
{src: "etc/linux-systemd/user/syncthing.service", dst: "deb/usr/lib/systemd/user/syncthing.service", perm: 0644},
{src: "etc/firewall-ufw/syncthing", dst: "deb/etc/ufw/applications.d/syncthing", perm: 0644},
{src: "etc/linux-desktop/syncthing-start.desktop", dst: "deb/usr/share/applications/syncthing-start.desktop", perm: 0644},
{src: "etc/linux-desktop/syncthing-ui.desktop", dst: "deb/usr/share/applications/syncthing-ui.desktop", perm: 0644},
{src: "assets/logo-32.png", dst: "deb/usr/share/icons/hicolor/32x32/apps/syncthing.png", perm: 0644},
{src: "assets/logo-64.png", dst: "deb/usr/share/icons/hicolor/64x64/apps/syncthing.png", perm: 0644},
{src: "assets/logo-128.png", dst: "deb/usr/share/icons/hicolor/128x128/apps/syncthing.png", perm: 0644},
{src: "assets/logo-256.png", dst: "deb/usr/share/icons/hicolor/256x256/apps/syncthing.png", perm: 0644},
{src: "assets/logo-512.png", dst: "deb/usr/share/icons/hicolor/512x512/apps/syncthing.png", perm: 0644},
{src: "assets/logo-only.svg", dst: "deb/usr/share/icons/hicolor/scalable/apps/syncthing.svg", perm: 0644},
},
},
"stdiscosrv": {
@@ -134,13 +120,13 @@ var targets = map[string]target{
archiveFiles: []archiveFile{
{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
{src: "cmd/stdiscosrv/README.md", dst: "README.txt", perm: 0644},
{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
{src: "cmd/stdiscosrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
},
installationFiles: []archiveFile{
{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
{src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/syncthing-discosrv/README.txt", perm: 0644},
{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-discosrv/LICENSE.txt", perm: 0644},
{src: "cmd/stdiscosrv/LICENSE", dst: "deb/usr/share/doc/syncthing-discosrv/LICENSE.txt", perm: 0644},
{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-discosrv/AUTHORS.txt", perm: 0644},
{src: "man/stdiscosrv.1", dst: "deb/usr/share/man/man1/stdiscosrv.1", perm: 0644},
},
@@ -157,14 +143,12 @@ var targets = map[string]target{
{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
{src: "cmd/strelaysrv/README.md", dst: "README.txt", perm: 0644},
{src: "cmd/strelaysrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
{src: "LICENSE", dst: "LICENSE.txt", perm: 0644},
{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
},
installationFiles: []archiveFile{
{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
{src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/syncthing-relaysrv/README.txt", perm: 0644},
{src: "cmd/strelaysrv/LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
{src: "LICENSE", dst: "deb/usr/share/doc/syncthing-relaysrv/LICENSE.txt", perm: 0644},
{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing-relaysrv/AUTHORS.txt", perm: 0644},
{src: "man/strelaysrv.1", dst: "deb/usr/share/man/man1/strelaysrv.1", perm: 0644},
},
@@ -191,19 +175,6 @@ var targets = map[string]target{
},
}
// These are repos we need to clone to run "go generate"
type dependencyRepo struct {
path string
repo string
commit string
}
var dependencyRepos = []dependencyRepo{
{path: "protobuf", repo: "https://github.com/gogo/protobuf.git", commit: gogoProtoVersion},
{path: "xdr", repo: "https://github.com/calmh/xdr.git", commit: "08e072f9cb16"},
}
func init() {
// The "syncthing" target includes a few more files found in the "etc"
// and "extra" dirs.
@@ -232,6 +203,27 @@ func main() {
}()
}
if gopath() == "" {
gopath, err := temporaryBuildDir()
if err != nil {
log.Fatal(err)
}
if !noBuildGopath {
lazyRebuildAssets()
if err := buildGOPATH(gopath); err != nil {
log.Fatal(err)
}
}
os.Setenv("GOPATH", gopath)
log.Println("GOPATH is", gopath)
}
// Set path to $GOPATH/bin:$PATH so that we can for sure find tools we
// might have installed during "build.go setup".
os.Setenv("PATH", fmt.Sprintf("%s%cbin%c%s", os.Getenv("GOPATH"), os.PathSeparator, os.PathListSeparator, os.Getenv("PATH")))
checkArchitecture()
// Invoking build.go with no parameters at all builds everything (incrementally),
// which is what you want for maximum error checking during development.
if flag.NArg() == 0 {
@@ -253,8 +245,20 @@ func main() {
}
}
func checkArchitecture() {
switch goarch {
case "386", "amd64", "arm", "arm64", "ppc64", "ppc64le", "mips", "mipsle":
break
default:
log.Printf("Unknown goarch %q; proceed with caution!", goarch)
}
}
func runCommand(cmd string, target target) {
switch cmd {
case "setup":
setup()
case "install":
var tags []string
if noupgrade {
@@ -302,6 +306,9 @@ func runCommand(cmd string, target target) {
case "snap":
buildSnap(target)
case "clean":
clean()
case "vet":
metalintShort()
@@ -314,6 +321,13 @@ func runCommand(cmd string, target target) {
case "version":
fmt.Println(getVersion())
case "gopath":
gopath, err := temporaryBuildDir()
if err != nil {
log.Fatal(err)
}
fmt.Println(gopath)
default:
log.Fatalf("Unknown command %q", cmd)
}
@@ -322,41 +336,61 @@ func runCommand(cmd string, target target) {
func parseFlags() {
flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
flag.StringVar(&goCmd, "gocmd", "go", "Specify `go` command")
flag.BoolVar(&noupgrade, "no-upgrade", noupgrade, "Disable upgrade functionality")
flag.StringVar(&version, "version", getVersion(), "Set compiled in version string")
flag.BoolVar(&race, "race", race, "Use race detector")
flag.BoolVar(&noBuildGopath, "no-build-gopath", noBuildGopath, "Don't build GOPATH, assume it's OK")
flag.StringVar(&extraTags, "tags", extraTags, "Extra tags, space separated")
flag.StringVar(&installSuffix, "installsuffix", installSuffix, "Install suffix, optional")
flag.StringVar(&pkgdir, "pkgdir", "", "Set -pkgdir parameter for `go build`")
flag.StringVar(&cc, "cc", os.Getenv("CC"), "Set CC environment variable for `go build`")
flag.BoolVar(&debugBinary, "debug-binary", debugBinary, "Create unoptimized binary to use with delve, set -gcflags='-N -l' and omit -ldflags")
flag.BoolVar(&coverage, "coverage", coverage, "Write coverage profile of tests to coverage.txt")
flag.Parse()
}
func setup() {
packages := []string{
"github.com/alecthomas/gometalinter",
"github.com/AlekSi/gocov-xml",
"github.com/axw/gocov/gocov",
"github.com/FiloSottile/gvt",
"github.com/golang/lint/golint",
"github.com/gordonklaus/ineffassign",
"github.com/mdempsky/unconvert",
"github.com/mitchellh/go-wordwrap",
"github.com/opennota/check/cmd/...",
"github.com/tsenart/deadcode",
"golang.org/x/net/html",
"golang.org/x/tools/cmd/cover",
"honnef.co/go/simple/cmd/gosimple",
"honnef.co/go/staticcheck/cmd/staticcheck",
"honnef.co/go/unused/cmd/unused",
}
for _, pkg := range packages {
fmt.Println(pkg)
runPrint("go", "get", "-u", pkg)
}
runPrint("go", "install", "-v", "github.com/syncthing/syncthing/vendor/github.com/gogo/protobuf/protoc-gen-gogofast")
}
func test(pkgs ...string) {
lazyRebuildAssets()
args := []string{"test", "-short", "-timeout", timeout, "-tags", "purego"}
if runtime.GOARCH == "amd64" {
switch runtime.GOOS {
case "darwin", "linux", "freebsd": // , "windows": # See https://github.com/golang/go/issues/27089
args = append(args, "-race")
}
useRace := runtime.GOARCH == "amd64"
switch runtime.GOOS {
case "darwin", "linux", "freebsd", "windows":
default:
useRace = false
}
if coverage {
args = append(args, "-covermode", "atomic", "-coverprofile", "coverage.txt")
if useRace {
runPrint("go", append([]string{"test", "-short", "-race", "-timeout", "60s", "-tags", "purego"}, pkgs...)...)
} else {
runPrint("go", append([]string{"test", "-short", "-timeout", "60s", "-tags", "purego"}, pkgs...)...)
}
runPrint(goCmd, append(args, pkgs...)...)
}
func bench(pkgs ...string) {
lazyRebuildAssets()
runPrint(goCmd, append([]string{"test", "-run", "NONE", "-bench", "."}, pkgs...)...)
runPrint("go", append([]string{"test", "-run", "NONE", "-bench", "."}, pkgs...)...)
}
func install(target target, tags []string) {
@@ -369,64 +403,7 @@ func install(target target, tags []string) {
log.Fatal(err)
}
os.Setenv("GOBIN", filepath.Join(cwd, "bin"))
args := []string{"install", "-v"}
args = appendParameters(args, tags, target)
os.Setenv("GOOS", goos)
os.Setenv("GOARCH", goarch)
os.Setenv("CC", cc)
// On Windows generate a special file which the Go compiler will
// automatically use when generating Windows binaries to set things like
// the file icon, version, etc.
if goos == "windows" {
sysoPath, err := shouldBuildSyso(cwd)
if err != nil {
log.Printf("Warning: Windows binaries will not have file information encoded: %v", err)
}
defer shouldCleanupSyso(sysoPath)
}
runPrint(goCmd, args...)
}
func build(target target, tags []string) {
lazyRebuildAssets()
tags = append(target.tags, tags...)
rmr(target.BinaryName())
args := []string{"build", "-v"}
args = appendParameters(args, tags, target)
os.Setenv("GOOS", goos)
os.Setenv("GOARCH", goarch)
os.Setenv("CC", cc)
// On Windows generate a special file which the Go compiler will
// automatically use when generating Windows binaries to set things like
// the file icon, version, etc.
if goos == "windows" {
cwd, err := os.Getwd()
if err != nil {
log.Fatal(err)
}
sysoPath, err := shouldBuildSyso(cwd)
if err != nil {
log.Printf("Warning: Windows binaries will not have file information encoded: %v", err)
}
defer shouldCleanupSyso(sysoPath)
}
runPrint(goCmd, args...)
}
func appendParameters(args []string, tags []string, target target) []string {
if pkgdir != "" {
args = append(args, "-pkgdir", pkgdir)
}
args := []string{"install", "-v", "-ldflags", ldflags()}
if len(tags) > 0 {
args = append(args, "-tags", strings.Join(tags, " "))
}
@@ -436,19 +413,34 @@ func appendParameters(args []string, tags []string, target target) []string {
if race {
args = append(args, "-race")
}
args = append(args, target.buildPkg)
if !debugBinary {
// Regular binaries get version tagged and skip some debug symbols
args = append(args, "-ldflags", ldflags())
} else {
// -gcflags to disable optimizations and inlining. Skip -ldflags
// because `Could not launch program: decoding dwarf section info at
// offset 0x0: too short` on 'dlv exec ...' see
// https://github.com/derekparker/delve/issues/79
args = append(args, "-gcflags", "-N -l")
os.Setenv("GOOS", goos)
os.Setenv("GOARCH", goarch)
runPrint("go", args...)
}
func build(target target, tags []string) {
lazyRebuildAssets()
tags = append(target.tags, tags...)
rmr(target.BinaryName())
args := []string{"build", "-i", "-v", "-ldflags", ldflags()}
if len(tags) > 0 {
args = append(args, "-tags", strings.Join(tags, " "))
}
if installSuffix != "" {
args = append(args, "-installsuffix", installSuffix)
}
if race {
args = append(args, "-race")
}
args = append(args, target.buildPkg)
return append(args, target.buildPkg)
os.Setenv("GOOS", goos)
os.Setenv("GOARCH", goarch)
runPrint("go", args...)
}
func buildTar(target target) {
@@ -489,10 +481,6 @@ func buildZip(target target) {
build(target, tags)
if goos == "windows" {
windowsCodesign(target.BinaryName())
}
for i := range target.archiveFiles {
target.archiveFiles[i].src = strings.Replace(target.archiveFiles[i].src, "{{binary}}", target.BinaryName(), 1)
target.archiveFiles[i].dst = strings.Replace(target.archiveFiles[i].dst, "{{binary}}", target.BinaryName(), 1)
@@ -577,8 +565,6 @@ func buildSnap(target target) {
snaparch := goarch
if snaparch == "armhf" {
goarch = "arm"
} else if snaparch == "i386" {
goarch = "386"
}
snapver := version
if strings.HasPrefix(snapver, "v") {
@@ -589,10 +575,9 @@ func buildSnap(target target) {
snapgrade = "stable"
}
err = tmpl.Execute(f, map[string]string{
"Version": snapver,
"HostArchitecture": runtime.GOARCH,
"TargetArchitecture": snaparch,
"Grade": snapgrade,
"Version": snapver,
"Architecture": snaparch,
"Grade": snapgrade,
})
if err != nil {
log.Fatal(err)
@@ -602,56 +587,6 @@ func buildSnap(target target) {
runPrint("snapcraft")
}
func shouldBuildSyso(dir string) (string, error) {
jsonPath := filepath.Join(dir, "versioninfo.json")
file, err := os.Create(filepath.Join(dir, "versioninfo.json"))
if err != nil {
return "", errors.New("failed to create " + jsonPath + ": " + err.Error())
}
major, minor, patch, build := semanticVersion()
fmt.Fprintf(file, `{
"FixedFileInfo": {
"FileVersion": {
"Major": %s,
"Minor": %s,
"Patch": %s,
"Build": %s
}
},
"StringFileInfo": {
"FileDescription": "Open Source Continuous File Synchronization",
"LegalCopyright": "The Syncthing Authors",
"ProductVersion": "%s",
"ProductName": "Syncthing"
},
"IconPath": "assets/logo.ico"
}`, major, minor, patch, build, getVersion())
file.Close()
defer func() {
if err := os.Remove(jsonPath); err != nil {
log.Printf("Warning: unable to remove generated %s: %v. Please remove it manually.", jsonPath, err)
}
}()
sysoPath := filepath.Join(dir, "cmd", "syncthing", "resource.syso")
if _, err := runError("goversioninfo", "-o", sysoPath); err != nil {
return "", errors.New("failed to create " + sysoPath + ": " + err.Error())
}
return sysoPath, nil
}
func shouldCleanupSyso(sysoFilePath string) {
if sysoFilePath == "" {
return
}
if err := os.Remove(sysoFilePath); err != nil {
log.Printf("Warning: unable to remove generated %s: %v. Please remove it manually.", sysoFilePath, err)
}
}
// copyFile copies a file from src to dst, ensuring the containing directory
// exists. The permission bits are copied as well. If dst already exists and
// the contents are identical to src the modification time is not updated.
@@ -699,12 +634,12 @@ func listFiles(dir string) []string {
}
func rebuildAssets() {
os.Setenv("SOURCE_DATE_EPOCH", fmt.Sprint(buildStamp()))
runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/auto", "github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto")
runPipe("lib/auto/gui.files.go", "go", "run", "script/genassets.go", "gui")
runPipe("cmd/strelaypoolsrv/auto/gui.go", "go", "run", "script/genassets.go", "cmd/strelaypoolsrv/gui")
}
func lazyRebuildAssets() {
if shouldRebuildAssets("lib/auto/gui.files.go", "gui") || shouldRebuildAssets("cmd/strelaypoolsrv/auto/gui.files.go", "cmd/strelaypoolsrv/auto/gui") {
if shouldRebuildAssets("lib/auto/gui.files.go", "gui") || shouldRebuildAssets("cmd/strelaypoolsrv/auto/gui.go", "cmd/strelaypoolsrv/auto/gui") {
rebuildAssets()
}
}
@@ -736,21 +671,12 @@ func shouldRebuildAssets(target, srcdir string) bool {
}
func proto() {
runPrint(goCmd, "get", fmt.Sprintf("github.com/gogo/protobuf/protoc-gen-gogofast@%v", gogoProtoVersion))
os.MkdirAll("repos", 0755)
for _, dep := range dependencyRepos {
path := filepath.Join("repos", dep.path)
if _, err := os.Stat(path); err != nil {
runPrintInDir("repos", "git", "clone", dep.repo, dep.path)
runPrintInDir(path, "git", "checkout", dep.commit)
}
}
runPrint(goCmd, "generate", "github.com/syncthing/syncthing/lib/...", "github.com/syncthing/syncthing/cmd/stdiscosrv")
runPrint("go", "generate", "github.com/syncthing/syncthing/lib/...")
}
func translate() {
os.Chdir("gui/default/assets/lang")
runPipe("lang-en-new.json", goCmd, "run", "../../../../script/translate.go", "lang-en.json", "../../../")
runPipe("lang-en-new.json", "go", "run", "../../../../script/translate.go", "lang-en.json", "../../../")
os.Remove("lang-en.json")
err := os.Rename("lang-en-new.json", "lang-en.json")
if err != nil {
@@ -761,7 +687,12 @@ func translate() {
func transifex() {
os.Chdir("gui/default/assets/lang")
runPrint(goCmd, "run", "../../../../script/transifexdl.go")
runPrint("go", "run", "../../../../script/transifexdl.go")
}
func clean() {
rmr("bin")
rmr(filepath.Join(os.Getenv("GOPATH"), fmt.Sprintf("pkg/%s_%s/github.com/syncthing", goos, goarch)))
}
func ldflags() string {
@@ -772,10 +703,10 @@ func ldflags() string {
b := new(bytes.Buffer)
b.WriteString("-w")
fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Version%c%s", sep, version)
fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Stamp%c%d", sep, buildStamp())
fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.User%c%s", sep, buildUser())
fmt.Fprintf(b, " -X github.com/syncthing/syncthing/lib/build.Host%c%s", sep, buildHost())
fmt.Fprintf(b, " -X main.Version%c%s", sep, version)
fmt.Fprintf(b, " -X main.BuildStamp%c%d", sep, buildStamp())
fmt.Fprintf(b, " -X main.BuildUser%c%s", sep, buildUser())
fmt.Fprintf(b, " -X main.BuildHost%c%s", sep, buildHost())
return b.String()
}
@@ -832,15 +763,6 @@ func getVersion() string {
return "unknown-dev"
}
func semanticVersion() (major, minor, patch, build string) {
r := regexp.MustCompile(`v(?P<Major>\d+)\.(?P<Minor>\d+).(?P<Patch>\d+).*\+(?P<CommitsAhead>\d+)`)
matches := r.FindStringSubmatch(getVersion())
if len(matches) != 5 {
return "0", "0", "0", "0"
}
return matches[1], matches[2], matches[3], matches[4]
}
func getBranchSuffix() string {
bs, err := runError("git", "branch", "-a", "--contains")
if err != nil {
@@ -935,7 +857,7 @@ func buildHost() string {
func buildArch() string {
os := goos
if os == "darwin" {
os = "macos"
os = "macosx"
}
return fmt.Sprintf("%s-%s", os, goarch)
}
@@ -958,10 +880,6 @@ func runError(cmd string, args ...string) ([]byte, error) {
}
func runPrint(cmd string, args ...string) {
runPrintInDir(".", cmd, args...)
}
func runPrintInDir(dir string, cmd string, args ...string) {
if debug {
t0 := time.Now()
log.Println("runPrint:", cmd, strings.Join(args, " "))
@@ -972,7 +890,6 @@ func runPrintInDir(dir string, cmd string, args ...string) {
ecmd := exec.Command(cmd, args...)
ecmd.Stdout = os.Stdout
ecmd.Stderr = os.Stderr
ecmd.Dir = dir
err := ecmd.Run()
if err != nil {
log.Fatal(err)
@@ -1153,51 +1070,14 @@ func macosCodesign(file string) {
}
}
func windowsCodesign(file string) {
st := "signtool.exe"
if path := os.Getenv("CODESIGN_SIGNTOOL"); path != "" {
st = path
}
for i, algo := range []string{"sha1", "sha256"} {
args := []string{"sign", "/fd", algo}
if f := os.Getenv("CODESIGN_CERTIFICATE_FILE"); f != "" {
args = append(args, "/f", f)
}
if p := os.Getenv("CODESIGN_CERTIFICATE_PASSWORD"); p != "" {
args = append(args, "/p", p)
}
if tr := os.Getenv("CODESIGN_TIMESTAMP_SERVER"); tr != "" {
switch algo {
case "sha256":
args = append(args, "/tr", tr, "/td", algo)
default:
args = append(args, "/t", tr)
}
}
if i > 0 {
args = append(args, "/as")
}
args = append(args, file)
bs, err := runError(st, args...)
if err != nil {
log.Println("Codesign: signing failed:", string(bs))
return
}
log.Println("Codesign: successfully signed", file, "using", algo)
}
}
func metalint() {
lazyRebuildAssets()
runPrint(goCmd, "test", "-run", "Metalint", "./meta")
runPrint("go", "test", "-run", "Metalint", "./meta")
}
func metalintShort() {
lazyRebuildAssets()
runPrint(goCmd, "test", "-short", "-run", "Metalint", "./meta")
runPrint("go", "test", "-short", "-run", "Metalint", "./meta")
}
func temporaryBuildDir() (string, error) {
@@ -1226,6 +1106,90 @@ func temporaryBuildDir() (string, error) {
return filepath.Join(tmpDir, base), nil
}
func buildGOPATH(gopath string) error {
pkg := filepath.Join(gopath, "src/github.com/syncthing/syncthing")
dirs := []string{"cmd", "lib", "meta", "script", "test", "vendor"}
if debug {
t0 := time.Now()
log.Println("build temporary GOPATH in", gopath)
defer func() {
log.Println("... in", time.Since(t0))
}()
}
// Walk the sources and copy the files into the temporary GOPATH.
// Remember which files are supposed to be present so we can clean
// out everything else in the next step. The copyFile() step will
// only actually copy the file if it doesn't exist or the contents
// differ.
exists := map[string]struct{}{}
for _, dir := range dirs {
err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
dst := filepath.Join(pkg, path)
exists[dst] = struct{}{}
if err := copyFile(path, dst, info.Mode()); err != nil {
return err
}
return nil
})
if err != nil {
return err
}
}
// Walk the temporary GOPATH and remove any files that we wouldn't
// have copied there in the previous step.
filepath.Walk(pkg, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
if _, ok := exists[path]; !ok {
os.Remove(path)
}
return nil
})
return nil
}
func gopath() string {
if gopath := os.Getenv("GOPATH"); gopath != "" {
// The env var is set, use that.
return gopath
}
// Ask Go what it thinks.
bs, err := runError("go", "env", "GOPATH")
if err != nil {
return ""
}
// We got something. Check if we are in fact available in that location.
gopath := string(bs)
if _, err := os.Stat(filepath.Join(gopath, "src/github.com/syncthing/syncthing/build.go")); err == nil {
// That seems to be the gopath.
return gopath
}
// The gopath is not valid.
return ""
}
func (t target) BinaryName() string {
if goos == "windows" {
return t.binaryName + ".exe"

View File

@@ -48,6 +48,9 @@ case "${1:-default}" in
;;
test)
ulimit -t 600 &>/dev/null || true
ulimit -d 512000 &>/dev/null || true
ulimit -m 512000 &>/dev/null || true
LOGGER_DISCARD=1 build test
;;
@@ -59,8 +62,8 @@ case "${1:-default}" in
go run script/authors.go
build transifex
pushd man ; ./refresh.sh ; popd
git add -A gui man AUTHORS
git commit -m 'gui, man, authors: Update docs, translations, and contributors'
git add -A gui man
git commit -m 'gui, man: Update docs & translations'
;;
noupgrade)
@@ -91,6 +94,9 @@ case "${1:-default}" in
;;
test-xunit)
ulimit -t 600 &>/dev/null || true
ulimit -d 512000 &>/dev/null || true
ulimit -m 512000 &>/dev/null || true
(GOPATH="$(pwd)/Godeps/_workspace:$GOPATH" go test -v -race ./lib/... ./cmd/... || true) > tests.out
go2xunit -output tests.xml -fail < tests.out

View File

@@ -92,7 +92,7 @@ loop:
return t1.Sub(t0)
}
// report stops the given process and reports on its resource usage in two
// report stops the given process and reports on it's resource usage in two
// ways: human readable to stderr, and CSV to stdout.
func report(p *rc.Process, wallTime time.Duration) {
sv, err := p.SystemVersion()

19
cmd/stcli/LICENSE Normal file
View File

@@ -0,0 +1,19 @@
Copyright (C) 2014 Audrius Butkevičius
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
- The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@@ -1,95 +1,115 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// Copyright (C) 2014 Audrius Butkevičius
package main
import (
"bytes"
"context"
"crypto/tls"
"fmt"
"net"
"net/http"
"strings"
"github.com/syncthing/syncthing/lib/config"
"github.com/AudriusButkevicius/cli"
)
type APIClient struct {
http.Client
cfg config.GUIConfiguration
apikey string
httpClient http.Client
endpoint string
apikey string
username string
password string
id string
csrf string
}
func getClient(cfg config.GUIConfiguration) *APIClient {
var instance *APIClient
func getClient(c *cli.Context) *APIClient {
if instance != nil {
return instance
}
endpoint := c.GlobalString("endpoint")
if !strings.HasPrefix(endpoint, "http") {
endpoint = "http://" + endpoint
}
httpClient := http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
DialContext: func(_ context.Context, _, _ string) (net.Conn, error) {
return net.Dial(cfg.Network(), cfg.Address())
InsecureSkipVerify: c.GlobalBool("insecure"),
},
},
}
return &APIClient{
Client: httpClient,
cfg: cfg,
apikey: cfg.APIKey,
client := APIClient{
httpClient: httpClient,
endpoint: endpoint,
apikey: c.GlobalString("apikey"),
username: c.GlobalString("username"),
password: c.GlobalString("password"),
}
}
func (c *APIClient) Endpoint() string {
if c.cfg.Network() == "unix" {
return "http://unix/"
}
url := c.cfg.URL()
if !strings.HasSuffix(url, "/") {
url += "/"
}
return url
}
func (c *APIClient) Do(req *http.Request) (*http.Response, error) {
req.Header.Set("X-API-Key", c.apikey)
resp, err := c.Client.Do(req)
if err != nil {
return nil, err
}
return resp, checkResponse(resp)
}
func (c *APIClient) Get(url string) (*http.Response, error) {
request, err := http.NewRequest("GET", c.Endpoint()+"rest/"+url, nil)
if err != nil {
return nil, err
}
return c.Do(request)
}
func (c *APIClient) Post(url, body string) (*http.Response, error) {
request, err := http.NewRequest("POST", c.Endpoint()+"rest/"+url, bytes.NewBufferString(body))
if err != nil {
return nil, err
}
return c.Do(request)
}
func checkResponse(response *http.Response) error {
if response.StatusCode == 404 {
return fmt.Errorf("Invalid endpoint or API call")
} else if response.StatusCode == 403 {
return fmt.Errorf("Invalid API key")
} else if response.StatusCode != 200 {
data, err := responseToBArray(response)
if err != nil {
return err
if client.apikey == "" {
request, err := http.NewRequest("GET", client.endpoint, nil)
die(err)
response := client.handleRequest(request)
client.id = response.Header.Get("X-Syncthing-ID")
if client.id == "" {
die("Failed to get device ID")
}
body := strings.TrimSpace(string(data))
return fmt.Errorf("Unexpected HTTP status returned: %s\n%s", response.Status, body)
for _, item := range response.Cookies() {
if item.Name == "CSRF-Token-"+client.id[:5] {
client.csrf = item.Value
goto csrffound
}
}
die("Failed to get CSRF token")
csrffound:
}
return nil
instance = &client
return &client
}
func (client *APIClient) handleRequest(request *http.Request) *http.Response {
if client.apikey != "" {
request.Header.Set("X-API-Key", client.apikey)
}
if client.username != "" || client.password != "" {
request.SetBasicAuth(client.username, client.password)
}
if client.csrf != "" {
request.Header.Set("X-CSRF-Token-"+client.id[:5], client.csrf)
}
response, err := client.httpClient.Do(request)
die(err)
if response.StatusCode == 404 {
die("Invalid endpoint or API call")
} else if response.StatusCode == 401 {
die("Invalid username or password")
} else if response.StatusCode == 403 {
if client.apikey == "" {
die("Invalid CSRF token")
}
die("Invalid API key")
} else if response.StatusCode != 200 {
body := strings.TrimSpace(string(responseToBArray(response)))
if body != "" {
die(body)
}
die("Unknown HTTP status returned: " + response.Status)
}
return response
}
func httpGet(c *cli.Context, url string) *http.Response {
client := getClient(c)
request, err := http.NewRequest("GET", client.endpoint+"/rest/"+url, nil)
die(err)
return client.handleRequest(request)
}
func httpPost(c *cli.Context, url string, body string) *http.Response {
client := getClient(c)
request, err := http.NewRequest("POST", client.endpoint+"/rest/"+url, bytes.NewBufferString(body))
die(err)
return client.handleRequest(request)
}

188
cmd/stcli/cmd_devices.go Normal file
View File

@@ -0,0 +1,188 @@
// Copyright (C) 2014 Audrius Butkevičius
package main
import (
"fmt"
"strings"
"github.com/AudriusButkevicius/cli"
"github.com/syncthing/syncthing/lib/config"
)
func init() {
cliCommands = append(cliCommands, cli.Command{
Name: "devices",
HideHelp: true,
Usage: "Device command group",
Subcommands: []cli.Command{
{
Name: "list",
Usage: "List registered devices",
Requires: &cli.Requires{},
Action: devicesList,
},
{
Name: "add",
Usage: "Add a new device",
Requires: &cli.Requires{"device id", "device name?"},
Action: devicesAdd,
},
{
Name: "remove",
Usage: "Remove an existing device",
Requires: &cli.Requires{"device id"},
Action: devicesRemove,
},
{
Name: "get",
Usage: "Get a property of a device",
Requires: &cli.Requires{"device id", "property"},
Action: devicesGet,
},
{
Name: "set",
Usage: "Set a property of a device",
Requires: &cli.Requires{"device id", "property", "value..."},
Action: devicesSet,
},
},
})
}
func devicesList(c *cli.Context) {
cfg := getConfig(c)
first := true
writer := newTableWriter()
for _, device := range cfg.Devices {
if !first {
fmt.Fprintln(writer)
}
fmt.Fprintln(writer, "ID:\t", device.DeviceID, "\t")
fmt.Fprintln(writer, "Name:\t", device.Name, "\t(name)")
fmt.Fprintln(writer, "Address:\t", strings.Join(device.Addresses, " "), "\t(address)")
fmt.Fprintln(writer, "Compression:\t", device.Compression, "\t(compression)")
fmt.Fprintln(writer, "Certificate name:\t", device.CertName, "\t(certname)")
fmt.Fprintln(writer, "Introducer:\t", device.Introducer, "\t(introducer)")
first = false
}
writer.Flush()
}
func devicesAdd(c *cli.Context) {
nid := c.Args()[0]
id := parseDeviceID(nid)
newDevice := config.DeviceConfiguration{
DeviceID: id,
Name: nid,
Addresses: []string{"dynamic"},
}
if len(c.Args()) > 1 {
newDevice.Name = c.Args()[1]
}
if len(c.Args()) > 2 {
addresses := c.Args()[2:]
for _, item := range addresses {
if item == "dynamic" {
continue
}
validAddress(item)
}
newDevice.Addresses = addresses
}
cfg := getConfig(c)
for _, device := range cfg.Devices {
if device.DeviceID == id {
die("Device " + nid + " already exists")
}
}
cfg.Devices = append(cfg.Devices, newDevice)
setConfig(c, cfg)
}
func devicesRemove(c *cli.Context) {
nid := c.Args()[0]
id := parseDeviceID(nid)
if nid == getMyID(c) {
die("Cannot remove yourself")
}
cfg := getConfig(c)
for i, device := range cfg.Devices {
if device.DeviceID == id {
last := len(cfg.Devices) - 1
cfg.Devices[i] = cfg.Devices[last]
cfg.Devices = cfg.Devices[:last]
setConfig(c, cfg)
return
}
}
die("Device " + nid + " not found")
}
func devicesGet(c *cli.Context) {
nid := c.Args()[0]
id := parseDeviceID(nid)
arg := c.Args()[1]
cfg := getConfig(c)
for _, device := range cfg.Devices {
if device.DeviceID != id {
continue
}
switch strings.ToLower(arg) {
case "name":
fmt.Println(device.Name)
case "address":
fmt.Println(strings.Join(device.Addresses, "\n"))
case "compression":
fmt.Println(device.Compression.String())
case "certname":
fmt.Println(device.CertName)
case "introducer":
fmt.Println(device.Introducer)
default:
die("Invalid property: " + arg + "\nAvailable properties: name, address, compression, certname, introducer")
}
return
}
die("Device " + nid + " not found")
}
func devicesSet(c *cli.Context) {
nid := c.Args()[0]
id := parseDeviceID(nid)
arg := c.Args()[1]
config := getConfig(c)
for i, device := range config.Devices {
if device.DeviceID != id {
continue
}
switch strings.ToLower(arg) {
case "name":
config.Devices[i].Name = strings.Join(c.Args()[2:], " ")
case "address":
for _, item := range c.Args()[2:] {
if item == "dynamic" {
continue
}
validAddress(item)
}
config.Devices[i].Addresses = c.Args()[2:]
case "compression":
err := config.Devices[i].Compression.UnmarshalText([]byte(c.Args()[2]))
die(err)
case "certname":
config.Devices[i].CertName = strings.Join(c.Args()[2:], " ")
case "introducer":
config.Devices[i].Introducer = parseBool(c.Args()[2])
default:
die("Invalid property: " + arg + "\nAvailable properties: name, address, compression, certname, introducer")
}
setConfig(c, config)
return
}
die("Device " + nid + " not found")
}

67
cmd/stcli/cmd_errors.go Normal file
View File

@@ -0,0 +1,67 @@
// Copyright (C) 2014 Audrius Butkevičius
package main
import (
"encoding/json"
"fmt"
"strings"
"github.com/AudriusButkevicius/cli"
)
func init() {
cliCommands = append(cliCommands, cli.Command{
Name: "errors",
HideHelp: true,
Usage: "Error command group",
Subcommands: []cli.Command{
{
Name: "show",
Usage: "Show pending errors",
Requires: &cli.Requires{},
Action: errorsShow,
},
{
Name: "push",
Usage: "Push an error to active clients",
Requires: &cli.Requires{"error message..."},
Action: errorsPush,
},
{
Name: "clear",
Usage: "Clear pending errors",
Requires: &cli.Requires{},
Action: wrappedHTTPPost("system/error/clear"),
},
},
})
}
func errorsShow(c *cli.Context) {
response := httpGet(c, "system/error")
var data map[string][]map[string]interface{}
json.Unmarshal(responseToBArray(response), &data)
writer := newTableWriter()
for _, item := range data["errors"] {
time := item["time"].(string)[:19]
time = strings.Replace(time, "T", " ", 1)
err := item["error"].(string)
err = strings.TrimSpace(err)
fmt.Fprintln(writer, time+":\t"+err)
}
writer.Flush()
}
func errorsPush(c *cli.Context) {
err := strings.Join(c.Args(), " ")
response := httpPost(c, "system/error", strings.TrimSpace(err))
if response.StatusCode != 200 {
err = fmt.Sprint("Failed to push error\nStatus code: ", response.StatusCode)
body := string(responseToBArray(response))
if body != "" {
err += "\nBody: " + body
}
die(err)
}
}

351
cmd/stcli/cmd_folders.go Normal file
View File

@@ -0,0 +1,351 @@
// Copyright (C) 2014 Audrius Butkevičius
package main
import (
"fmt"
"path/filepath"
"strings"
"github.com/AudriusButkevicius/cli"
"github.com/syncthing/syncthing/lib/config"
)
func init() {
cliCommands = append(cliCommands, cli.Command{
Name: "folders",
HideHelp: true,
Usage: "Folder command group",
Subcommands: []cli.Command{
{
Name: "list",
Usage: "List available folders",
Requires: &cli.Requires{},
Action: foldersList,
},
{
Name: "add",
Usage: "Add a new folder",
Requires: &cli.Requires{"folder id", "directory"},
Action: foldersAdd,
},
{
Name: "remove",
Usage: "Remove an existing folder",
Requires: &cli.Requires{"folder id"},
Action: foldersRemove,
},
{
Name: "override",
Usage: "Override changes from other nodes for a master folder",
Requires: &cli.Requires{"folder id"},
Action: foldersOverride,
},
{
Name: "get",
Usage: "Get a property of a folder",
Requires: &cli.Requires{"folder id", "property"},
Action: foldersGet,
},
{
Name: "set",
Usage: "Set a property of a folder",
Requires: &cli.Requires{"folder id", "property", "value..."},
Action: foldersSet,
},
{
Name: "unset",
Usage: "Unset a property of a folder",
Requires: &cli.Requires{"folder id", "property"},
Action: foldersUnset,
},
{
Name: "devices",
Usage: "Folder devices command group",
HideHelp: true,
Subcommands: []cli.Command{
{
Name: "list",
Usage: "List of devices which the folder is shared with",
Requires: &cli.Requires{"folder id"},
Action: foldersDevicesList,
},
{
Name: "add",
Usage: "Share a folder with a device",
Requires: &cli.Requires{"folder id", "device id"},
Action: foldersDevicesAdd,
},
{
Name: "remove",
Usage: "Unshare a folder with a device",
Requires: &cli.Requires{"folder id", "device id"},
Action: foldersDevicesRemove,
},
{
Name: "clear",
Usage: "Unshare a folder with all devices",
Requires: &cli.Requires{"folder id"},
Action: foldersDevicesClear,
},
},
},
},
})
}
func foldersList(c *cli.Context) {
cfg := getConfig(c)
first := true
writer := newTableWriter()
for _, folder := range cfg.Folders {
if !first {
fmt.Fprintln(writer)
}
fmt.Fprintln(writer, "ID:\t", folder.ID, "\t")
fmt.Fprintln(writer, "Path:\t", folder.RawPath, "\t(directory)")
fmt.Fprintln(writer, "Folder type:\t", folder.Type, "\t(type)")
fmt.Fprintln(writer, "Ignore permissions:\t", folder.IgnorePerms, "\t(permissions)")
fmt.Fprintln(writer, "Rescan interval in seconds:\t", folder.RescanIntervalS, "\t(rescan)")
if folder.Versioning.Type != "" {
fmt.Fprintln(writer, "Versioning:\t", folder.Versioning.Type, "\t(versioning)")
for key, value := range folder.Versioning.Params {
fmt.Fprintf(writer, "Versioning %s:\t %s \t(versioning-%s)\n", key, value, key)
}
}
first = false
}
writer.Flush()
}
func foldersAdd(c *cli.Context) {
cfg := getConfig(c)
abs, err := filepath.Abs(c.Args()[1])
die(err)
folder := config.FolderConfiguration{
ID: c.Args()[0],
RawPath: filepath.Clean(abs),
}
cfg.Folders = append(cfg.Folders, folder)
setConfig(c, cfg)
}
func foldersRemove(c *cli.Context) {
cfg := getConfig(c)
rid := c.Args()[0]
for i, folder := range cfg.Folders {
if folder.ID == rid {
last := len(cfg.Folders) - 1
cfg.Folders[i] = cfg.Folders[last]
cfg.Folders = cfg.Folders[:last]
setConfig(c, cfg)
return
}
}
die("Folder " + rid + " not found")
}
func foldersOverride(c *cli.Context) {
cfg := getConfig(c)
rid := c.Args()[0]
for _, folder := range cfg.Folders {
if folder.ID == rid && folder.Type == config.FolderTypeSendOnly {
response := httpPost(c, "db/override", "")
if response.StatusCode != 200 {
err := fmt.Sprint("Failed to override changes\nStatus code: ", response.StatusCode)
body := string(responseToBArray(response))
if body != "" {
err += "\nBody: " + body
}
die(err)
}
return
}
}
die("Folder " + rid + " not found or folder not master")
}
func foldersGet(c *cli.Context) {
cfg := getConfig(c)
rid := c.Args()[0]
arg := strings.ToLower(c.Args()[1])
for _, folder := range cfg.Folders {
if folder.ID != rid {
continue
}
if strings.HasPrefix(arg, "versioning-") {
arg = arg[11:]
value, ok := folder.Versioning.Params[arg]
if ok {
fmt.Println(value)
return
}
die("Versioning property " + c.Args()[1][11:] + " not found")
}
switch arg {
case "directory":
fmt.Println(folder.RawPath)
case "type":
fmt.Println(folder.Type)
case "permissions":
fmt.Println(folder.IgnorePerms)
case "rescan":
fmt.Println(folder.RescanIntervalS)
case "versioning":
if folder.Versioning.Type != "" {
fmt.Println(folder.Versioning.Type)
}
default:
die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, type, permissions, versioning, versioning-<key>")
}
return
}
die("Folder " + rid + " not found")
}
func foldersSet(c *cli.Context) {
rid := c.Args()[0]
arg := strings.ToLower(c.Args()[1])
val := strings.Join(c.Args()[2:], " ")
cfg := getConfig(c)
for i, folder := range cfg.Folders {
if folder.ID != rid {
continue
}
if strings.HasPrefix(arg, "versioning-") {
cfg.Folders[i].Versioning.Params[arg[11:]] = val
setConfig(c, cfg)
return
}
switch arg {
case "directory":
cfg.Folders[i].RawPath = val
case "type":
var t config.FolderType
if err := t.UnmarshalText([]byte(val)); err != nil {
die("Invalid folder type: " + err.Error())
}
cfg.Folders[i].Type = t
case "permissions":
cfg.Folders[i].IgnorePerms = parseBool(val)
case "rescan":
cfg.Folders[i].RescanIntervalS = parseInt(val)
case "versioning":
cfg.Folders[i].Versioning.Type = val
default:
die("Invalid property: " + c.Args()[1] + "\nAvailable properties: directory, master, permissions, versioning, versioning-<key>")
}
setConfig(c, cfg)
return
}
die("Folder " + rid + " not found")
}
func foldersUnset(c *cli.Context) {
rid := c.Args()[0]
arg := strings.ToLower(c.Args()[1])
cfg := getConfig(c)
for i, folder := range cfg.Folders {
if folder.ID != rid {
continue
}
if strings.HasPrefix(arg, "versioning-") {
arg = arg[11:]
if _, ok := folder.Versioning.Params[arg]; ok {
delete(cfg.Folders[i].Versioning.Params, arg)
setConfig(c, cfg)
return
}
die("Versioning property " + c.Args()[1][11:] + " not found")
}
switch arg {
case "versioning":
cfg.Folders[i].Versioning.Type = ""
cfg.Folders[i].Versioning.Params = make(map[string]string)
default:
die("Invalid property: " + c.Args()[1] + "\nAvailable properties: versioning, versioning-<key>")
}
setConfig(c, cfg)
return
}
die("Folder " + rid + " not found")
}
func foldersDevicesList(c *cli.Context) {
rid := c.Args()[0]
cfg := getConfig(c)
for _, folder := range cfg.Folders {
if folder.ID != rid {
continue
}
for _, device := range folder.Devices {
fmt.Println(device.DeviceID)
}
return
}
die("Folder " + rid + " not found")
}
func foldersDevicesAdd(c *cli.Context) {
rid := c.Args()[0]
nid := parseDeviceID(c.Args()[1])
cfg := getConfig(c)
for i, folder := range cfg.Folders {
if folder.ID != rid {
continue
}
for _, device := range folder.Devices {
if device.DeviceID == nid {
die("Device " + c.Args()[1] + " is already part of this folder")
}
}
for _, device := range cfg.Devices {
if device.DeviceID == nid {
cfg.Folders[i].Devices = append(folder.Devices, config.FolderDeviceConfiguration{
DeviceID: device.DeviceID,
})
setConfig(c, cfg)
return
}
}
die("Device " + c.Args()[1] + " not found in device list")
}
die("Folder " + rid + " not found")
}
func foldersDevicesRemove(c *cli.Context) {
rid := c.Args()[0]
nid := parseDeviceID(c.Args()[1])
cfg := getConfig(c)
for ri, folder := range cfg.Folders {
if folder.ID != rid {
continue
}
for ni, device := range folder.Devices {
if device.DeviceID == nid {
last := len(folder.Devices) - 1
cfg.Folders[ri].Devices[ni] = folder.Devices[last]
cfg.Folders[ri].Devices = cfg.Folders[ri].Devices[:last]
setConfig(c, cfg)
return
}
}
die("Device " + c.Args()[1] + " not found")
}
die("Folder " + rid + " not found")
}
func foldersDevicesClear(c *cli.Context) {
rid := c.Args()[0]
cfg := getConfig(c)
for i, folder := range cfg.Folders {
if folder.ID != rid {
continue
}
cfg.Folders[i].Devices = []config.FolderDeviceConfiguration{}
setConfig(c, cfg)
return
}
die("Folder " + rid + " not found")
}

78
cmd/stcli/cmd_general.go Normal file
View File

@@ -0,0 +1,78 @@
// Copyright (C) 2014 Audrius Butkevičius
package main
import (
"encoding/json"
"fmt"
"github.com/AudriusButkevicius/cli"
)
func init() {
cliCommands = append(cliCommands, []cli.Command{
{
Name: "id",
Usage: "Get ID of the Syncthing client",
Requires: &cli.Requires{},
Action: generalID,
},
{
Name: "status",
Usage: "Configuration status, whether or not a restart is required for changes to take effect",
Requires: &cli.Requires{},
Action: generalStatus,
},
{
Name: "restart",
Usage: "Restart syncthing",
Requires: &cli.Requires{},
Action: wrappedHTTPPost("system/restart"),
},
{
Name: "shutdown",
Usage: "Shutdown syncthing",
Requires: &cli.Requires{},
Action: wrappedHTTPPost("system/shutdown"),
},
{
Name: "reset",
Usage: "Reset syncthing deleting all folders and devices",
Requires: &cli.Requires{},
Action: wrappedHTTPPost("system/reset"),
},
{
Name: "upgrade",
Usage: "Upgrade syncthing (if a newer version is available)",
Requires: &cli.Requires{},
Action: wrappedHTTPPost("system/upgrade"),
},
{
Name: "version",
Usage: "Syncthing client version",
Requires: &cli.Requires{},
Action: generalVersion,
},
}...)
}
func generalID(c *cli.Context) {
fmt.Println(getMyID(c))
}
func generalStatus(c *cli.Context) {
response := httpGet(c, "system/config/insync")
var status struct{ ConfigInSync bool }
json.Unmarshal(responseToBArray(response), &status)
if !status.ConfigInSync {
die("Config out of sync")
}
fmt.Println("Config in sync")
}
func generalVersion(c *cli.Context) {
response := httpGet(c, "system/version")
version := make(map[string]interface{})
json.Unmarshal(responseToBArray(response), &version)
prettyPrintJSON(version)
}

127
cmd/stcli/cmd_gui.go Normal file
View File

@@ -0,0 +1,127 @@
// Copyright (C) 2014 Audrius Butkevičius
package main
import (
"fmt"
"strings"
"github.com/AudriusButkevicius/cli"
)
func init() {
cliCommands = append(cliCommands, cli.Command{
Name: "gui",
HideHelp: true,
Usage: "GUI command group",
Subcommands: []cli.Command{
{
Name: "dump",
Usage: "Show all GUI configuration settings",
Requires: &cli.Requires{},
Action: guiDump,
},
{
Name: "get",
Usage: "Get a GUI configuration setting",
Requires: &cli.Requires{"setting"},
Action: guiGet,
},
{
Name: "set",
Usage: "Set a GUI configuration setting",
Requires: &cli.Requires{"setting", "value"},
Action: guiSet,
},
{
Name: "unset",
Usage: "Unset a GUI configuration setting",
Requires: &cli.Requires{"setting"},
Action: guiUnset,
},
},
})
}
func guiDump(c *cli.Context) {
cfg := getConfig(c).GUI
writer := newTableWriter()
fmt.Fprintln(writer, "Enabled:\t", cfg.Enabled, "\t(enabled)")
fmt.Fprintln(writer, "Use HTTPS:\t", cfg.UseTLS(), "\t(tls)")
fmt.Fprintln(writer, "Listen Addresses:\t", cfg.Address(), "\t(address)")
if cfg.User != "" {
fmt.Fprintln(writer, "Authentication User:\t", cfg.User, "\t(username)")
fmt.Fprintln(writer, "Authentication Password:\t", cfg.Password, "\t(password)")
}
if cfg.APIKey != "" {
fmt.Fprintln(writer, "API Key:\t", cfg.APIKey, "\t(apikey)")
}
writer.Flush()
}
func guiGet(c *cli.Context) {
cfg := getConfig(c).GUI
arg := c.Args()[0]
switch strings.ToLower(arg) {
case "enabled":
fmt.Println(cfg.Enabled)
case "tls":
fmt.Println(cfg.UseTLS())
case "address":
fmt.Println(cfg.Address())
case "user":
if cfg.User != "" {
fmt.Println(cfg.User)
}
case "password":
if cfg.User != "" {
fmt.Println(cfg.Password)
}
case "apikey":
if cfg.APIKey != "" {
fmt.Println(cfg.APIKey)
}
default:
die("Invalid setting: " + arg + "\nAvailable settings: enabled, tls, address, user, password, apikey")
}
}
func guiSet(c *cli.Context) {
cfg := getConfig(c)
arg := c.Args()[0]
val := c.Args()[1]
switch strings.ToLower(arg) {
case "enabled":
cfg.GUI.Enabled = parseBool(val)
case "tls":
cfg.GUI.RawUseTLS = parseBool(val)
case "address":
validAddress(val)
cfg.GUI.RawAddress = val
case "user":
cfg.GUI.User = val
case "password":
cfg.GUI.Password = val
case "apikey":
cfg.GUI.APIKey = val
default:
die("Invalid setting: " + arg + "\nAvailable settings: enabled, tls, address, user, password, apikey")
}
setConfig(c, cfg)
}
func guiUnset(c *cli.Context) {
cfg := getConfig(c)
arg := c.Args()[0]
switch strings.ToLower(arg) {
case "user":
cfg.GUI.User = ""
case "password":
cfg.GUI.Password = ""
case "apikey":
cfg.GUI.APIKey = ""
default:
die("Invalid setting: " + arg + "\nAvailable settings: user, password, apikey")
}
setConfig(c, cfg)
}

173
cmd/stcli/cmd_options.go Normal file
View File

@@ -0,0 +1,173 @@
// Copyright (C) 2014 Audrius Butkevičius
package main
import (
"fmt"
"strings"
"github.com/AudriusButkevicius/cli"
)
func init() {
cliCommands = append(cliCommands, cli.Command{
Name: "options",
HideHelp: true,
Usage: "Options command group",
Subcommands: []cli.Command{
{
Name: "dump",
Usage: "Show all Syncthing option settings",
Requires: &cli.Requires{},
Action: optionsDump,
},
{
Name: "get",
Usage: "Get a Syncthing option setting",
Requires: &cli.Requires{"setting"},
Action: optionsGet,
},
{
Name: "set",
Usage: "Set a Syncthing option setting",
Requires: &cli.Requires{"setting", "value..."},
Action: optionsSet,
},
},
})
}
func optionsDump(c *cli.Context) {
cfg := getConfig(c).Options
writer := newTableWriter()
fmt.Fprintln(writer, "Sync protocol listen addresses:\t", strings.Join(cfg.ListenAddresses, " "), "\t(addresses)")
fmt.Fprintln(writer, "Global discovery enabled:\t", cfg.GlobalAnnEnabled, "\t(globalannenabled)")
fmt.Fprintln(writer, "Global discovery servers:\t", strings.Join(cfg.GlobalAnnServers, " "), "\t(globalannserver)")
fmt.Fprintln(writer, "Local discovery enabled:\t", cfg.LocalAnnEnabled, "\t(localannenabled)")
fmt.Fprintln(writer, "Local discovery port:\t", cfg.LocalAnnPort, "\t(localannport)")
fmt.Fprintln(writer, "Outgoing rate limit in KiB/s:\t", cfg.MaxSendKbps, "\t(maxsend)")
fmt.Fprintln(writer, "Incoming rate limit in KiB/s:\t", cfg.MaxRecvKbps, "\t(maxrecv)")
fmt.Fprintln(writer, "Reconnect interval in seconds:\t", cfg.ReconnectIntervalS, "\t(reconnect)")
fmt.Fprintln(writer, "Start browser:\t", cfg.StartBrowser, "\t(browser)")
fmt.Fprintln(writer, "Enable UPnP:\t", cfg.NATEnabled, "\t(nat)")
fmt.Fprintln(writer, "UPnP Lease in minutes:\t", cfg.NATLeaseM, "\t(natlease)")
fmt.Fprintln(writer, "UPnP Renewal period in minutes:\t", cfg.NATRenewalM, "\t(natrenew)")
fmt.Fprintln(writer, "Restart on Wake Up:\t", cfg.RestartOnWakeup, "\t(wake)")
reporting := "unrecognized value"
switch cfg.URAccepted {
case -1:
reporting = "false"
case 0:
reporting = "undecided/false"
case 1:
reporting = "true"
}
fmt.Fprintln(writer, "Anonymous usage reporting:\t", reporting, "\t(reporting)")
writer.Flush()
}
func optionsGet(c *cli.Context) {
cfg := getConfig(c).Options
arg := c.Args()[0]
switch strings.ToLower(arg) {
case "address":
fmt.Println(strings.Join(cfg.ListenAddresses, "\n"))
case "globalannenabled":
fmt.Println(cfg.GlobalAnnEnabled)
case "globalannservers":
fmt.Println(strings.Join(cfg.GlobalAnnServers, "\n"))
case "localannenabled":
fmt.Println(cfg.LocalAnnEnabled)
case "localannport":
fmt.Println(cfg.LocalAnnPort)
case "maxsend":
fmt.Println(cfg.MaxSendKbps)
case "maxrecv":
fmt.Println(cfg.MaxRecvKbps)
case "reconnect":
fmt.Println(cfg.ReconnectIntervalS)
case "browser":
fmt.Println(cfg.StartBrowser)
case "nat":
fmt.Println(cfg.NATEnabled)
case "natlease":
fmt.Println(cfg.NATLeaseM)
case "natrenew":
fmt.Println(cfg.NATRenewalM)
case "reporting":
switch cfg.URAccepted {
case -1:
fmt.Println("false")
case 0:
fmt.Println("undecided/false")
case 1:
fmt.Println("true")
default:
fmt.Println("unknown")
}
case "wake":
fmt.Println(cfg.RestartOnWakeup)
default:
die("Invalid setting: " + arg + "\nAvailable settings: address, globalannenabled, globalannserver, localannenabled, localannport, maxsend, maxrecv, reconnect, browser, upnp, upnplease, upnprenew, reporting, wake")
}
}
func optionsSet(c *cli.Context) {
config := getConfig(c)
arg := c.Args()[0]
val := c.Args()[1]
switch strings.ToLower(arg) {
case "address":
for _, item := range c.Args().Tail() {
validAddress(item)
}
config.Options.ListenAddresses = c.Args().Tail()
case "globalannenabled":
config.Options.GlobalAnnEnabled = parseBool(val)
case "globalannserver":
for _, item := range c.Args().Tail() {
validAddress(item)
}
config.Options.GlobalAnnServers = c.Args().Tail()
case "localannenabled":
config.Options.LocalAnnEnabled = parseBool(val)
case "localannport":
config.Options.LocalAnnPort = parsePort(val)
case "maxsend":
config.Options.MaxSendKbps = parseUint(val)
case "maxrecv":
config.Options.MaxRecvKbps = parseUint(val)
case "reconnect":
config.Options.ReconnectIntervalS = parseUint(val)
case "browser":
config.Options.StartBrowser = parseBool(val)
case "nat":
config.Options.NATEnabled = parseBool(val)
case "natlease":
config.Options.NATLeaseM = parseUint(val)
case "natrenew":
config.Options.NATRenewalM = parseUint(val)
case "reporting":
switch strings.ToLower(val) {
case "u", "undecided", "unset":
config.Options.URAccepted = 0
default:
boolvalue := parseBool(val)
if boolvalue {
config.Options.URAccepted = 1
} else {
config.Options.URAccepted = -1
}
}
case "wake":
config.Options.RestartOnWakeup = parseBool(val)
default:
die("Invalid setting: " + arg + "\nAvailable settings: address, globalannenabled, globalannserver, localannenabled, localannport, maxsend, maxrecv, reconnect, browser, upnp, upnplease, upnprenew, reporting, wake")
}
setConfig(c, config)
}

72
cmd/stcli/cmd_report.go Normal file
View File

@@ -0,0 +1,72 @@
// Copyright (C) 2014 Audrius Butkevičius
package main
import (
"encoding/json"
"fmt"
"github.com/AudriusButkevicius/cli"
)
func init() {
cliCommands = append(cliCommands, cli.Command{
Name: "report",
HideHelp: true,
Usage: "Reporting command group",
Subcommands: []cli.Command{
{
Name: "system",
Usage: "Report system state",
Requires: &cli.Requires{},
Action: reportSystem,
},
{
Name: "connections",
Usage: "Report about connections to other devices",
Requires: &cli.Requires{},
Action: reportConnections,
},
{
Name: "usage",
Usage: "Usage report",
Requires: &cli.Requires{},
Action: reportUsage,
},
},
})
}
func reportSystem(c *cli.Context) {
response := httpGet(c, "system/status")
data := make(map[string]interface{})
json.Unmarshal(responseToBArray(response), &data)
prettyPrintJSON(data)
}
func reportConnections(c *cli.Context) {
response := httpGet(c, "system/connections")
data := make(map[string]map[string]interface{})
json.Unmarshal(responseToBArray(response), &data)
var overall map[string]interface{}
for key, value := range data {
if key == "total" {
overall = value
continue
}
value["Device ID"] = key
prettyPrintJSON(value)
fmt.Println()
}
if overall != nil {
fmt.Println("=== Overall statistics ===")
prettyPrintJSON(overall)
}
}
func reportUsage(c *cli.Context) {
response := httpGet(c, "svc/report")
report := make(map[string]interface{})
json.Unmarshal(responseToBArray(response), &report)
prettyPrintJSON(report)
}

View File

@@ -1,60 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"fmt"
"strings"
"github.com/urfave/cli"
)
var errorsCommand = cli.Command{
Name: "errors",
HideHelp: true,
Usage: "Error command group",
Subcommands: []cli.Command{
{
Name: "show",
Usage: "Show pending errors",
Action: expects(0, dumpOutput("system/error")),
},
{
Name: "push",
Usage: "Push an error to active clients",
ArgsUsage: "[error message]",
Action: expects(1, errorsPush),
},
{
Name: "clear",
Usage: "Clear pending errors",
Action: expects(0, emptyPost("system/error/clear")),
},
},
}
func errorsPush(c *cli.Context) error {
client := c.App.Metadata["client"].(*APIClient)
errStr := strings.Join(c.Args(), " ")
response, err := client.Post("system/error", strings.TrimSpace(errStr))
if err != nil {
return err
}
if response.StatusCode != 200 {
errStr = fmt.Sprint("Failed to push error\nStatus code: ", response.StatusCode)
bytes, err := responseToBArray(response)
if err != nil {
return err
}
body := string(bytes)
if body != "" {
errStr += "\nBody: " + body
}
return fmt.Errorf(errStr)
}
return nil
}

31
cmd/stcli/labels.go Normal file
View File

@@ -0,0 +1,31 @@
// Copyright (C) 2014 Audrius Butkevičius
package main
var jsonAttributeLabels = map[string]string{
"folderMaxMiB": "Largest folder size in MiB",
"folderMaxFiles": "Largest folder file count",
"longVersion": "Long version",
"totMiB": "Total size in MiB",
"totFiles": "Total files",
"uniqueID": "Unique ID",
"numFolders": "Folder count",
"numDevices": "Device count",
"memoryUsageMiB": "Memory usage in MiB",
"memorySize": "Total memory in MiB",
"sha256Perf": "SHA256 Benchmark",
"At": "Last contacted",
"Completion": "Percent complete",
"InBytesTotal": "Total bytes received",
"OutBytesTotal": "Total bytes sent",
"ClientVersion": "Client version",
"alloc": "Memory allocated in bytes",
"sys": "Memory using in bytes",
"cpuPercent": "CPU load in percent",
"extAnnounceOK": "External announcments working",
"goroutines": "Number of Go routines",
"myID": "Client ID",
"tilde": "Tilde expands to",
"arch": "Architecture",
"os": "OS",
}

View File

@@ -1,192 +1,63 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// Copyright (C) 2014 Audrius Butkevičius
package main
import (
"bufio"
"crypto/tls"
"encoding/json"
"flag"
"log"
"os"
"reflect"
"strings"
"sort"
"github.com/AudriusButkevicius/recli"
"github.com/flynn-archive/go-shlex"
"github.com/mattn/go-isatty"
"github.com/pkg/errors"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/locations"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/urfave/cli"
"github.com/AudriusButkevicius/cli"
)
type ByAlphabet []cli.Command
func (a ByAlphabet) Len() int { return len(a) }
func (a ByAlphabet) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByAlphabet) Less(i, j int) bool { return a[i].Name < a[j].Name }
var cliCommands []cli.Command
func main() {
// This is somewhat a hack around a chicken and egg problem.
// We need to set the home directory and potentially other flags to know where the syncthing instance is running
// in order to get it's config ... which we then use to construct the actual CLI ... at which point it's too late
// to add flags there...
homeBaseDir := locations.GetBaseDir(locations.ConfigBaseDir)
guiCfg := config.GUIConfiguration{}
flags := flag.NewFlagSet("", flag.ContinueOnError)
flags.StringVar(&guiCfg.RawAddress, "gui-address", guiCfg.RawAddress, "Override GUI address (e.g. \"http://192.0.2.42:8443\")")
flags.StringVar(&guiCfg.APIKey, "gui-apikey", guiCfg.APIKey, "Override GUI API key")
flags.StringVar(&homeBaseDir, "home", homeBaseDir, "Set configuration directory")
// Implement the same flags at the lower CLI, with the same default values (pre-parse), but do nothing with them.
// This is so that we could reuse os.Args
fakeFlags := []cli.Flag{
cli.StringFlag{
Name: "gui-address",
Value: guiCfg.RawAddress,
Usage: "Override GUI address (e.g. \"http://192.0.2.42:8443\")",
},
cli.StringFlag{
Name: "gui-apikey",
Value: guiCfg.APIKey,
Usage: "Override GUI API key",
},
cli.StringFlag{
Name: "home",
Value: homeBaseDir,
Usage: "Set configuration directory",
},
}
// Do not print usage of these flags, and ignore errors as this can't understand plenty of things
flags.Usage = func() {}
_ = flags.Parse(os.Args[1:])
// Now if the API key and address is not provided (we are not connecting to a remote instance),
// try to rip it out of the config.
if guiCfg.RawAddress == "" && guiCfg.APIKey == "" {
// Update the base directory
err := locations.SetBaseDir(locations.ConfigBaseDir, homeBaseDir)
if err != nil {
log.Fatal(errors.Wrap(err, "setting home"))
}
// Load the certs and get the ID
cert, err := tls.LoadX509KeyPair(
locations.Get(locations.CertFile),
locations.Get(locations.KeyFile),
)
if err != nil {
log.Fatal(errors.Wrap(err, "reading device ID"))
}
myID := protocol.NewDeviceID(cert.Certificate[0])
// Load the config
cfg, err := config.Load(locations.Get(locations.ConfigFile), myID)
if err != nil {
log.Fatalln(errors.Wrap(err, "loading config"))
}
guiCfg = cfg.GUI()
} else if guiCfg.Address() == "" || guiCfg.APIKey == "" {
log.Fatalln("Both -gui-address and -gui-apikey should be specified")
}
if guiCfg.Address() == "" {
log.Fatalln("Could not find GUI Address")
}
if guiCfg.APIKey == "" {
log.Fatalln("Could not find GUI API key")
}
client := getClient(guiCfg)
cfg, err := getConfig(client)
original := cfg.Copy()
if err != nil {
log.Fatalln(errors.Wrap(err, "getting config"))
}
// Copy the config and set the default flags
recliCfg := recli.DefaultConfig
recliCfg.IDTag.Name = "xml"
recliCfg.SkipTag.Name = "json"
commands, err := recli.New(recliCfg).Construct(&cfg)
if err != nil {
log.Fatalln(errors.Wrap(err, "config reflect"))
}
// Construct the actual CLI
app := cli.NewApp()
app.Name = "stcli"
app.HelpName = app.Name
app.Author = "The Syncthing Authors"
app.Name = "syncthing-cli"
app.Author = "Audrius Butkevičius"
app.Email = "audrius.butkevicius@gmail.com"
app.Usage = "Syncthing command line interface"
app.Version = strings.Replace(build.LongVersion, "syncthing", app.Name, 1)
app.Flags = fakeFlags
app.Metadata = map[string]interface{}{
"client": client,
}
app.Commands = []cli.Command{
{
Name: "config",
HideHelp: true,
Usage: "Configuration modification command group",
Subcommands: commands,
app.Version = "0.1"
app.HideHelp = true
app.Flags = []cli.Flag{
cli.StringFlag{
Name: "endpoint, e",
Value: "http://127.0.0.1:8384",
Usage: "End point to connect to",
EnvVar: "STENDPOINT",
},
cli.StringFlag{
Name: "apikey, k",
Value: "",
Usage: "API Key",
EnvVar: "STAPIKEY",
},
cli.StringFlag{
Name: "username, u",
Value: "",
Usage: "Username",
EnvVar: "STUSERNAME",
},
cli.StringFlag{
Name: "password, p",
Value: "",
Usage: "Password",
EnvVar: "STPASSWORD",
},
cli.BoolFlag{
Name: "insecure, i",
Usage: "Do not verify SSL certificate",
EnvVar: "STINSECURE",
},
showCommand,
operationCommand,
errorsCommand,
}
tty := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd())
if !tty {
// Not a TTY, consume from stdin
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
input, err := shlex.Split(scanner.Text())
if err != nil {
log.Fatalln(errors.Wrap(err, "parsing input"))
}
if len(input) == 0 {
continue
}
err = app.Run(append(os.Args, input...))
if err != nil {
log.Fatalln(err)
}
}
err = scanner.Err()
if err != nil {
log.Fatalln(err)
}
} else {
err = app.Run(os.Args)
if err != nil {
log.Fatalln(err)
}
}
if !reflect.DeepEqual(cfg, original) {
body, err := json.MarshalIndent(cfg, "", " ")
if err != nil {
log.Fatalln(err)
}
resp, err := client.Post("system/config", string(body))
if err != nil {
log.Fatalln(err)
}
if resp.StatusCode != 200 {
body, err := responseToBArray(resp)
if err != nil {
log.Fatalln(err)
}
log.Fatalln(string(body))
}
}
sort.Sort(ByAlphabet(cliCommands))
app.Commands = cliCommands
app.RunAndExitOnError()
}

View File

@@ -1,78 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"fmt"
"github.com/urfave/cli"
)
var operationCommand = cli.Command{
Name: "operations",
HideHelp: true,
Usage: "Operation command group",
Subcommands: []cli.Command{
{
Name: "restart",
Usage: "Restart syncthing",
Action: expects(0, emptyPost("system/restart")),
},
{
Name: "shutdown",
Usage: "Shutdown syncthing",
Action: expects(0, emptyPost("system/shutdown")),
},
{
Name: "reset",
Usage: "Reset syncthing deleting all folders and devices",
Action: expects(0, emptyPost("system/reset")),
},
{
Name: "upgrade",
Usage: "Upgrade syncthing (if a newer version is available)",
Action: expects(0, emptyPost("system/upgrade")),
},
{
Name: "folder-override",
Usage: "Override changes on folder (remote for sendonly, local for receiveonly)",
ArgsUsage: "[folder id]",
Action: expects(1, foldersOverride),
},
},
}
func foldersOverride(c *cli.Context) error {
client := c.App.Metadata["client"].(*APIClient)
cfg, err := getConfig(client)
if err != nil {
return err
}
rid := c.Args()[0]
for _, folder := range cfg.Folders {
if folder.ID == rid {
response, err := client.Post("db/override", "")
if err != nil {
return err
}
if response.StatusCode != 200 {
errStr := fmt.Sprint("Failed to override changes\nStatus code: ", response.StatusCode)
bytes, err := responseToBArray(response)
if err != nil {
return err
}
body := string(bytes)
if body != "" {
errStr += "\nBody: " + body
}
return fmt.Errorf(errStr)
}
return nil
}
}
return fmt.Errorf("Folder " + rid + " not found")
}

View File

@@ -1,44 +0,0 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"github.com/urfave/cli"
)
var showCommand = cli.Command{
Name: "show",
HideHelp: true,
Usage: "Show command group",
Subcommands: []cli.Command{
{
Name: "version",
Usage: "Show syncthing client version",
Action: expects(0, dumpOutput("system/version")),
},
{
Name: "config-status",
Usage: "Show configuration status, whether or not a restart is required for changes to take effect",
Action: expects(0, dumpOutput("system/config/insync")),
},
{
Name: "system",
Usage: "Show system status",
Action: expects(0, dumpOutput("system/status")),
},
{
Name: "connections",
Usage: "Report about connections to other devices",
Action: expects(0, dumpOutput("system/connections")),
},
{
Name: "usage",
Usage: "Show usage report",
Action: expects(0, dumpOutput("svc/report")),
},
},
}

View File

@@ -1,8 +1,4 @@
// Copyright (C) 2019 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// Copyright (C) 2014 Audrius Butkevičius
package main
@@ -12,37 +8,78 @@ import (
"io/ioutil"
"net/http"
"os"
"regexp"
"sort"
"strconv"
"strings"
"text/tabwriter"
"unicode"
"github.com/AudriusButkevicius/cli"
"github.com/syncthing/syncthing/lib/config"
"github.com/urfave/cli"
"github.com/syncthing/syncthing/lib/protocol"
)
func responseToBArray(response *http.Response) ([]byte, error) {
func responseToBArray(response *http.Response) []byte {
defer response.Body.Close()
bytes, err := ioutil.ReadAll(response.Body)
if err != nil {
return nil, err
die(err)
}
return bytes, response.Body.Close()
return bytes
}
func emptyPost(url string) cli.ActionFunc {
return func(c *cli.Context) error {
client := c.App.Metadata["client"].(*APIClient)
_, err := client.Post(url, "")
return err
func die(vals ...interface{}) {
if len(vals) > 1 || vals[0] != nil {
os.Stderr.WriteString(fmt.Sprintln(vals...))
os.Exit(1)
}
}
func dumpOutput(url string) cli.ActionFunc {
return func(c *cli.Context) error {
client := c.App.Metadata["client"].(*APIClient)
response, err := client.Get(url)
if err != nil {
return err
func wrappedHTTPPost(url string) func(c *cli.Context) {
return func(c *cli.Context) {
httpPost(c, url, "")
}
}
func prettyPrintJSON(json map[string]interface{}) {
writer := newTableWriter()
remap := make(map[string]interface{})
for k, v := range json {
key, ok := jsonAttributeLabels[k]
if !ok {
key = firstUpper(k)
}
return prettyPrintResponse(c, response)
remap[key] = v
}
jsonKeys := make([]string, 0, len(remap))
for key := range remap {
jsonKeys = append(jsonKeys, key)
}
sort.Strings(jsonKeys)
for _, k := range jsonKeys {
var value string
rvalue := remap[k]
switch rvalue.(type) {
case int, int16, int32, int64, uint, uint16, uint32, uint64, float32, float64:
value = fmt.Sprintf("%.0f", rvalue)
default:
value = fmt.Sprint(rvalue)
}
if value == "" {
continue
}
fmt.Fprintln(writer, k+":\t"+value)
}
writer.Flush()
}
func firstUpper(str string) string {
for i, v := range str {
return string(unicode.ToUpper(v)) + str[i+1:]
}
return ""
}
func newTableWriter() *tabwriter.Writer {
@@ -51,51 +88,78 @@ func newTableWriter() *tabwriter.Writer {
return writer
}
func getConfig(c *APIClient) (config.Configuration, error) {
cfg := config.Configuration{}
response, err := c.Get("system/config")
if err != nil {
return cfg, err
}
bytes, err := responseToBArray(response)
if err != nil {
return cfg, err
}
err = json.Unmarshal(bytes, &cfg)
if err == nil {
return cfg, err
}
return cfg, nil
func getMyID(c *cli.Context) string {
response := httpGet(c, "system/status")
data := make(map[string]interface{})
json.Unmarshal(responseToBArray(response), &data)
return data["myID"].(string)
}
func expects(n int, actionFunc cli.ActionFunc) cli.ActionFunc {
return func(ctx *cli.Context) error {
if ctx.NArg() != n {
plural := ""
if n != 1 {
plural = "s"
}
return fmt.Errorf("expected %d argument%s, got %d", n, plural, ctx.NArg())
}
return actionFunc(ctx)
func getConfig(c *cli.Context) config.Configuration {
response := httpGet(c, "system/config")
config := config.Configuration{}
json.Unmarshal(responseToBArray(response), &config)
return config
}
func setConfig(c *cli.Context, cfg config.Configuration) {
body, err := json.Marshal(cfg)
die(err)
response := httpPost(c, "system/config", string(body))
if response.StatusCode != 200 {
die("Unexpected status code", response.StatusCode)
}
}
func prettyPrintJSON(data interface{}) error {
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
return enc.Encode(data)
func parseBool(input string) bool {
val, err := strconv.ParseBool(input)
if err != nil {
die(input + " is not a valid value for a boolean")
}
return val
}
func prettyPrintResponse(c *cli.Context, response *http.Response) error {
bytes, err := responseToBArray(response)
func parseInt(input string) int {
val, err := strconv.ParseInt(input, 0, 64)
if err != nil {
return err
die(input + " is not a valid value for an integer")
}
var data interface{}
if err := json.Unmarshal(bytes, &data); err != nil {
return err
}
// TODO: Check flag for pretty print format
return prettyPrintJSON(data)
return int(val)
}
func parseUint(input string) int {
val, err := strconv.ParseUint(input, 0, 64)
if err != nil {
die(input + " is not a valid value for an unsigned integer")
}
return int(val)
}
func parsePort(input string) int {
port := parseUint(input)
if port < 1 || port > 65535 {
die(input + " is not a valid port\nExpected value between 1 and 65535")
}
return port
}
func validAddress(input string) {
tokens := strings.Split(input, ":")
if len(tokens) != 2 {
die(input + " is not a valid value for an address\nExpected format <ip or hostname>:<port>")
}
matched, err := regexp.MatchString("^[a-zA-Z0-9]+([-a-zA-Z0-9.]+[-a-zA-Z0-9]+)?$", tokens[0])
die(err)
if !matched {
die(input + " is not a valid value for an address\nExpected format <ip or hostname>:<port>")
}
parsePort(tokens[1])
}
func parseDeviceID(input string) protocol.DeviceID {
device, err := protocol.DeviceIDFromString(input)
if err != nil {
die(input + " is not a valid device id")
}
return device
}

View File

@@ -88,10 +88,10 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) chan er
}
rn, _ := filepath.Rel(dir, path)
if rn == "." {
if rn == "." || rn == ".stfolder" {
return nil
}
if rn == ".stversions" || rn == ".stfolder" {
if rn == ".stversions" {
return filepath.SkipDir
}

19
cmd/stdiscosrv/LICENSE Normal file
View File

@@ -0,0 +1,19 @@
Copyright (C) 2014-2015 The Discosrv Authors
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
- The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@@ -6,5 +6,33 @@ This is the global discovery server for the `syncthing` project.
Usage
-----
https://docs.syncthing.net/users/stdiscosrv.html
The discovery server supports `ql` and `postgres` backends.
Specify the backend via `-db-backend` and the database DSN via `-db-dsn`.
By default it will use in-memory `ql` backend. If you wish to persist the
information on disk between restarts in `ql`, specify a file DSN:
```bash
$ stdiscosrv -db-dsn="file:///var/run/stdiscosrv.db"
```
For `postgres`, you will need to create a database and a user with permissions
to create tables in it, then start the stdiscosrv as follows:
```bash
$ export STDISCOSRV_DB_DSN="postgres://user:password@localhost/databasename"
$ stdiscosrv -db-backend="postgres"
```
You can pass the DSN as command line option, but the value what you pass in will
be visible in most process managers, potentially exposing the database password
to other users.
In all cases, the appropriate tables and indexes will be created at first
startup. If it doesn't exit with an error, you're fine.
See `stdiscosrv -help` for other options.
##### Third-party attribution
[cznic/lldb](https://github.com/cznic/lldb), Copyright (C) 2014 The lldb Authors.

View File

@@ -1,411 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bytes"
"context"
"crypto/tls"
"encoding/json"
"encoding/pem"
"fmt"
"log"
"math/rand"
"net"
"net/http"
"net/url"
"strconv"
"strings"
"sync"
"time"
"github.com/syncthing/syncthing/lib/protocol"
)
// announcement is the format received from and sent to clients
type announcement struct {
Seen time.Time `json:"seen"`
Addresses []string `json:"addresses"`
}
type apiSrv struct {
addr string
cert tls.Certificate
db database
listener net.Listener
repl replicator // optional
useHTTP bool
mapsMut sync.Mutex
misses map[string]int32
}
type requestID int64
func (i requestID) String() string {
return fmt.Sprintf("%016x", int64(i))
}
type contextKey int
const idKey contextKey = iota
func newAPISrv(addr string, cert tls.Certificate, db database, repl replicator, useHTTP bool) *apiSrv {
return &apiSrv{
addr: addr,
cert: cert,
db: db,
repl: repl,
useHTTP: useHTTP,
misses: make(map[string]int32),
}
}
func (s *apiSrv) Serve() {
if s.useHTTP {
listener, err := net.Listen("tcp", s.addr)
if err != nil {
log.Println("Listen:", err)
return
}
s.listener = listener
} else {
tlsCfg := &tls.Config{
Certificates: []tls.Certificate{s.cert},
ClientAuth: tls.RequestClientCert,
SessionTicketsDisabled: true,
MinVersion: tls.VersionTLS12,
CipherSuites: []uint16{
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
},
}
tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
if err != nil {
log.Println("Listen:", err)
return
}
s.listener = tlsListener
}
http.HandleFunc("/", s.handler)
http.HandleFunc("/ping", handlePing)
srv := &http.Server{
ReadTimeout: httpReadTimeout,
WriteTimeout: httpWriteTimeout,
MaxHeaderBytes: httpMaxHeaderBytes,
}
if err := srv.Serve(s.listener); err != nil {
log.Println("Serve:", err)
}
}
var topCtx = context.Background()
func (s *apiSrv) handler(w http.ResponseWriter, req *http.Request) {
t0 := time.Now()
lw := NewLoggingResponseWriter(w)
defer func() {
diff := time.Since(t0)
apiRequestsSeconds.WithLabelValues(req.Method).Observe(diff.Seconds())
apiRequestsTotal.WithLabelValues(req.Method, strconv.Itoa(lw.statusCode)).Inc()
}()
reqID := requestID(rand.Int63())
ctx := context.WithValue(topCtx, idKey, reqID)
if debug {
log.Println(reqID, req.Method, req.URL)
}
var remoteIP net.IP
if s.useHTTP {
remoteIP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
} else {
addr, err := net.ResolveTCPAddr("tcp", req.RemoteAddr)
if err != nil {
log.Println("remoteAddr:", err)
lw.Header().Set("Retry-After", errorRetryAfterString())
http.Error(lw, "Internal Server Error", http.StatusInternalServerError)
apiRequestsTotal.WithLabelValues("no_remote_addr").Inc()
return
}
remoteIP = addr.IP
}
switch req.Method {
case "GET":
s.handleGET(ctx, lw, req)
case "POST":
s.handlePOST(ctx, remoteIP, lw, req)
default:
http.Error(lw, "Method Not Allowed", http.StatusMethodNotAllowed)
}
}
func (s *apiSrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http.Request) {
reqID := ctx.Value(idKey).(requestID)
deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
if err != nil {
if debug {
log.Println(reqID, "bad device param")
}
lookupRequestsTotal.WithLabelValues("bad_request").Inc()
w.Header().Set("Retry-After", errorRetryAfterString())
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
key := deviceID.String()
rec, err := s.db.get(key)
if err != nil {
// some sort of internal error
lookupRequestsTotal.WithLabelValues("internal_error").Inc()
w.Header().Set("Retry-After", errorRetryAfterString())
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
if len(rec.Addresses) == 0 {
lookupRequestsTotal.WithLabelValues("not_found").Inc()
s.mapsMut.Lock()
misses := s.misses[key]
if misses < rec.Misses {
misses = rec.Misses + 1
} else {
misses++
}
s.misses[key] = misses
s.mapsMut.Unlock()
if misses%notFoundMissesWriteInterval == 0 {
rec.Misses = misses
rec.Missed = time.Now().UnixNano()
rec.Addresses = nil
// rec.Seen retained from get
s.db.put(key, rec)
}
w.Header().Set("Retry-After", notFoundRetryAfterString(int(misses)))
http.Error(w, "Not Found", http.StatusNotFound)
return
}
lookupRequestsTotal.WithLabelValues("success").Inc()
bs, _ := json.Marshal(announcement{
Seen: time.Unix(0, rec.Seen),
Addresses: addressStrs(rec.Addresses),
})
w.Header().Set("Content-Type", "application/json")
w.Write(bs)
}
func (s *apiSrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.ResponseWriter, req *http.Request) {
reqID := ctx.Value(idKey).(requestID)
rawCert := certificateBytes(req)
if rawCert == nil {
if debug {
log.Println(reqID, "no certificates")
}
announceRequestsTotal.WithLabelValues("no_certificate").Inc()
w.Header().Set("Retry-After", errorRetryAfterString())
http.Error(w, "Forbidden", http.StatusForbidden)
return
}
var ann announcement
if err := json.NewDecoder(req.Body).Decode(&ann); err != nil {
if debug {
log.Println(reqID, "decode:", err)
}
announceRequestsTotal.WithLabelValues("bad_request").Inc()
w.Header().Set("Retry-After", errorRetryAfterString())
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
deviceID := protocol.NewDeviceID(rawCert)
addresses := fixupAddresses(remoteIP, ann.Addresses)
if len(addresses) == 0 {
announceRequestsTotal.WithLabelValues("bad_request").Inc()
w.Header().Set("Retry-After", errorRetryAfterString())
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
if err := s.handleAnnounce(remoteIP, deviceID, addresses); err != nil {
announceRequestsTotal.WithLabelValues("internal_error").Inc()
w.Header().Set("Retry-After", errorRetryAfterString())
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
announceRequestsTotal.WithLabelValues("success").Inc()
w.Header().Set("Reannounce-After", reannounceAfterString())
w.WriteHeader(http.StatusNoContent)
}
func (s *apiSrv) Stop() {
s.listener.Close()
}
func (s *apiSrv) handleAnnounce(remote net.IP, deviceID protocol.DeviceID, addresses []string) error {
key := deviceID.String()
now := time.Now()
expire := now.Add(addressExpiryTime).UnixNano()
dbAddrs := make([]DatabaseAddress, len(addresses))
for i := range addresses {
dbAddrs[i].Address = addresses[i]
dbAddrs[i].Expires = expire
}
seen := now.UnixNano()
if s.repl != nil {
s.repl.send(key, dbAddrs, seen)
}
return s.db.merge(key, dbAddrs, seen)
}
func handlePing(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(204)
}
func certificateBytes(req *http.Request) []byte {
if req.TLS != nil && len(req.TLS.PeerCertificates) > 0 {
return req.TLS.PeerCertificates[0].Raw
}
if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
bs := []byte(hdr)
// The certificate is in PEM format but with spaces for newlines. We
// need to reinstate the newlines for the PEM decoder. But we need to
// leave the spaces in the BEGIN and END lines - the first and last
// space - alone.
firstSpace := bytes.Index(bs, []byte(" "))
lastSpace := bytes.LastIndex(bs, []byte(" "))
for i := firstSpace + 1; i < lastSpace; i++ {
if bs[i] == ' ' {
bs[i] = '\n'
}
}
block, _ := pem.Decode(bs)
if block == nil {
// Decoding failed
return nil
}
return block.Bytes
}
return nil
}
// fixupAddresses checks the list of addresses, removing invalid ones and
// replacing unspecified IPs with the given remote IP.
func fixupAddresses(remote net.IP, addresses []string) []string {
fixed := make([]string, 0, len(addresses))
for _, annAddr := range addresses {
uri, err := url.Parse(annAddr)
if err != nil {
continue
}
host, port, err := net.SplitHostPort(uri.Host)
if err != nil {
continue
}
ip := net.ParseIP(host)
// Some classes of IP are no-go.
if ip.IsLoopback() || ip.IsMulticast() {
continue
}
if host == "" || ip.IsUnspecified() {
// Replace the unspecified IP with the request source.
// ... unless the request source is the loopback address or
// multicast/unspecified (can't happen, really).
if remote.IsLoopback() || remote.IsMulticast() || remote.IsUnspecified() {
continue
}
// Do not use IPv6 remote address if requested scheme is ...4
// (i.e., tcp4, etc.)
if strings.HasSuffix(uri.Scheme, "4") && remote.To4() == nil {
continue
}
// Do not use IPv4 remote address if requested scheme is ...6
if strings.HasSuffix(uri.Scheme, "6") && remote.To4() != nil {
continue
}
host = remote.String()
}
uri.Host = net.JoinHostPort(host, port)
fixed = append(fixed, uri.String())
}
return fixed
}
type loggingResponseWriter struct {
http.ResponseWriter
statusCode int
}
func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
return &loggingResponseWriter{w, http.StatusOK}
}
func (lrw *loggingResponseWriter) WriteHeader(code int) {
lrw.statusCode = code
lrw.ResponseWriter.WriteHeader(code)
}
func addressStrs(dbAddrs []DatabaseAddress) []string {
res := make([]string, len(dbAddrs))
for i, a := range dbAddrs {
res[i] = a.Address
}
return res
}
func errorRetryAfterString() string {
return strconv.Itoa(errorRetryAfterSeconds + rand.Intn(errorRetryFuzzSeconds))
}
func notFoundRetryAfterString(misses int) string {
retryAfterS := notFoundRetryMinSeconds + notFoundRetryIncSeconds*misses
if retryAfterS > notFoundRetryMaxSeconds {
retryAfterS = notFoundRetryMaxSeconds
}
retryAfterS += rand.Intn(notFoundRetryFuzzSeconds)
return strconv.Itoa(retryAfterS)
}
func reannounceAfterString() string {
return strconv.Itoa(reannounceAfterSeconds + rand.Intn(reannounzeFuzzSeconds))
}

View File

@@ -1,65 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"fmt"
"net"
"testing"
)
func TestFixupAddresses(t *testing.T) {
cases := []struct {
remote net.IP
in []string
out []string
}{
{ // verbatim passthrough
in: []string{"tcp://1.2.3.4:22000"},
out: []string{"tcp://1.2.3.4:22000"},
}, { // unspecified replaced by remote
remote: net.ParseIP("1.2.3.4"),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://1.2.3.4:22000", "tcp://192.0.2.42:22000"},
}, { // unspecified not used as replacement
remote: net.ParseIP("0.0.0.0"),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://192.0.2.42:22000"},
}, { // unspecified not used as replacement
remote: net.ParseIP("::"),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://192.0.2.42:22000"},
}, { // localhost not used as replacement
remote: net.ParseIP("127.0.0.1"),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://192.0.2.42:22000"},
}, { // localhost not used as replacement
remote: net.ParseIP("::1"),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://192.0.2.42:22000"},
}, { // multicast not used as replacement
remote: net.ParseIP("224.0.0.1"),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://192.0.2.42:22000"},
}, { // multicast not used as replacement
remote: net.ParseIP("ff80::42"),
in: []string{"tcp://:22000", "tcp://192.0.2.42:22000"},
out: []string{"tcp://192.0.2.42:22000"},
}, { // explicitly announced weirdness is also filtered
remote: net.ParseIP("192.0.2.42"),
in: []string{"tcp://:22000", "tcp://127.1.2.3:22000", "tcp://[::1]:22000", "tcp://[ff80::42]:22000"},
out: []string{"tcp://192.0.2.42:22000"},
},
}
for _, tc := range cases {
out := fixupAddresses(tc.remote, tc.in)
if fmt.Sprint(out) != fmt.Sprint(tc.out) {
t.Errorf("fixupAddresses(%v, %v) => %v, expected %v", tc.remote, tc.in, out, tc.out)
}
}
}

75
cmd/stdiscosrv/clean.go Normal file
View File

@@ -0,0 +1,75 @@
// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
package main
import (
"database/sql"
"log"
"time"
)
type cleansrv struct {
intv time.Duration
db *sql.DB
prep map[string]*sql.Stmt
}
func (s *cleansrv) Serve() {
for {
time.Sleep(next(s.intv))
err := s.cleanOldEntries()
if err != nil {
log.Println("Clean:", err)
}
}
}
func (s *cleansrv) Stop() {
panic("stop unimplemented")
}
func (s *cleansrv) cleanOldEntries() (err error) {
var tx *sql.Tx
tx, err = s.db.Begin()
if err != nil {
return err
}
defer func() {
if err == nil {
err = tx.Commit()
} else {
tx.Rollback()
}
}()
res, err := tx.Stmt(s.prep["cleanAddress"]).Exec()
if err != nil {
return err
}
if rows, _ := res.RowsAffected(); rows > 0 {
log.Printf("Clean: %d old addresses", rows)
}
res, err = tx.Stmt(s.prep["cleanDevice"]).Exec()
if err != nil {
return err
}
if rows, _ := res.RowsAffected(); rows > 0 {
log.Printf("Clean: %d old devices", rows)
}
var devs, addrs int
row := tx.Stmt(s.prep["countDevice"]).QueryRow()
if err = row.Scan(&devs); err != nil {
return err
}
row = tx.Stmt(s.prep["countAddress"]).QueryRow()
if err = row.Scan(&addrs); err != nil {
return err
}
log.Printf("Database: %d devices, %d addresses", devs, addrs)
return nil
}

View File

@@ -1,354 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
//go:generate go run ../../script/protofmt.go database.proto
//go:generate protoc -I ../../ -I . --gogofast_out=. database.proto
package main
import (
"sort"
"time"
"github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/util"
)
type clock interface {
Now() time.Time
}
type defaultClock struct{}
func (defaultClock) Now() time.Time {
return time.Now()
}
type database interface {
put(key string, rec DatabaseRecord) error
merge(key string, addrs []DatabaseAddress, seen int64) error
get(key string) (DatabaseRecord, error)
}
type levelDBStore struct {
db *leveldb.DB
inbox chan func()
stop chan struct{}
clock clock
marshalBuf []byte
}
func newLevelDBStore(dir string) (*levelDBStore, error) {
db, err := leveldb.OpenFile(dir, levelDBOptions)
if err != nil {
return nil, err
}
return &levelDBStore{
db: db,
inbox: make(chan func(), 16),
stop: make(chan struct{}),
clock: defaultClock{},
}, nil
}
func (s *levelDBStore) put(key string, rec DatabaseRecord) error {
t0 := time.Now()
defer func() {
databaseOperationSeconds.WithLabelValues(dbOpPut).Observe(time.Since(t0).Seconds())
}()
rc := make(chan error)
s.inbox <- func() {
size := rec.Size()
if len(s.marshalBuf) < size {
s.marshalBuf = make([]byte, size)
}
n, _ := rec.MarshalTo(s.marshalBuf)
rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
}
err := <-rc
if err != nil {
databaseOperations.WithLabelValues(dbOpPut, dbResError).Inc()
} else {
databaseOperations.WithLabelValues(dbOpPut, dbResSuccess).Inc()
}
return err
}
func (s *levelDBStore) merge(key string, addrs []DatabaseAddress, seen int64) error {
t0 := time.Now()
defer func() {
databaseOperationSeconds.WithLabelValues(dbOpMerge).Observe(time.Since(t0).Seconds())
}()
rc := make(chan error)
newRec := DatabaseRecord{
Addresses: addrs,
Seen: seen,
}
s.inbox <- func() {
// grab the existing record
oldRec, err := s.get(key)
if err != nil {
// "not found" is not an error from get, so this is serious
// stuff only
rc <- err
return
}
newRec = merge(newRec, oldRec)
// We replicate s.put() functionality here ourselves instead of
// calling it because we want to serialize our get above together
// with the put in the same function.
size := newRec.Size()
if len(s.marshalBuf) < size {
s.marshalBuf = make([]byte, size)
}
n, _ := newRec.MarshalTo(s.marshalBuf)
rc <- s.db.Put([]byte(key), s.marshalBuf[:n], nil)
}
err := <-rc
if err != nil {
databaseOperations.WithLabelValues(dbOpMerge, dbResError).Inc()
} else {
databaseOperations.WithLabelValues(dbOpMerge, dbResSuccess).Inc()
}
return err
}
func (s *levelDBStore) get(key string) (DatabaseRecord, error) {
t0 := time.Now()
defer func() {
databaseOperationSeconds.WithLabelValues(dbOpGet).Observe(time.Since(t0).Seconds())
}()
keyBs := []byte(key)
val, err := s.db.Get(keyBs, nil)
if err == leveldb.ErrNotFound {
databaseOperations.WithLabelValues(dbOpGet, dbResNotFound).Inc()
return DatabaseRecord{}, nil
}
if err != nil {
databaseOperations.WithLabelValues(dbOpGet, dbResError).Inc()
return DatabaseRecord{}, err
}
var rec DatabaseRecord
if err := rec.Unmarshal(val); err != nil {
databaseOperations.WithLabelValues(dbOpGet, dbResUnmarshalError).Inc()
return DatabaseRecord{}, nil
}
rec.Addresses = expire(rec.Addresses, s.clock.Now().UnixNano())
databaseOperations.WithLabelValues(dbOpGet, dbResSuccess).Inc()
return rec, nil
}
func (s *levelDBStore) Serve() {
t := time.NewTimer(0)
defer t.Stop()
defer s.db.Close()
// Start the statistics serve routine. It will exit with us when
// statisticsTrigger is closed.
statisticsTrigger := make(chan struct{})
statisticsDone := make(chan struct{})
go s.statisticsServe(statisticsTrigger, statisticsDone)
loop:
for {
select {
case fn := <-s.inbox:
// Run function in serialized order.
fn()
case <-t.C:
// Trigger the statistics routine to do its thing in the
// background.
statisticsTrigger <- struct{}{}
case <-statisticsDone:
// The statistics routine is done with one iteratation, schedule
// the next.
t.Reset(databaseStatisticsInterval)
case <-s.stop:
// We're done.
close(statisticsTrigger)
break loop
}
}
// Also wait for statisticsServe to return
<-statisticsDone
}
func (s *levelDBStore) statisticsServe(trigger <-chan struct{}, done chan<- struct{}) {
defer close(done)
for range trigger {
t0 := time.Now()
nowNanos := t0.UnixNano()
cutoff24h := t0.Add(-24 * time.Hour).UnixNano()
cutoff1w := t0.Add(-7 * 24 * time.Hour).UnixNano()
cutoff2Mon := t0.Add(-60 * 24 * time.Hour).UnixNano()
current, last24h, last1w, inactive, errors := 0, 0, 0, 0, 0
iter := s.db.NewIterator(&util.Range{}, nil)
for iter.Next() {
// Attempt to unmarshal the record and count the
// failure if there's something wrong with it.
var rec DatabaseRecord
if err := rec.Unmarshal(iter.Value()); err != nil {
errors++
continue
}
// If there are addresses that have not expired it's a current
// record, otherwise account it based on when it was last seen
// (last 24 hours or last week) or finally as inactice.
switch {
case len(expire(rec.Addresses, nowNanos)) > 0:
current++
case rec.Seen > cutoff24h:
last24h++
case rec.Seen > cutoff1w:
last1w++
case rec.Seen > cutoff2Mon:
inactive++
case rec.Missed < cutoff2Mon:
// It hasn't been seen lately and we haven't recorded
// someone asking for this device in a long time either;
// delete the record.
if err := s.db.Delete(iter.Key(), nil); err != nil {
databaseOperations.WithLabelValues(dbOpDelete, dbResError).Inc()
} else {
databaseOperations.WithLabelValues(dbOpDelete, dbResSuccess).Inc()
}
default:
inactive++
}
}
iter.Release()
databaseKeys.WithLabelValues("current").Set(float64(current))
databaseKeys.WithLabelValues("last24h").Set(float64(last24h))
databaseKeys.WithLabelValues("last1w").Set(float64(last1w))
databaseKeys.WithLabelValues("inactive").Set(float64(inactive))
databaseKeys.WithLabelValues("error").Set(float64(errors))
databaseStatisticsSeconds.Set(time.Since(t0).Seconds())
// Signal that we are done and can be scheduled again.
done <- struct{}{}
}
}
func (s *levelDBStore) Stop() {
close(s.stop)
}
// merge returns the merged result of the two database records a and b. The
// result is the union of the two address sets, with the newer expiry time
// chosen for any duplicates.
func merge(a, b DatabaseRecord) DatabaseRecord {
// Both lists must be sorted for this to work.
sort.Slice(a.Addresses, func(i, j int) bool {
return a.Addresses[i].Address < a.Addresses[j].Address
})
sort.Slice(b.Addresses, func(i, j int) bool {
return b.Addresses[i].Address < b.Addresses[j].Address
})
res := DatabaseRecord{
Addresses: make([]DatabaseAddress, 0, len(a.Addresses)+len(b.Addresses)),
Seen: a.Seen,
}
if b.Seen > a.Seen {
res.Seen = b.Seen
}
aIdx := 0
bIdx := 0
aAddrs := a.Addresses
bAddrs := b.Addresses
loop:
for {
switch {
case aIdx == len(aAddrs) && bIdx == len(bAddrs):
// both lists are exhausted, we are done
break loop
case aIdx == len(aAddrs):
// a is exhausted, pick from b and continue
res.Addresses = append(res.Addresses, bAddrs[bIdx])
bIdx++
continue
case bIdx == len(bAddrs):
// b is exhausted, pick from a and continue
res.Addresses = append(res.Addresses, aAddrs[aIdx])
aIdx++
continue
}
// We have values left on both sides.
aVal := aAddrs[aIdx]
bVal := bAddrs[bIdx]
switch {
case aVal.Address == bVal.Address:
// update for same address, pick newer
if aVal.Expires > bVal.Expires {
res.Addresses = append(res.Addresses, aVal)
} else {
res.Addresses = append(res.Addresses, bVal)
}
aIdx++
bIdx++
case aVal.Address < bVal.Address:
// a is smallest, pick it and continue
res.Addresses = append(res.Addresses, aVal)
aIdx++
default:
// b is smallest, pick it and continue
res.Addresses = append(res.Addresses, bVal)
bIdx++
}
}
return res
}
// expire returns the list of addresses after removing expired entries.
// Expiration happen in place, so the slice given as the parameter is
// destroyed. Internal order is not preserved.
func expire(addrs []DatabaseAddress, now int64) []DatabaseAddress {
i := 0
for i < len(addrs) {
if addrs[i].Expires < now {
// This item is expired. Replace it with the last in the list
// (noop if we are at the last item).
addrs[i] = addrs[len(addrs)-1]
// Wipe the last item of the list to release references to
// strings and stuff.
addrs[len(addrs)-1] = DatabaseAddress{}
// Shorten the slice.
addrs = addrs[:len(addrs)-1]
continue
}
i++
}
return addrs
}

View File

@@ -1,836 +0,0 @@
// Code generated by protoc-gen-gogo. DO NOT EDIT.
// source: database.proto
package main
import proto "github.com/gogo/protobuf/proto"
import fmt "fmt"
import math "math"
import _ "github.com/gogo/protobuf/gogoproto"
import io "io"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
type DatabaseRecord struct {
Addresses []DatabaseAddress `protobuf:"bytes,1,rep,name=addresses,proto3" json:"addresses"`
Misses int32 `protobuf:"varint,2,opt,name=misses,proto3" json:"misses,omitempty"`
Seen int64 `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
Missed int64 `protobuf:"varint,4,opt,name=missed,proto3" json:"missed,omitempty"`
}
func (m *DatabaseRecord) Reset() { *m = DatabaseRecord{} }
func (m *DatabaseRecord) String() string { return proto.CompactTextString(m) }
func (*DatabaseRecord) ProtoMessage() {}
func (*DatabaseRecord) Descriptor() ([]byte, []int) {
return fileDescriptor_database_0f49e029703a04f5, []int{0}
}
func (m *DatabaseRecord) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *DatabaseRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_DatabaseRecord.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalTo(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (dst *DatabaseRecord) XXX_Merge(src proto.Message) {
xxx_messageInfo_DatabaseRecord.Merge(dst, src)
}
func (m *DatabaseRecord) XXX_Size() int {
return m.Size()
}
func (m *DatabaseRecord) XXX_DiscardUnknown() {
xxx_messageInfo_DatabaseRecord.DiscardUnknown(m)
}
var xxx_messageInfo_DatabaseRecord proto.InternalMessageInfo
type ReplicationRecord struct {
Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
Addresses []DatabaseAddress `protobuf:"bytes,2,rep,name=addresses,proto3" json:"addresses"`
Seen int64 `protobuf:"varint,3,opt,name=seen,proto3" json:"seen,omitempty"`
}
func (m *ReplicationRecord) Reset() { *m = ReplicationRecord{} }
func (m *ReplicationRecord) String() string { return proto.CompactTextString(m) }
func (*ReplicationRecord) ProtoMessage() {}
func (*ReplicationRecord) Descriptor() ([]byte, []int) {
return fileDescriptor_database_0f49e029703a04f5, []int{1}
}
func (m *ReplicationRecord) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *ReplicationRecord) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_ReplicationRecord.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalTo(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (dst *ReplicationRecord) XXX_Merge(src proto.Message) {
xxx_messageInfo_ReplicationRecord.Merge(dst, src)
}
func (m *ReplicationRecord) XXX_Size() int {
return m.Size()
}
func (m *ReplicationRecord) XXX_DiscardUnknown() {
xxx_messageInfo_ReplicationRecord.DiscardUnknown(m)
}
var xxx_messageInfo_ReplicationRecord proto.InternalMessageInfo
type DatabaseAddress struct {
Address string `protobuf:"bytes,1,opt,name=address,proto3" json:"address,omitempty"`
Expires int64 `protobuf:"varint,2,opt,name=expires,proto3" json:"expires,omitempty"`
}
func (m *DatabaseAddress) Reset() { *m = DatabaseAddress{} }
func (m *DatabaseAddress) String() string { return proto.CompactTextString(m) }
func (*DatabaseAddress) ProtoMessage() {}
func (*DatabaseAddress) Descriptor() ([]byte, []int) {
return fileDescriptor_database_0f49e029703a04f5, []int{2}
}
func (m *DatabaseAddress) XXX_Unmarshal(b []byte) error {
return m.Unmarshal(b)
}
func (m *DatabaseAddress) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
if deterministic {
return xxx_messageInfo_DatabaseAddress.Marshal(b, m, deterministic)
} else {
b = b[:cap(b)]
n, err := m.MarshalTo(b)
if err != nil {
return nil, err
}
return b[:n], nil
}
}
func (dst *DatabaseAddress) XXX_Merge(src proto.Message) {
xxx_messageInfo_DatabaseAddress.Merge(dst, src)
}
func (m *DatabaseAddress) XXX_Size() int {
return m.Size()
}
func (m *DatabaseAddress) XXX_DiscardUnknown() {
xxx_messageInfo_DatabaseAddress.DiscardUnknown(m)
}
var xxx_messageInfo_DatabaseAddress proto.InternalMessageInfo
func init() {
proto.RegisterType((*DatabaseRecord)(nil), "main.DatabaseRecord")
proto.RegisterType((*ReplicationRecord)(nil), "main.ReplicationRecord")
proto.RegisterType((*DatabaseAddress)(nil), "main.DatabaseAddress")
}
func (m *DatabaseRecord) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *DatabaseRecord) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.Addresses) > 0 {
for _, msg := range m.Addresses {
dAtA[i] = 0xa
i++
i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
n, err := msg.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n
}
}
if m.Misses != 0 {
dAtA[i] = 0x10
i++
i = encodeVarintDatabase(dAtA, i, uint64(m.Misses))
}
if m.Seen != 0 {
dAtA[i] = 0x18
i++
i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
}
if m.Missed != 0 {
dAtA[i] = 0x20
i++
i = encodeVarintDatabase(dAtA, i, uint64(m.Missed))
}
return i, nil
}
func (m *ReplicationRecord) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *ReplicationRecord) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.Key) > 0 {
dAtA[i] = 0xa
i++
i = encodeVarintDatabase(dAtA, i, uint64(len(m.Key)))
i += copy(dAtA[i:], m.Key)
}
if len(m.Addresses) > 0 {
for _, msg := range m.Addresses {
dAtA[i] = 0x12
i++
i = encodeVarintDatabase(dAtA, i, uint64(msg.Size()))
n, err := msg.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n
}
}
if m.Seen != 0 {
dAtA[i] = 0x18
i++
i = encodeVarintDatabase(dAtA, i, uint64(m.Seen))
}
return i, nil
}
func (m *DatabaseAddress) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *DatabaseAddress) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.Address) > 0 {
dAtA[i] = 0xa
i++
i = encodeVarintDatabase(dAtA, i, uint64(len(m.Address)))
i += copy(dAtA[i:], m.Address)
}
if m.Expires != 0 {
dAtA[i] = 0x10
i++
i = encodeVarintDatabase(dAtA, i, uint64(m.Expires))
}
return i, nil
}
func encodeVarintDatabase(dAtA []byte, offset int, v uint64) int {
for v >= 1<<7 {
dAtA[offset] = uint8(v&0x7f | 0x80)
v >>= 7
offset++
}
dAtA[offset] = uint8(v)
return offset + 1
}
func (m *DatabaseRecord) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
if len(m.Addresses) > 0 {
for _, e := range m.Addresses {
l = e.Size()
n += 1 + l + sovDatabase(uint64(l))
}
}
if m.Misses != 0 {
n += 1 + sovDatabase(uint64(m.Misses))
}
if m.Seen != 0 {
n += 1 + sovDatabase(uint64(m.Seen))
}
if m.Missed != 0 {
n += 1 + sovDatabase(uint64(m.Missed))
}
return n
}
func (m *ReplicationRecord) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.Key)
if l > 0 {
n += 1 + l + sovDatabase(uint64(l))
}
if len(m.Addresses) > 0 {
for _, e := range m.Addresses {
l = e.Size()
n += 1 + l + sovDatabase(uint64(l))
}
}
if m.Seen != 0 {
n += 1 + sovDatabase(uint64(m.Seen))
}
return n
}
func (m *DatabaseAddress) Size() (n int) {
if m == nil {
return 0
}
var l int
_ = l
l = len(m.Address)
if l > 0 {
n += 1 + l + sovDatabase(uint64(l))
}
if m.Expires != 0 {
n += 1 + sovDatabase(uint64(m.Expires))
}
return n
}
func sovDatabase(x uint64) (n int) {
for {
n++
x >>= 7
if x == 0 {
break
}
}
return n
}
func sozDatabase(x uint64) (n int) {
return sovDatabase(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (m *DatabaseRecord) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: DatabaseRecord: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: DatabaseRecord: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthDatabase
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Addresses = append(m.Addresses, DatabaseAddress{})
if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Misses", wireType)
}
m.Misses = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Misses |= (int32(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
}
m.Seen = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Seen |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 4:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Missed", wireType)
}
m.Missed = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Missed |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipDatabase(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthDatabase
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *ReplicationRecord) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: ReplicationRecord: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: ReplicationRecord: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthDatabase
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Key = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthDatabase
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Addresses = append(m.Addresses, DatabaseAddress{})
if err := m.Addresses[len(m.Addresses)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Seen", wireType)
}
m.Seen = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Seen |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipDatabase(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthDatabase
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *DatabaseAddress) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: DatabaseAddress: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: DatabaseAddress: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Address", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthDatabase
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Address = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Expires", wireType)
}
m.Expires = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowDatabase
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Expires |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipDatabase(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthDatabase
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipDatabase(dAtA []byte) (n int, err error) {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowDatabase
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
wireType := int(wire & 0x7)
switch wireType {
case 0:
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowDatabase
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
iNdEx++
if dAtA[iNdEx-1] < 0x80 {
break
}
}
return iNdEx, nil
case 1:
iNdEx += 8
return iNdEx, nil
case 2:
var length int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowDatabase
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
length |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
iNdEx += length
if length < 0 {
return 0, ErrInvalidLengthDatabase
}
return iNdEx, nil
case 3:
for {
var innerWire uint64
var start int = iNdEx
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowDatabase
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
innerWire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
innerWireType := int(innerWire & 0x7)
if innerWireType == 4 {
break
}
next, err := skipDatabase(dAtA[start:])
if err != nil {
return 0, err
}
iNdEx = start + next
}
return iNdEx, nil
case 4:
return iNdEx, nil
case 5:
iNdEx += 4
return iNdEx, nil
default:
return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
}
}
panic("unreachable")
}
var (
ErrInvalidLengthDatabase = fmt.Errorf("proto: negative length found during unmarshaling")
ErrIntOverflowDatabase = fmt.Errorf("proto: integer overflow")
)
func init() { proto.RegisterFile("database.proto", fileDescriptor_database_0f49e029703a04f5) }
var fileDescriptor_database_0f49e029703a04f5 = []byte{
// 270 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x90, 0x41, 0x4a, 0xc4, 0x30,
0x18, 0x85, 0x9b, 0x49, 0x1d, 0x99, 0x08, 0xa3, 0x06, 0x94, 0x20, 0x12, 0x4b, 0xdd, 0x74, 0xd5,
0x01, 0x5d, 0xb9, 0x74, 0xd0, 0x0b, 0xe4, 0x06, 0xe9, 0xe4, 0x77, 0x08, 0x3a, 0x4d, 0x49, 0x2a,
0xe8, 0x29, 0xf4, 0x58, 0x5d, 0xce, 0xd2, 0x95, 0x68, 0x7b, 0x11, 0x69, 0x26, 0x55, 0x14, 0x37,
0xb3, 0x7b, 0xdf, 0xff, 0xbf, 0x97, 0xbc, 0x84, 0x4c, 0x95, 0xac, 0x65, 0x21, 0x1d, 0xe4, 0x95,
0x35, 0xb5, 0xa1, 0xf1, 0x4a, 0xea, 0xf2, 0xe4, 0xdc, 0x42, 0x65, 0xdc, 0xcc, 0x8f, 0x8a, 0xc7,
0xbb, 0xd9, 0xd2, 0x2c, 0x8d, 0x07, 0xaf, 0x36, 0xd6, 0xf4, 0x05, 0x91, 0xe9, 0x4d, 0x48, 0x0b,
0x58, 0x18, 0xab, 0xe8, 0x15, 0x99, 0x48, 0xa5, 0x2c, 0x38, 0x07, 0x8e, 0xa1, 0x04, 0x67, 0x7b,
0x17, 0x47, 0x79, 0x7f, 0x62, 0x3e, 0x18, 0xaf, 0x37, 0xeb, 0x79, 0xdc, 0xbc, 0x9f, 0x45, 0xe2,
0xc7, 0x4d, 0x8f, 0xc9, 0x78, 0xa5, 0x7d, 0x6e, 0x94, 0xa0, 0x6c, 0x47, 0x04, 0xa2, 0x94, 0xc4,
0x0e, 0xa0, 0x64, 0x38, 0x41, 0x19, 0x16, 0x5e, 0x7f, 0x7b, 0x15, 0x8b, 0xfd, 0x34, 0x50, 0x5a,
0x93, 0x43, 0x01, 0xd5, 0x83, 0x5e, 0xc8, 0x5a, 0x9b, 0x32, 0x74, 0x3a, 0x20, 0xf8, 0x1e, 0x9e,
0x19, 0x4a, 0x50, 0x36, 0x11, 0xbd, 0xfc, 0xdd, 0x72, 0xb4, 0x55, 0xcb, 0x7f, 0xda, 0xa4, 0xb7,
0x64, 0xff, 0x4f, 0x8e, 0x32, 0xb2, 0x1b, 0x32, 0xe1, 0xde, 0x01, 0xfb, 0x0d, 0x3c, 0x55, 0xda,
0x86, 0x77, 0x62, 0x31, 0xe0, 0xfc, 0xb4, 0xf9, 0xe4, 0x51, 0xd3, 0x72, 0xb4, 0x6e, 0x39, 0xfa,
0x68, 0x39, 0x7a, 0xed, 0x78, 0xb4, 0xee, 0x78, 0xf4, 0xd6, 0xf1, 0xa8, 0x18, 0xfb, 0x3f, 0xbf,
0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x7a, 0xa2, 0xf6, 0x1e, 0xb0, 0x01, 0x00, 0x00,
}

View File

@@ -1,36 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
syntax = "proto3";
package main;
import "repos/protobuf/gogoproto/gogo.proto";
option (gogoproto.goproto_getters_all) = false;
option (gogoproto.goproto_unkeyed_all) = false;
option (gogoproto.goproto_unrecognized_all) = false;
option (gogoproto.goproto_sizecache_all) = false;
message DatabaseRecord {
repeated DatabaseAddress addresses = 1 [(gogoproto.nullable) = false];
int32 misses = 2; // Number of lookups* without hits
int64 seen = 3; // Unix nanos, last device announce
int64 missed = 4; // Unix nanos, last* failed lookup
}
// *) Not every lookup results in a write, so may not be completely accurate
message ReplicationRecord {
string key = 1;
repeated DatabaseAddress addresses = 2 [(gogoproto.nullable) = false];
int64 seen = 3; // Unix nanos, last device announce
}
message DatabaseAddress {
string address = 1;
int64 expires = 2; // Unix nanos
}

View File

@@ -1,211 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"fmt"
"os"
"testing"
"time"
)
func TestDatabaseGetSet(t *testing.T) {
os.RemoveAll("_database")
defer os.RemoveAll("_database")
db, err := newLevelDBStore("_database")
if err != nil {
t.Fatal(err)
}
go db.Serve()
defer db.Stop()
// Check missing record
rec, err := db.get("abcd")
if err != nil {
t.Error("not found should not be an error")
}
if len(rec.Addresses) != 0 {
t.Error("addresses should be empty")
}
if rec.Misses != 0 {
t.Error("missing should be zero")
}
// Set up a clock
now := time.Now()
tc := &testClock{now}
db.clock = tc
// Put a record
rec.Addresses = []DatabaseAddress{
{Address: "tcp://1.2.3.4:5", Expires: tc.Now().Add(time.Minute).UnixNano()},
}
if err := db.put("abcd", rec); err != nil {
t.Fatal(err)
}
// Verify it
rec, err = db.get("abcd")
if err != nil {
t.Fatal(err)
}
if len(rec.Addresses) != 1 {
t.Log(rec.Addresses)
t.Fatal("should have one address")
}
if rec.Addresses[0].Address != "tcp://1.2.3.4:5" {
t.Log(rec.Addresses)
t.Error("incorrect address")
}
// Wind the clock one half expiry, and merge in a new address
tc.wind(30 * time.Second)
addrs := []DatabaseAddress{
{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
}
if err := db.merge("abcd", addrs, tc.Now().UnixNano()); err != nil {
t.Fatal(err)
}
// Verify it
rec, err = db.get("abcd")
if err != nil {
t.Fatal(err)
}
if len(rec.Addresses) != 2 {
t.Log(rec.Addresses)
t.Fatal("should have two addresses")
}
if rec.Addresses[0].Address != "tcp://1.2.3.4:5" {
t.Log(rec.Addresses)
t.Error("incorrect address[0]")
}
if rec.Addresses[1].Address != "tcp://6.7.8.9:0" {
t.Log(rec.Addresses)
t.Error("incorrect address[1]")
}
// Pass the first expiry time
tc.wind(45 * time.Second)
// Verify it
rec, err = db.get("abcd")
if err != nil {
t.Fatal(err)
}
if len(rec.Addresses) != 1 {
t.Log(rec.Addresses)
t.Fatal("should have one address")
}
if rec.Addresses[0].Address != "tcp://6.7.8.9:0" {
t.Log(rec.Addresses)
t.Error("incorrect address")
}
// Put a record with misses
rec = DatabaseRecord{Misses: 42}
if err := db.put("efgh", rec); err != nil {
t.Fatal(err)
}
// Verify it
rec, err = db.get("efgh")
if err != nil {
t.Fatal(err)
}
if len(rec.Addresses) != 0 {
t.Log(rec.Addresses)
t.Fatal("should have no addresses")
}
if rec.Misses != 42 {
t.Log(rec.Misses)
t.Error("incorrect misses")
}
// Set an address
addrs = []DatabaseAddress{
{Address: "tcp://6.7.8.9:0", Expires: tc.Now().Add(time.Minute).UnixNano()},
}
if err := db.merge("efgh", addrs, tc.Now().UnixNano()); err != nil {
t.Fatal(err)
}
// Verify it
rec, err = db.get("efgh")
if err != nil {
t.Fatal(err)
}
if len(rec.Addresses) != 1 {
t.Log(rec.Addresses)
t.Fatal("should have one address")
}
if rec.Misses != 0 {
t.Log(rec.Misses)
t.Error("should have no misses")
}
}
func TestFilter(t *testing.T) {
// all cases are expired with t=10
cases := []struct {
a []DatabaseAddress
b []DatabaseAddress
}{
{
a: nil,
b: nil,
},
{
a: []DatabaseAddress{{Address: "a", Expires: 9}, {Address: "b", Expires: 9}, {Address: "c", Expires: 9}},
b: []DatabaseAddress{},
},
{
a: []DatabaseAddress{{Address: "a", Expires: 10}},
b: []DatabaseAddress{{Address: "a", Expires: 10}},
},
{
a: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
b: []DatabaseAddress{{Address: "a", Expires: 10}, {Address: "b", Expires: 10}, {Address: "c", Expires: 10}},
},
{
a: []DatabaseAddress{{Address: "a", Expires: 5}, {Address: "b", Expires: 15}, {Address: "c", Expires: 5}, {Address: "d", Expires: 15}, {Address: "e", Expires: 5}},
b: []DatabaseAddress{{Address: "d", Expires: 15}, {Address: "b", Expires: 15}}, // gets reordered
},
}
for _, tc := range cases {
res := expire(tc.a, 10)
if fmt.Sprint(res) != fmt.Sprint(tc.b) {
t.Errorf("Incorrect result %v, expected %v", res, tc.b)
}
}
}
type testClock struct {
now time.Time
}
func (t *testClock) wind(d time.Duration) {
t.now = t.now.Add(d)
}
func (t *testClock) Now() time.Time {
return t.now
}

32
cmd/stdiscosrv/db.go Normal file
View File

@@ -0,0 +1,32 @@
// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
package main
import (
"database/sql"
"fmt"
)
type setupFunc func(db *sql.DB) error
type compileFunc func(db *sql.DB) (map[string]*sql.Stmt, error)
var (
setupFuncs = make(map[string]setupFunc)
compileFuncs = make(map[string]compileFunc)
)
func register(name string, setup setupFunc, compile compileFunc) {
setupFuncs[name] = setup
compileFuncs[name] = compile
}
func setup(backend string, db *sql.DB) (map[string]*sql.Stmt, error) {
setup, ok := setupFuncs[backend]
if !ok {
return nil, fmt.Errorf("Unsupported backend")
}
if err := setup(db); err != nil {
return nil, err
}
return compileFuncs[backend](db)
}

View File

@@ -1,70 +1,29 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
package main
import (
"crypto/tls"
"database/sql"
"flag"
"fmt"
"log"
"net"
"net/http"
"os"
"runtime"
"strconv"
"strings"
"time"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/tlsutil"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/thejerf/suture"
)
const (
addressExpiryTime = 2 * time.Hour
databaseStatisticsInterval = 5 * time.Minute
// Reannounce-After is set to reannounceAfterSeconds +
// random(reannounzeFuzzSeconds), similar for Retry-After
reannounceAfterSeconds = 3300
reannounzeFuzzSeconds = 300
errorRetryAfterSeconds = 1500
errorRetryFuzzSeconds = 300
// Retry for not found is minSeconds + failures * incSeconds +
// random(fuzz), where failures is the number of consecutive lookups
// with no answer, up to maxSeconds. The fuzz is applied after capping
// to maxSeconds.
notFoundRetryMinSeconds = 60
notFoundRetryMaxSeconds = 3540
notFoundRetryIncSeconds = 10
notFoundRetryFuzzSeconds = 60
// How often (in requests) we serialize the missed counter to database.
notFoundMissesWriteInterval = 10
httpReadTimeout = 5 * time.Second
httpWriteTimeout = 5 * time.Second
httpMaxHeaderBytes = 1 << 10
// Size of the replication outbox channel
replicationOutboxSize = 10000
minNegCache = 60 // seconds
maxNegCache = 3600 // seconds
maxDeviceAge = 7 * 86400 // one week, in seconds
)
// These options make the database a little more optimized for writes, at
// the expense of some memory usage and risk of losing writes in a (system)
// crash.
var levelDBOptions = &opt.Options{
NoSync: true,
WriteBuffer: 32 << 20, // default 4<<20
}
var (
Version string
BuildStamp string
@@ -84,119 +43,104 @@ func init() {
}
var (
debug = false
lruSize = 10240
limitAvg = 5
limitBurst = 20
globalStats stats
statsFile string
backend = "ql"
dsn = getEnvDefault("STDISCOSRV_DB_DSN", "memory://stdiscosrv")
certFile = "cert.pem"
keyFile = "key.pem"
debug = false
useHTTP = false
)
func main() {
const (
cleanIntv = 1 * time.Hour
statsIntv = 5 * time.Minute
)
var listen string
var dir string
var metricsListen string
var replicationListen string
var replicationPeers string
var certFile string
var keyFile string
var useHTTP bool
log.SetOutput(os.Stdout)
log.SetFlags(0)
flag.StringVar(&certFile, "cert", "./cert.pem", "Certificate file")
flag.StringVar(&dir, "db-dir", "./discovery.db", "Database directory")
flag.BoolVar(&debug, "debug", false, "Print debug output")
flag.BoolVar(&useHTTP, "http", false, "Listen on HTTP (behind an HTTPS proxy)")
flag.StringVar(&listen, "listen", ":8443", "Listen address")
flag.StringVar(&keyFile, "key", "./key.pem", "Key file")
flag.StringVar(&metricsListen, "metrics-listen", "", "Metrics listen address")
flag.StringVar(&replicationPeers, "replicate", "", "Replication peers, id@address, comma separated")
flag.StringVar(&replicationListen, "replication-listen", ":19200", "Replication listen address")
flag.IntVar(&lruSize, "limit-cache", lruSize, "Limiter cache entries")
flag.IntVar(&limitAvg, "limit-avg", limitAvg, "Allowed average package rate, per 10 s")
flag.IntVar(&limitBurst, "limit-burst", limitBurst, "Allowed burst size, packets")
flag.StringVar(&statsFile, "stats-file", statsFile, "File to write periodic operation stats to")
flag.StringVar(&backend, "db-backend", backend, "Database backend to use")
flag.StringVar(&dsn, "db-dsn", dsn, "Database DSN")
flag.StringVar(&certFile, "cert", certFile, "Certificate file")
flag.StringVar(&keyFile, "key", keyFile, "Key file")
flag.BoolVar(&debug, "debug", debug, "Debug")
flag.BoolVar(&useHTTP, "http", useHTTP, "Listen on HTTP (behind an HTTPS proxy)")
flag.Parse()
log.Println(LongVersion)
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
log.Println("Failed to load keypair. Generating one, this might take a while...")
cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv")
var cert tls.Certificate
var err error
if !useHTTP {
cert, err = tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
log.Fatalln("Failed to generate X509 key pair:", err)
log.Println("Failed to load keypair. Generating one, this might take a while...")
cert, err = tlsutil.NewCertificate(certFile, keyFile, "stdiscosrv", 3072)
if err != nil {
log.Fatalln("Failed to generate X509 key pair:", err)
}
}
devID := protocol.NewDeviceID(cert.Certificate[0])
log.Println("Server device ID is", devID)
}
devID := protocol.NewDeviceID(cert.Certificate[0])
log.Println("Server device ID is", devID)
// Parse the replication specs, if any.
var allowedReplicationPeers []protocol.DeviceID
var replicationDestinations []string
parts := strings.Split(replicationPeers, ",")
for _, part := range parts {
fields := strings.Split(part, "@")
switch len(fields) {
case 2:
// This is an id@address specification. Grab the address for the
// destination list. Try to resolve it once to catch obvious
// syntax errors here rather than having the sender service fail
// repeatedly later.
_, err := net.ResolveTCPAddr("tcp", fields[1])
if err != nil {
log.Fatalln("Resolving address:", err)
}
replicationDestinations = append(replicationDestinations, fields[1])
fallthrough // N.B.
case 1:
// The first part is always a device ID.
id, err := protocol.DeviceIDFromString(fields[0])
if err != nil {
log.Fatalln("Parsing device ID:", err)
}
allowedReplicationPeers = append(allowedReplicationPeers, id)
default:
log.Fatalln("Unrecognized replication spec:", part)
}
db, err := sql.Open(backend, dsn)
if err != nil {
log.Fatalln("sql.Open:", err)
}
prep, err := setup(backend, db)
if err != nil {
log.Fatalln("Setup:", err)
}
// Root of the service tree.
main := suture.New("main", suture.Spec{
PassThroughPanics: true,
main := suture.NewSimple("main")
main.Add(&querysrv{
addr: listen,
cert: cert,
db: db,
prep: prep,
})
// Start the database.
db, err := newLevelDBStore(dir)
if err != nil {
log.Fatalln("Open database:", err)
}
main.Add(db)
main.Add(&cleansrv{
intv: cleanIntv,
db: db,
prep: prep,
})
// Start any replication senders.
var repl replicationMultiplexer
for _, dst := range replicationDestinations {
rs := newReplicationSender(dst, cert, allowedReplicationPeers)
main.Add(rs)
repl = append(repl, rs)
}
main.Add(&statssrv{
intv: statsIntv,
file: statsFile,
db: db,
})
// If we have replication configured, start the replication listener.
if len(allowedReplicationPeers) > 0 {
rl := newReplicationListener(replicationListen, cert, allowedReplicationPeers, db)
main.Add(rl)
}
// Start the main API server.
qs := newAPISrv(listen, cert, db, repl, useHTTP)
main.Add(qs)
// If we have a metrics port configured, start a metrics handler.
if metricsListen != "" {
go func() {
mux := http.NewServeMux()
mux.Handle("/metrics", promhttp.Handler())
log.Fatal(http.ListenAndServe(metricsListen, mux))
}()
}
// Engage!
globalStats.Reset()
main.Serve()
}
func getEnvDefault(key, def string) string {
if val := os.Getenv(key); val != "" {
return val
}
return def
}
func next(intv time.Duration) time.Duration {
t0 := time.Now()
t1 := t0.Add(intv).Truncate(intv)
return t1.Sub(t0)
}

98
cmd/stdiscosrv/psql.go Normal file
View File

@@ -0,0 +1,98 @@
// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
package main
import (
"database/sql"
"fmt"
_ "github.com/lib/pq"
)
func init() {
register("postgres", postgresSetup, postgresCompile)
}
func postgresSetup(db *sql.DB) error {
var err error
db.SetMaxIdleConns(4)
db.SetMaxOpenConns(8)
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS Devices (
DeviceID CHAR(63) NOT NULL PRIMARY KEY,
Seen TIMESTAMP NOT NULL
)`)
if err != nil {
return err
}
var tmp string
row := db.QueryRow(`SELECT 'DevicesDeviceIDIndex'::regclass`)
if err = row.Scan(&tmp); err != nil {
_, err = db.Exec(`CREATE INDEX DevicesDeviceIDIndex ON Devices (DeviceID)`)
}
if err != nil {
return err
}
row = db.QueryRow(`SELECT 'DevicesSeenIndex'::regclass`)
if err = row.Scan(&tmp); err != nil {
_, err = db.Exec(`CREATE INDEX DevicesSeenIndex ON Devices (Seen)`)
}
if err != nil {
return err
}
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS Addresses (
DeviceID CHAR(63) NOT NULL,
Seen TIMESTAMP NOT NULL,
Address VARCHAR(2048) NOT NULL
)`)
if err != nil {
return err
}
row = db.QueryRow(`SELECT 'AddressesDeviceIDSeenIndex'::regclass`)
if err = row.Scan(&tmp); err != nil {
_, err = db.Exec(`CREATE INDEX AddressesDeviceIDSeenIndex ON Addresses (DeviceID, Seen)`)
}
if err != nil {
return err
}
row = db.QueryRow(`SELECT 'AddressesDeviceIDAddressIndex'::regclass`)
if err = row.Scan(&tmp); err != nil {
_, err = db.Exec(`CREATE INDEX AddressesDeviceIDAddressIndex ON Addresses (DeviceID, Address)`)
}
if err != nil {
return err
}
return nil
}
func postgresCompile(db *sql.DB) (map[string]*sql.Stmt, error) {
stmts := map[string]string{
"cleanAddress": "DELETE FROM Addresses WHERE Seen < now() - '2 hour'::INTERVAL",
"cleanDevice": fmt.Sprintf("DELETE FROM Devices WHERE Seen < now() - '%d hour'::INTERVAL", maxDeviceAge/3600),
"countAddress": "SELECT count(*) FROM Addresses",
"countDevice": "SELECT count(*) FROM Devices",
"insertAddress": "INSERT INTO Addresses (DeviceID, Seen, Address) VALUES ($1, now(), $2)",
"insertDevice": "INSERT INTO Devices (DeviceID, Seen) VALUES ($1, now())",
"selectAddress": "SELECT Address FROM Addresses WHERE DeviceID=$1 AND Seen > now() - '1 hour'::INTERVAL ORDER BY random() LIMIT 16",
"selectDevice": "SELECT Seen FROM Devices WHERE DeviceID=$1",
"updateAddress": "UPDATE Addresses SET Seen=now() WHERE DeviceID=$1 AND Address=$2",
"updateDevice": "UPDATE Devices SET Seen=now() WHERE DeviceID=$1",
}
res := make(map[string]*sql.Stmt, len(stmts))
for key, stmt := range stmts {
prep, err := db.Prepare(stmt)
if err != nil {
return nil, err
}
res[key] = prep
}
return res, nil
}

81
cmd/stdiscosrv/ql.go Normal file
View File

@@ -0,0 +1,81 @@
// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
package main
import (
"database/sql"
"fmt"
"log"
"github.com/cznic/ql"
)
func init() {
ql.RegisterDriver()
register("ql", qlSetup, qlCompile)
}
func qlSetup(db *sql.DB) (err error) {
tx, err := db.Begin()
if err != nil {
return
}
defer func() {
if err == nil {
err = tx.Commit()
} else {
tx.Rollback()
}
}()
_, err = tx.Exec(`CREATE TABLE IF NOT EXISTS Devices (
DeviceID STRING NOT NULL,
Seen TIME NOT NULL
)`)
if err != nil {
return
}
if _, err = tx.Exec(`CREATE INDEX IF NOT EXISTS DevicesDeviceIDIndex ON Devices (DeviceID)`); err != nil {
return
}
_, err = tx.Exec(`CREATE TABLE IF NOT EXISTS Addresses (
DeviceID STRING NOT NULL,
Seen TIME NOT NULL,
Address STRING NOT NULL,
)`)
if err != nil {
return
}
_, err = tx.Exec(`CREATE INDEX IF NOT EXISTS AddressesDeviceIDAddressIndex ON Addresses (DeviceID, Address)`)
return
}
func qlCompile(db *sql.DB) (map[string]*sql.Stmt, error) {
stmts := map[string]string{
"cleanAddress": `DELETE FROM Addresses WHERE Seen < now() - duration("2h")`,
"cleanDevice": fmt.Sprintf(`DELETE FROM Devices WHERE Seen < now() - duration("%dh")`, maxDeviceAge/3600),
"countAddress": "SELECT count(*) FROM Addresses",
"countDevice": "SELECT count(*) FROM Devices",
"insertAddress": "INSERT INTO Addresses (DeviceID, Seen, Address) VALUES ($1, now(), $2)",
"insertDevice": "INSERT INTO Devices (DeviceID, Seen) VALUES ($1, now())",
"selectAddress": `SELECT Address from Addresses WHERE DeviceID==$1 AND Seen > now() - duration("1h") LIMIT 16`,
"selectDevice": "SELECT Seen FROM Devices WHERE DeviceID==$1",
"updateAddress": "UPDATE Addresses Seen=now() WHERE DeviceID==$1 AND Address==$2",
"updateDevice": "UPDATE Devices Seen=now() WHERE DeviceID==$1",
}
res := make(map[string]*sql.Stmt, len(stmts))
for key, stmt := range stmts {
prep, err := db.Prepare(stmt)
if err != nil {
log.Println("Failed to compile", stmt)
return nil, err
}
res[key] = prep
}
return res, nil
}

492
cmd/stdiscosrv/querysrv.go Normal file
View File

@@ -0,0 +1,492 @@
// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
package main
import (
"bytes"
"crypto/tls"
"database/sql"
"encoding/json"
"encoding/pem"
"fmt"
"log"
"math/rand"
"net"
"net/http"
"net/url"
"strconv"
"sync"
"time"
"github.com/golang/groupcache/lru"
"github.com/syncthing/syncthing/lib/protocol"
"golang.org/x/net/context"
"golang.org/x/time/rate"
)
type querysrv struct {
addr string
db *sql.DB
prep map[string]*sql.Stmt
limiter *safeCache
cert tls.Certificate
listener net.Listener
}
type announcement struct {
Seen time.Time `json:"seen"`
Addresses []string `json:"addresses"`
}
type safeCache struct {
*lru.Cache
mut sync.Mutex
}
func (s *safeCache) Get(key string) (val interface{}, ok bool) {
s.mut.Lock()
val, ok = s.Cache.Get(key)
s.mut.Unlock()
return
}
func (s *safeCache) Add(key string, val interface{}) {
s.mut.Lock()
s.Cache.Add(key, val)
s.mut.Unlock()
}
type requestID int64
func (i requestID) String() string {
return fmt.Sprintf("%016x", int64(i))
}
type contextKey int
const idKey contextKey = iota
func negCacheFor(lastSeen time.Time) int {
since := time.Since(lastSeen).Seconds()
if since >= maxDeviceAge {
return maxNegCache
}
if since < 0 {
// That's weird
return minNegCache
}
// Return a value linearly scaled from minNegCache (at zero seconds ago)
// to maxNegCache (at maxDeviceAge seconds ago).
r := since / maxDeviceAge
return int(minNegCache + r*(maxNegCache-minNegCache))
}
func (s *querysrv) Serve() {
s.limiter = &safeCache{
Cache: lru.New(lruSize),
}
if useHTTP {
listener, err := net.Listen("tcp", s.addr)
if err != nil {
log.Println("Listen:", err)
return
}
s.listener = listener
} else {
tlsCfg := &tls.Config{
Certificates: []tls.Certificate{s.cert},
ClientAuth: tls.RequestClientCert,
SessionTicketsDisabled: true,
MinVersion: tls.VersionTLS12,
CipherSuites: []uint16{
tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
},
}
tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
if err != nil {
log.Println("Listen:", err)
return
}
s.listener = tlsListener
}
http.HandleFunc("/v2/", s.handler)
http.HandleFunc("/ping", handlePing)
srv := &http.Server{
ReadTimeout: 5 * time.Second,
WriteTimeout: 5 * time.Second,
MaxHeaderBytes: 1 << 10,
}
if err := srv.Serve(s.listener); err != nil {
log.Println("Serve:", err)
}
}
var topCtx = context.Background()
func (s *querysrv) handler(w http.ResponseWriter, req *http.Request) {
reqID := requestID(rand.Int63())
ctx := context.WithValue(topCtx, idKey, reqID)
if debug {
log.Println(reqID, req.Method, req.URL)
}
t0 := time.Now()
defer func() {
diff := time.Since(t0)
var comment string
if diff > time.Second {
comment = "(very slow request)"
} else if diff > 100*time.Millisecond {
comment = "(slow request)"
}
if comment != "" || debug {
log.Println(reqID, req.Method, req.URL, "completed in", diff, comment)
}
}()
var remoteIP net.IP
if useHTTP {
remoteIP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
} else {
addr, err := net.ResolveTCPAddr("tcp", req.RemoteAddr)
if err != nil {
log.Println("remoteAddr:", err)
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
remoteIP = addr.IP
}
if s.limit(remoteIP) {
if debug {
log.Println(remoteIP, "is limited")
}
w.Header().Set("Retry-After", "60")
http.Error(w, "Too Many Requests", 429)
return
}
switch req.Method {
case "GET":
s.handleGET(ctx, w, req)
case "POST":
s.handlePOST(ctx, remoteIP, w, req)
default:
globalStats.Error()
http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
}
}
func (s *querysrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http.Request) {
reqID := ctx.Value(idKey).(requestID)
deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
if err != nil {
if debug {
log.Println(reqID, "bad device param")
}
globalStats.Error()
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
var ann announcement
ann.Seen, err = s.getDeviceSeen(deviceID)
negCache := strconv.Itoa(negCacheFor(ann.Seen))
w.Header().Set("Retry-After", negCache)
w.Header().Set("Cache-Control", "public, max-age="+negCache)
if err != nil {
// The device is not in the database.
globalStats.Query()
http.Error(w, "Not Found", http.StatusNotFound)
return
}
t0 := time.Now()
ann.Addresses, err = s.getAddresses(ctx, deviceID)
if err != nil {
log.Println(reqID, "getAddresses:", err)
globalStats.Error()
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
if debug {
log.Println(reqID, "getAddresses in", time.Since(t0))
}
globalStats.Query()
if len(ann.Addresses) == 0 {
http.Error(w, "Not Found", http.StatusNotFound)
return
}
globalStats.Answer()
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode(ann)
}
func (s *querysrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.ResponseWriter, req *http.Request) {
reqID := ctx.Value(idKey).(requestID)
rawCert := certificateBytes(req)
if rawCert == nil {
if debug {
log.Println(reqID, "no certificates")
}
globalStats.Error()
http.Error(w, "Forbidden", http.StatusForbidden)
return
}
var ann announcement
if err := json.NewDecoder(req.Body).Decode(&ann); err != nil {
if debug {
log.Println(reqID, "decode:", err)
}
globalStats.Error()
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
deviceID := protocol.NewDeviceID(rawCert)
// handleAnnounce returns *two* errors. The first indicates a problem with
// something the client posted to us. We should return a 400 Bad Request
// and not worry about it. The second indicates that the request was fine,
// but something internal messed up. We should log it and respond with a
// more apologetic 500 Internal Server Error.
userErr, internalErr := s.handleAnnounce(ctx, remoteIP, deviceID, ann.Addresses)
if userErr != nil {
if debug {
log.Println(reqID, "handleAnnounce:", userErr)
}
globalStats.Error()
http.Error(w, "Bad Request", http.StatusBadRequest)
return
}
if internalErr != nil {
log.Println(reqID, "handleAnnounce:", internalErr)
globalStats.Error()
http.Error(w, "Internal Server Error", http.StatusInternalServerError)
return
}
globalStats.Announce()
// TODO: Slowly increase this for stable clients
w.Header().Set("Reannounce-After", "1800")
// We could return the lookup result here, but it's kind of unnecessarily
// expensive to go query the database again so we let the client decide to
// do a lookup if they really care.
w.WriteHeader(http.StatusNoContent)
}
func (s *querysrv) Stop() {
s.listener.Close()
}
func (s *querysrv) handleAnnounce(ctx context.Context, remote net.IP, deviceID protocol.DeviceID, addresses []string) (userErr, internalErr error) {
reqID := ctx.Value(idKey).(requestID)
tx, err := s.db.Begin()
if err != nil {
internalErr = err
return
}
defer func() {
// Since we return from a bunch of different places, we handle
// rollback in the defer.
if internalErr != nil || userErr != nil {
tx.Rollback()
}
}()
for _, annAddr := range addresses {
uri, err := url.Parse(annAddr)
if err != nil {
userErr = err
return
}
host, port, err := net.SplitHostPort(uri.Host)
if err != nil {
userErr = err
return
}
ip := net.ParseIP(host)
if host == "" || ip.IsUnspecified() {
// Do not use IPv6 remote address if requested scheme is tcp4
if uri.Scheme == "tcp4" && remote.To4() == nil {
continue
}
// Do not use IPv4 remote address if requested scheme is tcp6
if uri.Scheme == "tcp6" && remote.To4() != nil {
continue
}
host = remote.String()
}
uri.Host = net.JoinHostPort(host, port)
if err := s.updateAddress(ctx, tx, deviceID, uri.String()); err != nil {
internalErr = err
return
}
}
if err := s.updateDevice(ctx, tx, deviceID); err != nil {
internalErr = err
return
}
t0 := time.Now()
internalErr = tx.Commit()
if debug {
log.Println(reqID, "commit in", time.Since(t0))
}
return
}
func (s *querysrv) limit(remote net.IP) bool {
key := remote.String()
bkt, ok := s.limiter.Get(key)
if ok {
bkt := bkt.(*rate.Limiter)
if !bkt.Allow() {
// Rate limit exceeded; ignore packet
return true
}
} else {
// limitAvg is in packets per ten seconds.
s.limiter.Add(key, rate.NewLimiter(rate.Limit(limitAvg)/10, limitBurst))
}
return false
}
func (s *querysrv) updateDevice(ctx context.Context, tx *sql.Tx, device protocol.DeviceID) error {
reqID := ctx.Value(idKey).(requestID)
t0 := time.Now()
res, err := tx.Stmt(s.prep["updateDevice"]).Exec(device.String())
if err != nil {
return err
}
if debug {
log.Println(reqID, "updateDevice in", time.Since(t0))
}
if rows, _ := res.RowsAffected(); rows == 0 {
t0 = time.Now()
_, err := tx.Stmt(s.prep["insertDevice"]).Exec(device.String())
if err != nil {
return err
}
if debug {
log.Println(reqID, "insertDevice in", time.Since(t0))
}
}
return nil
}
func (s *querysrv) updateAddress(ctx context.Context, tx *sql.Tx, device protocol.DeviceID, uri string) error {
res, err := tx.Stmt(s.prep["updateAddress"]).Exec(device.String(), uri)
if err != nil {
return err
}
if rows, _ := res.RowsAffected(); rows == 0 {
_, err := tx.Stmt(s.prep["insertAddress"]).Exec(device.String(), uri)
if err != nil {
return err
}
}
return nil
}
func (s *querysrv) getAddresses(ctx context.Context, device protocol.DeviceID) ([]string, error) {
rows, err := s.prep["selectAddress"].Query(device.String())
if err != nil {
return nil, err
}
defer rows.Close()
var res []string
for rows.Next() {
var addr string
err := rows.Scan(&addr)
if err != nil {
log.Println("Scan:", err)
continue
}
res = append(res, addr)
}
return res, nil
}
func (s *querysrv) getDeviceSeen(device protocol.DeviceID) (time.Time, error) {
row := s.prep["selectDevice"].QueryRow(device.String())
var seen time.Time
if err := row.Scan(&seen); err != nil {
return time.Time{}, err
}
return seen.In(time.UTC), nil
}
func handlePing(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(204)
}
func certificateBytes(req *http.Request) []byte {
if req.TLS != nil && len(req.TLS.PeerCertificates) > 0 {
return req.TLS.PeerCertificates[0].Raw
}
if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
bs := []byte(hdr)
// The certificate is in PEM format but with spaces for newlines. We
// need to reinstate the newlines for the PEM decoder. But we need to
// leave the spaces in the BEGIN and END lines - the first and last
// space - alone.
firstSpace := bytes.Index(bs, []byte(" "))
lastSpace := bytes.LastIndex(bs, []byte(" "))
for i := firstSpace + 1; i < lastSpace; i++ {
if bs[i] == ' ' {
bs[i] = '\n'
}
}
block, _ := pem.Decode(bs)
if block == nil {
// Decoding failed
return nil
}
return block.Bytes
}
return nil
}

View File

@@ -1,326 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"crypto/tls"
"encoding/binary"
"fmt"
io "io"
"log"
"net"
"time"
"github.com/syncthing/syncthing/lib/protocol"
)
const replicationReadTimeout = time.Minute
const replicationHeartbeatInterval = time.Second * 30
type replicator interface {
send(key string, addrs []DatabaseAddress, seen int64)
}
// a replicationSender tries to connect to the remote address and provide
// them with a feed of replication updates.
type replicationSender struct {
dst string
cert tls.Certificate // our certificate
allowedIDs []protocol.DeviceID
outbox chan ReplicationRecord
stop chan struct{}
}
func newReplicationSender(dst string, cert tls.Certificate, allowedIDs []protocol.DeviceID) *replicationSender {
return &replicationSender{
dst: dst,
cert: cert,
allowedIDs: allowedIDs,
outbox: make(chan ReplicationRecord, replicationOutboxSize),
stop: make(chan struct{}),
}
}
func (s *replicationSender) Serve() {
// Sleep a little at startup. Peers often restart at the same time, and
// this avoid the service failing and entering backoff state
// unnecessarily, while also reducing the reconnect rate to something
// reasonable by default.
time.Sleep(2 * time.Second)
tlsCfg := &tls.Config{
Certificates: []tls.Certificate{s.cert},
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: true,
}
// Dial the TLS connection.
conn, err := tls.Dial("tcp", s.dst, tlsCfg)
if err != nil {
log.Println("Replication connect:", err)
return
}
defer func() {
conn.SetWriteDeadline(time.Now().Add(time.Second))
conn.Close()
}()
// Get the other side device ID.
remoteID, err := deviceID(conn)
if err != nil {
log.Println("Replication connect:", err)
return
}
// Verify it's in the set of allowed device IDs.
if !deviceIDIn(remoteID, s.allowedIDs) {
log.Println("Replication connect: unexpected device ID:", remoteID)
return
}
heartBeatTicker := time.NewTicker(replicationHeartbeatInterval)
defer heartBeatTicker.Stop()
// Send records.
buf := make([]byte, 1024)
for {
select {
case <-heartBeatTicker.C:
if len(s.outbox) > 0 {
// No need to send heartbeats if there are events/prevrious
// heartbeats to send, they will keep the connection alive.
continue
}
// Empty replication message is the heartbeat:
s.outbox <- ReplicationRecord{}
case rec := <-s.outbox:
// Buffer must hold record plus four bytes for size
size := rec.Size()
if len(buf) < size+4 {
buf = make([]byte, size+4)
}
// Record comes after the four bytes size
n, err := rec.MarshalTo(buf[4:])
if err != nil {
// odd to get an error here, but we haven't sent anything
// yet so it's not fatal
replicationSendsTotal.WithLabelValues("error").Inc()
log.Println("Replication marshal:", err)
continue
}
binary.BigEndian.PutUint32(buf, uint32(n))
// Send
conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
if _, err := conn.Write(buf[:4+n]); err != nil {
replicationSendsTotal.WithLabelValues("error").Inc()
log.Println("Replication write:", err)
// Yes, we are loosing the replication event here.
return
}
replicationSendsTotal.WithLabelValues("success").Inc()
case <-s.stop:
return
}
}
}
func (s *replicationSender) Stop() {
close(s.stop)
}
func (s *replicationSender) String() string {
return fmt.Sprintf("replicationSender(%q)", s.dst)
}
func (s *replicationSender) send(key string, ps []DatabaseAddress, seen int64) {
item := ReplicationRecord{
Key: key,
Addresses: ps,
}
// The send should never block. The inbox is suitably buffered for at
// least a few seconds of stalls, which shouldn't happen in practice.
select {
case s.outbox <- item:
default:
replicationSendsTotal.WithLabelValues("drop").Inc()
}
}
// a replicationMultiplexer sends to multiple replicators
type replicationMultiplexer []replicator
func (m replicationMultiplexer) send(key string, ps []DatabaseAddress, seen int64) {
for _, s := range m {
// each send is nonblocking
s.send(key, ps, seen)
}
}
// replicationListener accepts incoming connections and reads replication
// items from them. Incoming items are applied to the KV store.
type replicationListener struct {
addr string
cert tls.Certificate
allowedIDs []protocol.DeviceID
db database
stop chan struct{}
}
func newReplicationListener(addr string, cert tls.Certificate, allowedIDs []protocol.DeviceID, db database) *replicationListener {
return &replicationListener{
addr: addr,
cert: cert,
allowedIDs: allowedIDs,
db: db,
stop: make(chan struct{}),
}
}
func (l *replicationListener) Serve() {
tlsCfg := &tls.Config{
Certificates: []tls.Certificate{l.cert},
ClientAuth: tls.RequestClientCert,
MinVersion: tls.VersionTLS12,
InsecureSkipVerify: true,
}
lst, err := tls.Listen("tcp", l.addr, tlsCfg)
if err != nil {
log.Println("Replication listen:", err)
return
}
defer lst.Close()
for {
select {
case <-l.stop:
return
default:
}
// Accept a connection
conn, err := lst.Accept()
if err != nil {
log.Println("Replication accept:", err)
return
}
// Figure out the other side device ID
remoteID, err := deviceID(conn.(*tls.Conn))
if err != nil {
log.Println("Replication accept:", err)
conn.SetWriteDeadline(time.Now().Add(time.Second))
conn.Close()
continue
}
// Verify it is in the set of allowed device IDs
if !deviceIDIn(remoteID, l.allowedIDs) {
log.Println("Replication accept: unexpected device ID:", remoteID)
conn.SetWriteDeadline(time.Now().Add(time.Second))
conn.Close()
continue
}
go l.handle(conn)
}
}
func (l *replicationListener) Stop() {
close(l.stop)
}
func (l *replicationListener) String() string {
return fmt.Sprintf("replicationListener(%q)", l.addr)
}
func (l *replicationListener) handle(conn net.Conn) {
defer func() {
conn.SetWriteDeadline(time.Now().Add(time.Second))
conn.Close()
}()
buf := make([]byte, 1024)
for {
select {
case <-l.stop:
return
default:
}
conn.SetReadDeadline(time.Now().Add(replicationReadTimeout))
// First four bytes are the size
if _, err := io.ReadFull(conn, buf[:4]); err != nil {
log.Println("Replication read size:", err)
replicationRecvsTotal.WithLabelValues("error").Inc()
return
}
// Read the rest of the record
size := int(binary.BigEndian.Uint32(buf[:4]))
if len(buf) < size {
buf = make([]byte, size)
}
if size == 0 {
// Heartbeat, ignore
continue
}
if _, err := io.ReadFull(conn, buf[:size]); err != nil {
log.Println("Replication read record:", err)
replicationRecvsTotal.WithLabelValues("error").Inc()
return
}
// Unmarshal
var rec ReplicationRecord
if err := rec.Unmarshal(buf[:size]); err != nil {
log.Println("Replication unmarshal:", err)
replicationRecvsTotal.WithLabelValues("error").Inc()
continue
}
// Store
l.db.merge(rec.Key, rec.Addresses, rec.Seen)
replicationRecvsTotal.WithLabelValues("success").Inc()
}
}
func deviceID(conn *tls.Conn) (protocol.DeviceID, error) {
// Handshake may not be complete on the server side yet, which we need
// to get the client certificate.
if !conn.ConnectionState().HandshakeComplete {
if err := conn.Handshake(); err != nil {
return protocol.DeviceID{}, err
}
}
// We expect exactly one certificate.
certs := conn.ConnectionState().PeerCertificates
if len(certs) != 1 {
return protocol.DeviceID{}, fmt.Errorf("unexpected number of certificates (%d != 1)", len(certs))
}
return protocol.NewDeviceID(certs[0].Raw), nil
}
func deviceIDIn(id protocol.DeviceID, ids []protocol.DeviceID) bool {
for _, candidate := range ids {
if id == candidate {
return true
}
}
return false
}

View File

@@ -1,123 +1,141 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
package main
import (
"bytes"
"database/sql"
"fmt"
"io/ioutil"
"log"
"os"
"github.com/prometheus/client_golang/prometheus"
"sync/atomic"
"time"
)
var (
apiRequestsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "api_requests_total",
Help: "Number of API requests.",
}, []string{"type", "result"})
apiRequestsSeconds = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "api_requests_seconds",
Help: "Latency of API requests.",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
}, []string{"type"})
type stats struct {
// Incremented atomically
announces int64
queries int64
answers int64
errors int64
}
lookupRequestsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "lookup_requests_total",
Help: "Number of lookup requests.",
}, []string{"result"})
announceRequestsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "announcement_requests_total",
Help: "Number of announcement requests.",
}, []string{"result"})
func (s *stats) Announce() {
atomic.AddInt64(&s.announces, 1)
}
replicationSendsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "replication_sends_total",
Help: "Number of replication sends.",
}, []string{"result"})
replicationRecvsTotal = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "replication_recvs_total",
Help: "Number of replication receives.",
}, []string{"result"})
func (s *stats) Query() {
atomic.AddInt64(&s.queries, 1)
}
databaseKeys = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "database_keys",
Help: "Number of database keys at last count.",
}, []string{"category"})
databaseStatisticsSeconds = prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "database_statistics_seconds",
Help: "Time spent running the statistics routine.",
})
func (s *stats) Answer() {
atomic.AddInt64(&s.answers, 1)
}
databaseOperations = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "database_operations_total",
Help: "Number of database operations.",
}, []string{"operation", "result"})
databaseOperationSeconds = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: "syncthing",
Subsystem: "discovery",
Name: "database_operation_seconds",
Help: "Latency of database operations.",
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
}, []string{"operation"})
)
func (s *stats) Error() {
atomic.AddInt64(&s.errors, 1)
}
const (
dbOpGet = "get"
dbOpPut = "put"
dbOpMerge = "merge"
dbOpDelete = "delete"
dbResSuccess = "success"
dbResNotFound = "not_found"
dbResError = "error"
dbResUnmarshalError = "unmarsh_err"
)
func init() {
prometheus.MustRegister(apiRequestsTotal, apiRequestsSeconds,
lookupRequestsTotal, announceRequestsTotal,
replicationSendsTotal, replicationRecvsTotal,
databaseKeys, databaseStatisticsSeconds,
databaseOperations, databaseOperationSeconds)
processCollectorOpts := prometheus.ProcessCollectorOpts{
Namespace: "syncthing_discovery",
PidFn: func() (int, error) {
return os.Getpid(), nil
},
// Reset returns a copy of the current stats and resets the counters to
// zero.
func (s *stats) Reset() stats {
// Create a copy of the stats using atomic reads
copy := stats{
announces: atomic.LoadInt64(&s.announces),
queries: atomic.LoadInt64(&s.queries),
answers: atomic.LoadInt64(&s.answers),
errors: atomic.LoadInt64(&s.errors),
}
prometheus.MustRegister(
prometheus.NewProcessCollector(processCollectorOpts),
)
// Reset the stats by subtracting the values that we copied
atomic.AddInt64(&s.announces, -copy.announces)
atomic.AddInt64(&s.queries, -copy.queries)
atomic.AddInt64(&s.answers, -copy.answers)
atomic.AddInt64(&s.errors, -copy.errors)
return copy
}
type statssrv struct {
intv time.Duration
file string
db *sql.DB
}
func (s *statssrv) Serve() {
lastReset := time.Now()
for {
time.Sleep(next(s.intv))
stats := globalStats.Reset()
d := time.Since(lastReset).Seconds()
lastReset = time.Now()
log.Printf("Stats: %.02f announces/s, %.02f queries/s, %.02f answers/s, %.02f errors/s",
float64(stats.announces)/d, float64(stats.queries)/d, float64(stats.answers)/d, float64(stats.errors)/d)
if s.file != "" {
s.writeToFile(stats, d)
}
}
}
func (s *statssrv) Stop() {
panic("stop unimplemented")
}
func (s *statssrv) writeToFile(stats stats, secs float64) {
newLine := []byte("\n")
var addrs int
row := s.db.QueryRow("SELECT COUNT(*) FROM Addresses")
if err := row.Scan(&addrs); err != nil {
log.Println("stats query:", err)
return
}
fd, err := os.OpenFile(s.file, os.O_RDWR|os.O_CREATE, 0666)
if err != nil {
log.Println("stats file:", err)
return
}
defer func() {
err = fd.Close()
if err != nil {
log.Println("stats file:", err)
}
}()
bs, err := ioutil.ReadAll(fd)
if err != nil {
log.Println("stats file:", err)
return
}
lines := bytes.Split(bytes.TrimSpace(bs), newLine)
if len(lines) > 12 {
lines = lines[len(lines)-12:]
}
latest := fmt.Sprintf("%v: %6d addresses, %8.02f announces/s, %8.02f queries/s, %8.02f answers/s, %8.02f errors/s\n",
time.Now().UTC().Format(time.RFC3339), addrs,
float64(stats.announces)/secs, float64(stats.queries)/secs, float64(stats.answers)/secs, float64(stats.errors)/secs)
lines = append(lines, []byte(latest))
_, err = fd.Seek(0, 0)
if err != nil {
log.Println("stats file:", err)
return
}
err = fd.Truncate(0)
if err != nil {
log.Println("stats file:", err)
return
}
_, err = fd.Write(bytes.Join(lines, newLine))
if err != nil {
log.Println("stats file:", err)
return
}
}

View File

@@ -68,8 +68,8 @@ func main() {
}
blockSize := int(fi.Size())
if *standardBlocks || blockSize < protocol.MinBlockSize {
blockSize = protocol.BlockSize(fi.Size())
if *standardBlocks || blockSize < protocol.BlockSize {
blockSize = protocol.BlockSize
}
bs, err := scanner.Blocks(context.TODO(), fd, blockSize, fi.Size(), nil, true)
if err != nil {

View File

@@ -1,44 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
// Commmand stfindignored lists ignored files under a given folder root.
package main
import (
"flag"
"fmt"
"os"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/ignore"
)
func main() {
flag.Parse()
root := flag.Arg(0)
if root == "" {
root = "."
}
vfs := fs.NewWalkFilesystem(fs.NewFilesystem(fs.FilesystemTypeBasic, root))
ign := ignore.New(vfs)
if err := ign.Load(".stignore"); err != nil {
fmt.Fprintf(os.Stderr, "Fatal: loading ignores: %v\n", err)
os.Exit(1)
}
vfs.Walk(".", func(path string, info fs.FileInfo, err error) error {
if err != nil {
fmt.Fprintf(os.Stderr, "Warning: %s: %v\n", path, err)
return fs.SkipDir
}
if ign.Match(path).IsIgnored() {
fmt.Println(path)
}
return nil
})
}

View File

@@ -82,7 +82,7 @@ func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
return err
}
os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
_ = os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
t := time.Now().Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
return os.Chtimes(p1, t, t)

View File

@@ -16,7 +16,7 @@ import (
"github.com/syncthing/syncthing/lib/protocol"
)
func dump(ldb *db.Lowlevel) {
func dump(ldb *db.Instance) {
it := ldb.NewIterator(nil, nil)
for it.Next() {
key := it.Key()

View File

@@ -37,7 +37,7 @@ func (h *ElementHeap) Pop() interface{} {
return x
}
func dumpsize(ldb *db.Lowlevel) {
func dumpsize(ldb *db.Instance) {
h := &ElementHeap{}
heap.Init(h)

View File

@@ -1,242 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bytes"
"encoding/binary"
"fmt"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/protocol"
)
type fileInfoKey struct {
folder uint32
device uint32
name string
}
type globalKey struct {
folder uint32
name string
}
type sequenceKey struct {
folder uint32
sequence uint64
}
func idxck(ldb *db.Lowlevel) (success bool) {
folders := make(map[uint32]string)
devices := make(map[uint32]string)
deviceToIDs := make(map[string]uint32)
fileInfos := make(map[fileInfoKey]protocol.FileInfo)
globals := make(map[globalKey]db.VersionList)
sequences := make(map[sequenceKey]string)
needs := make(map[globalKey]struct{})
var localDeviceKey uint32
success = true
it := ldb.NewIterator(nil, nil)
for it.Next() {
key := it.Key()
switch key[0] {
case db.KeyTypeDevice:
folder := binary.BigEndian.Uint32(key[1:])
device := binary.BigEndian.Uint32(key[1+4:])
name := nulString(key[1+4+4:])
var f protocol.FileInfo
err := f.Unmarshal(it.Value())
if err != nil {
fmt.Println("Unable to unmarshal FileInfo:", err)
success = false
continue
}
fileInfos[fileInfoKey{folder, device, name}] = f
case db.KeyTypeGlobal:
folder := binary.BigEndian.Uint32(key[1:])
name := nulString(key[1+4:])
var flv db.VersionList
if err := flv.Unmarshal(it.Value()); err != nil {
fmt.Println("Unable to unmarshal VersionList:", err)
success = false
continue
}
globals[globalKey{folder, name}] = flv
case db.KeyTypeFolderIdx:
key := binary.BigEndian.Uint32(it.Key()[1:])
folders[key] = string(it.Value())
case db.KeyTypeDeviceIdx:
key := binary.BigEndian.Uint32(it.Key()[1:])
devices[key] = string(it.Value())
deviceToIDs[string(it.Value())] = key
if bytes.Equal(it.Value(), protocol.LocalDeviceID[:]) {
localDeviceKey = key
}
case db.KeyTypeSequence:
folder := binary.BigEndian.Uint32(key[1:])
seq := binary.BigEndian.Uint64(key[5:])
val := it.Value()
sequences[sequenceKey{folder, seq}] = string(val[9:])
case db.KeyTypeNeed:
folder := binary.BigEndian.Uint32(key[1:])
name := nulString(key[1+4:])
needs[globalKey{folder, name}] = struct{}{}
}
}
if localDeviceKey == 0 {
fmt.Println("Missing key for local device in device index (bailing out)")
success = false
return
}
for fk, fi := range fileInfos {
if fk.name != fi.Name {
fmt.Printf("Mismatching FileInfo name, %q (key) != %q (actual)\n", fk.name, fi.Name)
success = false
}
folder := folders[fk.folder]
if folder == "" {
fmt.Printf("Unknown folder ID %d for FileInfo %q\n", fk.folder, fk.name)
success = false
continue
}
if devices[fk.device] == "" {
fmt.Printf("Unknown device ID %d for FileInfo %q, folder %q\n", fk.folder, fk.name, folder)
success = false
}
if fk.device == localDeviceKey {
name, ok := sequences[sequenceKey{fk.folder, uint64(fi.Sequence)}]
if !ok {
fmt.Printf("Sequence entry missing for FileInfo %q, folder %q, seq %d\n", fi.Name, folder, fi.Sequence)
success = false
continue
}
if name != fi.Name {
fmt.Printf("Sequence entry refers to wrong name, %q (seq) != %q (FileInfo), folder %q, seq %d\n", name, fi.Name, folder, fi.Sequence)
success = false
}
}
}
for gk, vl := range globals {
folder := folders[gk.folder]
if folder == "" {
fmt.Printf("Unknown folder ID %d for VersionList %q\n", gk.folder, gk.name)
success = false
}
for i, fv := range vl.Versions {
dev, ok := deviceToIDs[string(fv.Device)]
if !ok {
fmt.Printf("VersionList %q, folder %q refers to unknown device %q\n", gk.name, folder, fv.Device)
success = false
}
fi, ok := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
if !ok {
fmt.Printf("VersionList %q, folder %q, entry %d refers to unknown FileInfo\n", gk.name, folder, i)
success = false
}
if !fi.Version.Equal(fv.Version) {
fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo version mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Version, fi.Version)
success = false
}
if fi.IsInvalid() != fv.Invalid {
fmt.Printf("VersionList %q, folder %q, entry %d, FileInfo invalid mismatch, %v (VersionList) != %v (FileInfo)\n", gk.name, folder, i, fv.Invalid, fi.IsInvalid())
success = false
}
}
// If we need this file we should have a need entry for it. False
// positives from needsLocally for deleted files, where we might
// legitimately lack an entry if we never had it, and ignored files.
if needsLocally(vl) {
_, ok := needs[gk]
if !ok {
dev := deviceToIDs[string(vl.Versions[0].Device)]
fi := fileInfos[fileInfoKey{gk.folder, dev, gk.name}]
if !fi.IsDeleted() && !fi.IsIgnored() {
fmt.Printf("Missing need entry for needed file %q, folder %q\n", gk.name, folder)
}
}
}
}
seenSeq := make(map[fileInfoKey]uint64)
for sk, name := range sequences {
folder := folders[sk.folder]
if folder == "" {
fmt.Printf("Unknown folder ID %d for sequence entry %d, %q\n", sk.folder, sk.sequence, name)
success = false
continue
}
if prev, ok := seenSeq[fileInfoKey{folder: sk.folder, name: name}]; ok {
fmt.Printf("Duplicate sequence entry for %q, folder %q, seq %d (prev %d)\n", name, folder, sk.sequence, prev)
success = false
}
seenSeq[fileInfoKey{folder: sk.folder, name: name}] = sk.sequence
fi, ok := fileInfos[fileInfoKey{sk.folder, localDeviceKey, name}]
if !ok {
fmt.Printf("Missing FileInfo for sequence entry %d, folder %q, %q\n", sk.sequence, folder, name)
success = false
continue
}
if fi.Sequence != int64(sk.sequence) {
fmt.Printf("Sequence mismatch for %q, folder %q, %d (key) != %d (FileInfo)\n", name, folder, sk.sequence, fi.Sequence)
success = false
}
}
for nk := range needs {
folder := folders[nk.folder]
if folder == "" {
fmt.Printf("Unknown folder ID %d for need entry %q\n", nk.folder, nk.name)
success = false
continue
}
vl, ok := globals[nk]
if !ok {
fmt.Printf("Missing global for need entry %q, folder %q\n", nk.name, folder)
success = false
continue
}
if !needsLocally(vl) {
fmt.Printf("Need entry for file we don't need, %q, folder %q\n", nk.name, folder)
success = false
}
}
return
}
func needsLocally(vl db.VersionList) bool {
var lv *protocol.Vector
for _, fv := range vl.Versions {
if bytes.Equal(fv.Device, protocol.LocalDeviceID[:]) {
lv = &fv.Version
break
}
}
if lv == nil {
return true // proviosinally, it looks like we need the file
}
return !lv.GreaterEqual(vl.Versions[0].Version)
}

View File

@@ -21,7 +21,7 @@ func main() {
log.SetFlags(0)
log.SetOutput(os.Stdout)
flag.StringVar(&mode, "mode", "dump", "Mode of operation: dump, dumpsize, idxck")
flag.StringVar(&mode, "mode", "dump", "Mode of operation: dump, dumpsize")
flag.Parse()
@@ -30,7 +30,9 @@ func main() {
path = filepath.Join(defaultConfigDir(), "index-v0.14.0.db")
}
ldb, err := db.OpenRO(path)
fmt.Println("Path:", path)
ldb, err := db.Open(path)
if err != nil {
log.Fatal(err)
}
@@ -39,10 +41,6 @@ func main() {
dump(ldb)
} else if mode == "dumpsize" {
dumpsize(ldb)
} else if mode == "idxck" {
if !idxck(ldb) {
os.Exit(1)
}
} else {
fmt.Println("Unknown mode")
}

View File

@@ -12,7 +12,7 @@ import (
"path/filepath"
"runtime"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/osutil"
)
func nulString(bs []byte) string {
@@ -33,7 +33,7 @@ func defaultConfigDir() string {
return filepath.Join(os.Getenv("AppData"), "Syncthing")
case "darwin":
dir, err := fs.ExpandTilde("~/Library/Application Support/Syncthing")
dir, err := osutil.ExpandTilde("~/Library/Application Support/Syncthing")
if err != nil {
log.Fatal(err)
}
@@ -43,7 +43,7 @@ func defaultConfigDir() string {
if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
return filepath.Join(xdgCfg, "syncthing")
}
dir, err := fs.ExpandTilde("~/.config/syncthing")
dir, err := osutil.ExpandTilde("~/.config/syncthing")
if err != nil {
log.Fatal(err)
}

View File

@@ -1 +1 @@
gui.files.go
gui.go

View File

@@ -10,7 +10,7 @@
<title>Relay stats</title>
<link href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet"/>
<link rel="stylesheet" href="//use.fontawesome.com/releases/v5.0.13/css/all.css"/>
<link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/font-awesome/4.6.1/css/font-awesome.min.css"/>
<style>
#map {
@@ -56,83 +56,83 @@
<tr>
<th rowspan="2">Address</td>
<th rowspan="2">
<a ng-click="sortType = 'stats.numActiveSessions'; sortReverse = !sortReverse">
<a ng-click="sortType = 'status.numActiveSessions'; sortReverse = !sortReverse">
Sessions
<span ng-show="sortType == 'stats.numActiveSessions' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.numActiveSessions' && sortReverse" class="fas fa-caret-up"></span>
<span ng-show="sortType == 'status.numActiveSessions' && !sortReverse" class="fa fa-caret-down"></span>
<span ng-show="sortType == 'status.numActiveSessions' && sortReverse" class="fa fa-caret-up"></span>
</a>
</th>
<th rowspan="2">
<a ng-click="sortType = 'stats.numConnections'; sortReverse = !sortReverse">
<a ng-click="sortType = 'status.numConnections'; sortReverse = !sortReverse">
Connections
<span ng-show="sortType == 'stats.numConnections' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.numConnections' && sortReverse" class="fas fa-caret-up"></span>
<span ng-show="sortType == 'status.numConnections' && !sortReverse" class="fa fa-caret-down"></span>
<span ng-show="sortType == 'status.numConnections' && sortReverse" class="fa fa-caret-up"></span>
</a>
</th>
<th rowspan="2">
<a ng-click="sortType = 'stats.bytesProxied'; sortReverse = !sortReverse">
<a ng-click="sortType = 'status.bytesProxied'; sortReverse = !sortReverse">
Data relayed
<span ng-show="sortType == 'stats.bytesProxied' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.bytesProxied' && sortReverse" class="fas fa-caret-up"></span>
<span ng-show="sortType == 'status.bytesProxied' && !sortReverse" class="fa fa-caret-down"></span>
<span ng-show="sortType == 'status.bytesProxied' && sortReverse" class="fa fa-caret-up"></span>
</a>
</th>
<th colspan="6" class="text-center">Transfer rate in the last period</th>
<th rowspan="2">
<a ng-click="sortType = 'stats.uptimeSeconds'; sortReverse = !sortReverse">
<a ng-click="sortType = 'status.uptimeSeconds'; sortReverse = !sortReverse">
Uptime hours
<span ng-show="sortType == 'stats.uptimeSeconds' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'status.uptimeSeconds' && sortReverse" class="fas fa-caret-up"></span>
<span ng-show="sortType == 'status.uptimeSeconds' && !sortReverse" class="fa fa-caret-down"></span>
<span ng-show="sortType == 'status.uptimeSeconds' && sortReverse" class="fa fa-caret-up"></span>
</a>
</th>
<th rowspan="2">
<a ng-click="sortType = 'stats.options[\'provided-by\'] || \'\''; sortReverse = !sortReverse">
<a ng-click="sortType = 'status.options[\'provided-by\'] || \'\''; sortReverse = !sortReverse">
Provided by
<span ng-show="sortType == 'stats.options[\'provided-by\'] || \'\'' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.options[\'provided-by\'] || \'\'' && sortReverse" class="fas fa-caret-up"></span>
<span ng-show="sortType == 'status.options[\'provided-by\'] || \'\'' && !sortReverse" class="fa fa-caret-down"></span>
<span ng-show="sortType == 'status.options[\'provided-by\'] || \'\'' && sortReverse" class="fa fa-caret-up"></span>
</a>
</th>
</tr>
<tr>
<th>
<a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[0]'; sortReverse = !sortReverse">
<a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[0]'; sortReverse = !sortReverse">
10s
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[0]' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[0]' && sortReverse" class="fas fa-caret-up"></span>
<span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[0]' && !sortReverse" class="fa fa-caret-down"></span>
<span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[0]' && sortReverse" class="fa fa-caret-up"></span>
</a>
</th>
<th>
<a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[1]'; sortReverse = !sortReverse">
<a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[1]'; sortReverse = !sortReverse">
1m
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[1]' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[1]' && sortReverse" class="fas fa-caret-up"></span>
<span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[1]' && !sortReverse" class="fa fa-caret-down"></span>
<span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[1]' && sortReverse" class="fa fa-caret-up"></span>
</a>
</th>
<th>
<a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[2]'; sortReverse = !sortReverse">
<a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[2]'; sortReverse = !sortReverse">
5m
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[2]' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[2]' && sortReverse" class="fas fa-caret-up"></span>
<span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[2]' && !sortReverse" class="fa fa-caret-down"></span>
<span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[2]' && sortReverse" class="fa fa-caret-up"></span>
</a>
</th>
<th>
<a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[3]'; sortReverse = !sortReverse">
<a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[3]'; sortReverse = !sortReverse">
15m
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[3]' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[3]' && sortReverse" class="fas fa-caret-up"></span>
<span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[3]' && !sortReverse" class="fa fa-caret-down"></span>
<span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[3]' && sortReverse" class="fa fa-caret-up"></span>
</a>
</th>
<th>
<a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[4]'; sortReverse = !sortReverse">
<a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[4]'; sortReverse = !sortReverse">
30m
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[4]' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[4]' && sortReverse" class="fas fa-caret-up"></span>
<span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[4]' && !sortReverse" class="fa fa-caret-down"></span>
<span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[4]' && sortReverse" class="fa fa-caret-up"></span>
</a>
</th>
<th>
<a ng-click="sortType = 'stats.kbps10s1m5m15m30m60m[5]'; sortReverse = !sortReverse">
<a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[5]'; sortReverse = !sortReverse">
60m
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[5]' && !sortReverse" class="fas fa-caret-down"></span>
<span ng-show="sortType == 'stats.kbps10s1m5m15m30m60m[5]' && sortReverse" class="fas fa-caret-up"></span>
<span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[5]' && !sortReverse" class="fa fa-caret-down"></span>
<span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[5]' && sortReverse" class="fa fa-caret-up"></span>
</a>
</th>
</tr>
@@ -140,21 +140,21 @@
<tbody>
<tr ng-repeat="relay in relays | orderBy:sortType:sortReverse:sortCompare" ng-mouseover="relay.showMarker()" ng-mouseleave="relay.hideMarker()">
<td>{{ relay.address }}</td>
<td ng-if="!relay.stats" colspan="11"></td>
<td ng-if-start="relay.stats">{{ relay.stats.numActiveSessions }}</td>
<td>{{ relay.stats.numConnections }}</td>
<td>{{ relay.stats.bytesProxied | bytes }}</td>
<td>{{ relay.stats.kbps10s1m5m15m30m60m[0] * 128 | bytes }}/s</td>
<td>{{ relay.stats.kbps10s1m5m15m30m60m[1] * 128 | bytes }}/s</td>
<td>{{ relay.stats.kbps10s1m5m15m30m60m[2] * 128 | bytes }}/s</td>
<td>{{ relay.stats.kbps10s1m5m15m30m60m[3] * 128 | bytes }}/s</td>
<td>{{ relay.stats.kbps10s1m5m15m30m60m[4] * 128 | bytes }}/s</td>
<td>{{ relay.stats.kbps10s1m5m15m30m60m[5] * 128 | bytes }}/s</td>
<td ng-if="relay.stats.uptimeSeconds != undefined">{{ relay.stats.uptimeSeconds/60/60 | number:0 }}</td>
<td ng-if="relay.stats.uptimeSeconds == undefined"></td>
<td title="{{ relay.stats.options['provided-by'] || '' }}" ng-if-end>
{{ relay.stats.options['provided-by'] || '' | limitTo:50 }}
<span ng-if="(relay.stats.options['provided-by'] || '').length > 50">&hellip;
<td ng-if="relay.status === undefined" colspan="11" class="text-center">Looking up...</td>
<td ng-if-start="relay.status !== undefined">{{ relay.status.numActiveSessions }}</td>
<td>{{ relay.status.numConnections }}</td>
<td>{{ relay.status.bytesProxied | bytes }}</td>
<td>{{ relay.status.kbps10s1m5m15m30m60m[0] * 128 | bytes }}/s</td>
<td>{{ relay.status.kbps10s1m5m15m30m60m[1] * 128 | bytes }}/s</td>
<td>{{ relay.status.kbps10s1m5m15m30m60m[2] * 128 | bytes }}/s</td>
<td>{{ relay.status.kbps10s1m5m15m30m60m[3] * 128 | bytes }}/s</td>
<td>{{ relay.status.kbps10s1m5m15m30m60m[4] * 128 | bytes }}/s</td>
<td>{{ relay.status.kbps10s1m5m15m30m60m[5] * 128 | bytes }}/s</td>
<td ng-if="relay.status.uptimeSeconds != undefined">{{ relay.status.uptimeSeconds/60/60 | number:0 }}</td>
<td ng-if="relay.status.uptimeSeconds == undefined"></td>
<td title="{{ relay.status.options['provided-by'] || '' }}" ng-if-end>
{{ relay.status.options['provided-by'] || '' | limitTo:50 }}
<span ng-if="(relay.status.options['provided-by'] || '').length > 50">&hellip;
</td>
</tr>
</tbody>
@@ -187,7 +187,7 @@
<script type="text/javascript" src="//code.jquery.com/jquery-2.1.4.min.js"></script>
<script type="text/javascript" src="//cdnjs.cloudflare.com/ajax/libs/angular.js/1.5.8/angular.min.js"></script>
<script type="text/javascript" src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
<script type="text/javascript" src="//maps.googleapis.com/maps/api/js?key=AIzaSyDk5WJ8s7ueLKb99X5DbQ-vkWtPDAKqYs0"></script>
<script type="text/javascript" src="//maps.googleapis.com/maps/api/js"></script>
</body>
<script>
@@ -235,16 +235,16 @@
$scope.mapBounds = new google.maps.LatLngBounds();
$scope.tooltipTemplate = $('#infoTemplate').html();
$scope.usedLocations = {};
$scope.sortType = 'stats.numActiveSessions';
$scope.sortType = 'status.numActiveSessions';
$scope.sortReverse = true;
$scope.sortCompare = function(a, b) {
if (a.value == b.value) {
return 0;
}
if (a.type == "undefined" || a.type == "null") {
if (a.type == "undefined") {
return -1;
}
if (b.type == "undefined" || b.type == "null") {
if (b.type == "undefined") {
return 1;
}
return a.value > b.value ? 1 : -1;
@@ -252,31 +252,25 @@
$http.get("/endpoint").then(function(response) {
$scope.relays = response.data.relays;
var promises = [];
angular.forEach($scope.relays, function(relay) {
relay.uri = constructURI(relay.url);
relay.address = relay.url.split('/')[2];
addMarkerToMap(relay);
if (relay.stats) {
angular.forEach($scope.totals, function(value, key) {
if (typeof $scope.totals[key] == 'number') {
$scope.totals[key] += relay.stats[key];
} else if (typeof $scope.totals[key] == 'object' && $scope.totals[key] instanceof Array) {
angular.forEach($scope.totals[key], function(value, index) {
$scope.totals[key][index] += relay.stats[key][index];
});
}
});
}
promises.push(getRelayStatus(relay));
});
// After the totals were calculated, add circles.
angular.forEach($scope.relays, function(relay) {
if (relay.stats) {
addCircleToMap(relay);
}
// Can only add circles once we know the totals for transfers, which means
// we need to resolve all statuses.
$q.all(promises).then(function() {
angular.forEach($scope.relays, function(relay) {
if (relay.status) {
addCircleToMap(relay);
}
});
});
$scope.map.fitBounds($scope.mapBounds);
@@ -336,10 +330,41 @@
fillOpacity: 0.35,
map: $scope.map,
center: relay.marker.position,
radius: ((relay.stats.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000
radius: ((relay.status.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000
});
}
function getRelayStatus(relay) {
// Normal timeout doesn't deal with relays which accept the TCP connection
// but don't respond (some firewalls do that), so deal with it this way.
var timeoutRequest = $q.defer();
var resolveStatus = $q.defer();
$http.get("http://" + relay.uri.hostname + ':' + ((relay.uri.args.statusAddr && relay.uri.args.statusAddr.split(':')[1]) || "22070") + "/status", { timeout: timeoutRequest.promise }).then(function (response) {
relay.status = response.data;
resolveStatus.resolve();
angular.forEach($scope.totals, function(value, key) {
if (typeof $scope.totals[key] == 'number') {
$scope.totals[key] += response.data[key];
} else if (typeof $scope.totals[key] == 'object' && $scope.totals[key] instanceof Array) {
angular.forEach($scope.totals[key], function(value, index) {
$scope.totals[key][index] += response.data[key][index];
});
}
});
}, function() {
relay.status = null;
resolveStatus.resolve();
});
$timeout(function() {
timeoutRequest.resolve();
}, 5000);
return resolveStatus.promise;
}
function constructURI(url) {
var uri = document.createElement('a');
@@ -360,25 +385,25 @@
<script type="text/template" id="infoTemplate">
<div>
<p><b>{{ relay.uri.hostname }}</b> <span ng-if="relay.stats.options['provided-by']">provided by <u>{{ relay.stats.options['provided-by'] }}</u></span></p>
<div ng-if="relay.stats">
<span ng-if="relay.stats.startTime">Start time: {{ relay.stats.startTime | date:"medium" }}</br></span>
<span ng-if="relay.stats.bytesProxied != undefined">Proxied: {{ relay.stats.bytesProxied | bytes }}</br></span>
<span ng-if="relay.stats.numActiveSessions != undefined">Sessions: {{ relay.stats.numActiveSessions }}</br></span>
<span ng-if="relay.stats.numConnections != undefined">Clients: {{ relay.stats.numConnections }}</br></span>
<span ng-if="relay.stats.options.pools">Pools: {{ relay.stats.options.pools.join(', ') }}</br></span>
<span ng-if="relay.stats.options['global-rate'] != undefined">
<span ng-if="relay.stats.options['global-rate'] > 0">Global rate limit: {{ relay.stats.options['global-rate'] | bytes }}/s</span>
<span ng-if="relay.stats.options['global-rate'] == 0">Global rate limit: unlimited</span>
<p><b>{{ relay.uri.hostname }}</b> <span ng-if="relay.status.options['provided-by']">provided by <u>{{ relay.status.options['provided-by'] }}</u></span></p>
<div ng-if="relay.status">
<span ng-if="relay.status.startTime">Start time: {{ relay.status.startTime | date:"medium" }}</br></span>
<span ng-if="relay.status.bytesProxied != undefined">Proxied: {{ relay.status.bytesProxied | bytes }}</br></span>
<span ng-if="relay.status.numActiveSessions != undefined">Sessions: {{ relay.status.numActiveSessions }}</br></span>
<span ng-if="relay.status.numConnections != undefined">Clients: {{ relay.status.numConnections }}</br></span>
<span ng-if="relay.status.options.pools">Pools: {{ relay.status.options.pools.join(', ') }}</br></span>
<span ng-if="relay.status.options['global-rate'] != undefined">
<span ng-if="relay.status.options['global-rate'] > 0">Global rate limit: {{ relay.status.options['global-rate'] | bytes }}/s</span>
<span ng-if="relay.status.options['global-rate'] == 0">Global rate limit: unlimited</span>
<br/>
</span>
<span ng-if="relay.stats.options['per-session-rate'] != undefined">
<span ng-if="relay.stats.options['per-session-rate'] > 0">Session rate limit: {{ relay.stats.options['per-session-rate'] | bytes }}/s</span>
<span ng-if="relay.stats.options['per-session-rate'] == 0">Session rate limit: unlimited</span>
<span ng-if="relay.status.options['per-session-rate'] != undefined">
<span ng-if="relay.status.options['per-session-rate'] > 0">Session rate limit: {{ relay.status.options['per-session-rate'] | bytes }}/s</span>
<span ng-if="relay.status.options['per-session-rate'] == 0">Session rate limit: unlimited</span>
<br/>
</span>
</div>
<div ng-if="!relay.stats">
<div ng-if="!relay.status">
Data unavailable.
<div>
</div>

View File

@@ -18,16 +18,12 @@ import (
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/golang/groupcache/lru"
"github.com/oschwald/geoip2-golang"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto"
"github.com/syncthing/syncthing/lib/relay/client"
"github.com/syncthing/syncthing/lib/sync"
@@ -38,42 +34,12 @@ import (
type location struct {
Latitude float64 `json:"latitude"`
Longitude float64 `json:"longitude"`
City string `json:"city"`
Country string `json:"country"`
Continent string `json:"continent"`
}
type relay struct {
URL string `json:"url"`
Location location `json:"location"`
uri *url.URL
Stats *stats `json:"stats"`
StatsRetrieved time.Time `json:"statsRetrieved"`
}
type stats struct {
StartTime time.Time `json:"startTime"`
UptimeSeconds int `json:"uptimeSeconds"`
PendingSessionKeys int `json:"numPendingSessionKeys"`
ActiveSessions int `json:"numActiveSessions"`
Connections int `json:"numConnections"`
Proxies int `json:"numProxies"`
BytesProxied int `json:"bytesProxied"`
GoVersion string `json:"goVersion"`
GoOS string `json:"goOS"`
GoArch string `json:"goArch"`
GoMaxProcs int `json:"goMaxProcs"`
GoRoutines int `json:"goNumRoutine"`
Rates []int64 `json:"kbps10s1m5m15m30m60m"`
Options struct {
NetworkTimeout int `json:"network-timeout"`
PintInterval int `json:"ping-interval"`
MessageTimeout int `json:"message-timeout"`
SessionRate int `json:"per-session-rate"`
GlobalRate int `json:"global-rate"`
Pools []string `json:"pools"`
ProvidedBy string `json:"provided-by"`
} `json:"options"`
URL string `json:"url"`
Location location `json:"location"`
uri *url.URL
}
func (r relay) String() string {
@@ -81,9 +47,9 @@ func (r relay) String() string {
}
type request struct {
relay *relay
result chan result
queueTimer *prometheus.Timer
relay relay
uri *url.URL
result chan result
}
type result struct {
@@ -92,25 +58,23 @@ type result struct {
}
var (
testCert tls.Certificate
knownRelaysFile = filepath.Join(os.TempDir(), "strelaypoolsrv_known_relays")
listen = ":80"
dir string
evictionTime = time.Hour
debug bool
getLRUSize = 10 << 10
getLimitBurst = 10
getLimitAvg = 2
postLRUSize = 1 << 10
postLimitBurst = 2
postLimitAvg = 2
getLimit time.Duration
postLimit time.Duration
permRelaysFile string
ipHeader string
geoipPath string
proto string
statsRefresh = time.Minute / 2
testCert tls.Certificate
listen = ":80"
dir string
evictionTime = time.Hour
debug bool
getLRUSize = 10 << 10
getLimitBurst = 10
getLimitAvg = 1
postLRUSize = 1 << 10
postLimitBurst = 2
postLimitAvg = 1
getLimit time.Duration
postLimit time.Duration
permRelaysFile string
ipHeader string
geoipPath string
proto string
getMut = sync.NewRWMutex()
getLRUCache *lru.Cache
@@ -121,31 +85,26 @@ var (
requests = make(chan request, 10)
mut = sync.NewRWMutex()
knownRelays = make([]*relay, 0)
permanentRelays = make([]*relay, 0)
knownRelays = make([]relay, 0)
permanentRelays = make([]relay, 0)
evictionTimers = make(map[string]*time.Timer)
)
const (
httpStatusEnhanceYourCalm = 429
)
func main() {
flag.StringVar(&listen, "listen", listen, "Listen address")
flag.StringVar(&dir, "keys", dir, "Directory where http-cert.pem and http-key.pem is stored for TLS listening")
flag.BoolVar(&debug, "debug", debug, "Enable debug output")
flag.DurationVar(&evictionTime, "eviction", evictionTime, "After how long the relay is evicted")
flag.IntVar(&getLRUSize, "get-limit-cache", getLRUSize, "Get request limiter cache size")
flag.IntVar(&getLimitAvg, "get-limit-avg", getLimitAvg, "Allowed average get request rate, per 10 s")
flag.IntVar(&getLimitAvg, "get-limit-avg", 2, "Allowed average get request rate, per 10 s")
flag.IntVar(&getLimitBurst, "get-limit-burst", getLimitBurst, "Allowed burst get requests")
flag.IntVar(&postLRUSize, "post-limit-cache", postLRUSize, "Post request limiter cache size")
flag.IntVar(&postLimitAvg, "post-limit-avg", postLimitAvg, "Allowed average post request rate, per minute")
flag.IntVar(&postLimitAvg, "post-limit-avg", 2, "Allowed average post request rate, per minute")
flag.IntVar(&postLimitBurst, "post-limit-burst", postLimitBurst, "Allowed burst post requests")
flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays")
flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.")
flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
flag.StringVar(&proto, "protocol", "tcp", "Protocol used for listening. 'tcp' for IPv4 and IPv6, 'tcp4' for IPv4, 'tcp6' for IPv6")
flag.DurationVar(&statsRefresh, "stats-refresh", statsRefresh, "Interval at which to refresh relay stats")
flag.Parse()
@@ -159,31 +118,13 @@ func main() {
var err error
if permRelaysFile != "" {
permanentRelays = loadRelays(permRelaysFile)
loadPermanentRelays(permRelaysFile)
}
testCert = createTestCertificate()
go requestProcessor()
// Load relays from cache in the background.
// Load them in a serial fashion to make sure any genuine requests
// are not dropped.
go func() {
for _, relay := range loadRelays(knownRelaysFile) {
resultChan := make(chan result)
requests <- request{relay, resultChan, nil}
result := <-resultChan
if result.err != nil {
relayTestsTotal.WithLabelValues("failed").Inc()
} else {
relayTestsTotal.WithLabelValues("success").Inc()
}
}
// Run the the stats refresher once the relays are loaded.
statsRefresher(statsRefresh)
}()
if dir != "" {
if debug {
log.Println("Starting TLS listener on", listen)
@@ -228,7 +169,6 @@ func main() {
handler := http.NewServeMux()
handler.HandleFunc("/", handleAssets)
handler.HandleFunc("/endpoint", handleRequest)
handler.HandleFunc("/metrics", handleMetrics)
srv := http.Server{
Handler: handler,
@@ -241,18 +181,7 @@ func main() {
}
}
func handleMetrics(w http.ResponseWriter, r *http.Request) {
timer := prometheus.NewTimer(metricsRequestsSeconds)
// Acquire the mutex just to make sure we're not caught mid-way stats collection
mut.RLock()
promhttp.Handler().ServeHTTP(w, r)
mut.RUnlock()
timer.ObserveDuration()
}
func handleAssets(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "no-cache, must-revalidate")
assets := auto.Assets()
path := r.URL.Path[1:]
if path == "" {
@@ -265,28 +194,11 @@ func handleAssets(w http.ResponseWriter, r *http.Request) {
return
}
etag := fmt.Sprintf("%d", auto.Generated)
modified := time.Unix(auto.Generated, 0).UTC()
w.Header().Set("Last-Modified", modified.Format(http.TimeFormat))
w.Header().Set("Etag", etag)
mtype := mimeTypeForFile(path)
if len(mtype) != 0 {
w.Header().Set("Content-Type", mtype)
}
if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modified.Add(time.Second).After(t) {
w.WriteHeader(http.StatusNotModified)
return
}
if match := r.Header.Get("If-None-Match"); match != "" {
if strings.Contains(match, etag) {
w.WriteHeader(http.StatusNotModified)
return
}
}
if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
w.Header().Set("Content-Encoding", "gzip")
} else {
@@ -329,15 +241,6 @@ func mimeTypeForFile(file string) string {
}
func handleRequest(w http.ResponseWriter, r *http.Request) {
timer := prometheus.NewTimer(apiRequestsSeconds.WithLabelValues(r.Method))
lw := NewLoggingResponseWriter(w)
defer func() {
timer.ObserveDuration()
apiRequestsTotal.WithLabelValues(r.Method, strconv.Itoa(lw.statusCode)).Inc()
}()
if ipHeader != "" {
r.RemoteAddr = r.Header.Get(ipHeader)
}
@@ -345,13 +248,13 @@ func handleRequest(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "GET":
if limit(r.RemoteAddr, getLRUCache, getMut, getLimit, getLimitBurst) {
w.WriteHeader(httpStatusEnhanceYourCalm)
w.WriteHeader(429)
return
}
handleGetRequest(w, r)
case "POST":
if limit(r.RemoteAddr, postLRUCache, postMut, postLimit, postLimitBurst) {
w.WriteHeader(httpStatusEnhanceYourCalm)
w.WriteHeader(429)
return
}
handlePostRequest(w, r)
@@ -375,7 +278,7 @@ func handleGetRequest(w http.ResponseWriter, r *http.Request) {
relays[i], relays[j] = relays[j], relays[i]
}
json.NewEncoder(w).Encode(map[string][]*relay{
json.NewEncoder(w).Encode(map[string][]relay{
"relays": relays,
})
}
@@ -412,9 +315,13 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
}
// Get the IP address of the client
rhost := r.RemoteAddr
if host, _, err := net.SplitHostPort(rhost); err == nil {
rhost = host
rhost, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
if debug {
log.Println("Failed to split remote address", r.RemoteAddr)
}
http.Error(w, err.Error(), 500)
return
}
ip := net.ParseIP(host)
@@ -426,18 +333,18 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
if debug {
log.Println("IP address advertised does not match client IP address", r.RemoteAddr, uri)
}
http.Error(w, fmt.Sprintf("IP advertised %s does not match client IP %s", host, rhost), http.StatusUnauthorized)
http.Error(w, "IP address does not match client IP", http.StatusUnauthorized)
return
}
newRelay.uri = uri
newRelay.Location = getLocation(uri.Host)
for _, current := range permanentRelays {
if current.uri.Host == newRelay.uri.Host {
if debug {
log.Println("Asked to add a relay", newRelay, "which exists in permanent list")
}
http.Error(w, "Invalid request", http.StatusBadRequest)
http.Error(w, "Invalid request", 500)
return
}
}
@@ -445,105 +352,78 @@ func handlePostRequest(w http.ResponseWriter, r *http.Request) {
reschan := make(chan result)
select {
case requests <- request{&newRelay, reschan, prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("queue"))}:
case requests <- request{newRelay, uri, reschan}:
result := <-reschan
if result.err != nil {
relayTestsTotal.WithLabelValues("failed").Inc()
http.Error(w, result.err.Error(), http.StatusBadRequest)
http.Error(w, result.err.Error(), 500)
return
}
relayTestsTotal.WithLabelValues("success").Inc()
w.Header().Set("Content-Type", "application/json; charset=utf-8")
json.NewEncoder(w).Encode(map[string]time.Duration{
"evictionIn": result.eviction,
})
default:
relayTestsTotal.WithLabelValues("dropped").Inc()
if debug {
log.Println("Dropping request")
}
w.WriteHeader(httpStatusEnhanceYourCalm)
w.WriteHeader(429)
}
}
func requestProcessor() {
for request := range requests {
if request.queueTimer != nil {
request.queueTimer.ObserveDuration()
}
timer := prometheus.NewTimer(relayTestActionsSeconds.WithLabelValues("test"))
handleRelayTest(request)
timer.ObserveDuration()
}
}
func handleRelayTest(request request) {
if debug {
log.Println("Request for", request.relay)
}
if !client.TestRelay(request.relay.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3) {
if debug {
log.Println("Test for relay", request.relay, "failed")
log.Println("Request for", request.relay)
}
request.result <- result{fmt.Errorf("connection test failed"), 0}
return
}
stats := fetchStats(request.relay)
location := getLocation(request.relay.uri.Host)
mut.Lock()
if stats != nil {
updateMetrics(request.relay.uri.Host, *stats, location)
}
request.relay.Stats = stats
request.relay.StatsRetrieved = time.Now()
request.relay.Location = location
timer, ok := evictionTimers[request.relay.uri.Host]
if ok {
if debug {
log.Println("Stopping existing timer for", request.relay)
}
timer.Stop()
}
for i, current := range knownRelays {
if current.uri.Host == request.relay.uri.Host {
if !client.TestRelay(request.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3) {
if debug {
log.Println("Relay", request.relay, "already exists")
log.Println("Test for relay", request.relay, "failed")
}
// Evict the old entry anyway, as configuration might have changed.
last := len(knownRelays) - 1
knownRelays[i] = knownRelays[last]
knownRelays = knownRelays[:last]
goto found
request.result <- result{fmt.Errorf("test failed"), 0}
continue
}
mut.Lock()
timer, ok := evictionTimers[request.relay.uri.Host]
if ok {
if debug {
log.Println("Stopping existing timer for", request.relay)
}
timer.Stop()
}
for i, current := range knownRelays {
if current.uri.Host == request.relay.uri.Host {
if debug {
log.Println("Relay", request.relay, "already exists")
}
// Evict the old entry anyway, as configuration might have changed.
last := len(knownRelays) - 1
knownRelays[i] = knownRelays[last]
knownRelays = knownRelays[:last]
goto found
}
}
if debug {
log.Println("Adding new relay", request.relay)
}
found:
knownRelays = append(knownRelays, request.relay)
evictionTimers[request.relay.uri.Host] = time.AfterFunc(evictionTime, evict(request.relay))
mut.Unlock()
request.result <- result{nil, evictionTime}
}
if debug {
log.Println("Adding new relay", request.relay)
}
found:
knownRelays = append(knownRelays, request.relay)
evictionTimers[request.relay.uri.Host] = time.AfterFunc(evictionTime, evict(request.relay))
mut.Unlock()
if err := saveRelays(knownRelaysFile, knownRelays); err != nil {
log.Println("Failed to write known relays: " + err.Error())
}
request.result <- result{nil, evictionTime}
}
func evict(relay *relay) func() {
func evict(relay relay) func() {
return func() {
mut.Lock()
defer mut.Unlock()
@@ -558,7 +438,6 @@ func evict(relay *relay) func() {
last := len(knownRelays) - 1
knownRelays[i] = knownRelays[last]
knownRelays = knownRelays[:last]
deleteMetrics(current.uri.Host)
}
}
delete(evictionTimers, relay.uri.Host)
@@ -566,12 +445,13 @@ func evict(relay *relay) func() {
}
func limit(addr string, cache *lru.Cache, lock sync.RWMutex, intv time.Duration, burst int) bool {
if host, _, err := net.SplitHostPort(addr); err == nil {
addr = host
host, _, err := net.SplitHostPort(addr)
if err != nil {
return false
}
lock.RLock()
bkt, ok := cache.Get(addr)
bkt, ok := cache.Get(host)
lock.RUnlock()
if ok {
bkt := bkt.(*rate.Limiter)
@@ -581,20 +461,18 @@ func limit(addr string, cache *lru.Cache, lock sync.RWMutex, intv time.Duration,
}
} else {
lock.Lock()
cache.Add(addr, rate.NewLimiter(rate.Every(intv), burst))
cache.Add(host, rate.NewLimiter(rate.Every(intv), burst))
lock.Unlock()
}
return false
}
func loadRelays(file string) []*relay {
func loadPermanentRelays(file string) {
content, err := ioutil.ReadFile(file)
if err != nil {
log.Println("Failed to load relays: " + err.Error())
return nil
log.Fatal(err)
}
var relays []*relay
for _, line := range strings.Split(string(content), "\n") {
if len(line) == 0 {
continue
@@ -603,30 +481,21 @@ func loadRelays(file string) []*relay {
uri, err := url.Parse(line)
if err != nil {
if debug {
log.Println("Skipping relay", line, "due to parse error", err)
log.Println("Skipping permanent relay", line, "due to parse error", err)
}
continue
}
relays = append(relays, &relay{
permanentRelays = append(permanentRelays, relay{
URL: line,
Location: getLocation(uri.Host),
uri: uri,
})
if debug {
log.Println("Adding relay", line)
log.Println("Adding permanent relay", line)
}
}
return relays
}
func saveRelays(file string, relays []*relay) error {
var content string
for _, relay := range relays {
content += relay.uri.String() + "\n"
}
return ioutil.WriteFile(file, []byte(content), 0777)
}
func createTestCertificate() tls.Certificate {
@@ -636,7 +505,7 @@ func createTestCertificate() tls.Certificate {
}
certFile, keyFile := filepath.Join(tmpDir, "cert.pem"), filepath.Join(tmpDir, "key.pem")
cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv")
cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 3072)
if err != nil {
log.Fatalln("Failed to create test X509 key pair:", err)
}
@@ -645,8 +514,6 @@ func createTestCertificate() tls.Certificate {
}
func getLocation(host string) location {
timer := prometheus.NewTimer(locationLookupSeconds)
defer timer.ObserveDuration()
db, err := geoip2.Open(geoipPath)
if err != nil {
return location{}
@@ -664,24 +531,7 @@ func getLocation(host string) location {
}
return location{
Longitude: city.Location.Longitude,
Latitude: city.Location.Latitude,
City: city.City.Names["en"],
Country: city.Country.IsoCode,
Continent: city.Continent.Code,
Longitude: city.Location.Longitude,
}
}
type loggingResponseWriter struct {
http.ResponseWriter
statusCode int
}
func NewLoggingResponseWriter(w http.ResponseWriter) *loggingResponseWriter {
return &loggingResponseWriter{w, http.StatusOK}
}
func (lrw *loggingResponseWriter) WriteHeader(code int) {
lrw.statusCode = code
lrw.ResponseWriter.WriteHeader(code)
}

View File

@@ -1,259 +0,0 @@
// Copyright (C) 2018 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
package main
import (
"encoding/json"
"net"
"net/http"
"os"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/syncthing/syncthing/lib/sync"
)
func init() {
processCollectorOpts := prometheus.ProcessCollectorOpts{
Namespace: "syncthing_relaypoolsrv",
PidFn: func() (int, error) {
return os.Getpid(), nil
},
}
prometheus.MustRegister(
prometheus.NewProcessCollector(processCollectorOpts),
)
}
var (
statusClient = http.Client{
Timeout: 5 * time.Second,
}
apiRequestsTotal = makeCounter("api_requests_total", "Number of API requests.", "type", "result")
apiRequestsSeconds = makeSummary("api_requests_seconds", "Latency of API requests.", "type")
relayTestsTotal = makeCounter("tests_total", "Number of relay tests.", "result")
relayTestActionsSeconds = makeSummary("test_actions_seconds", "Latency of relay test actions.", "type")
locationLookupSeconds = makeSummary("location_lookup_seconds", "Latency of location lookups.").WithLabelValues()
metricsRequestsSeconds = makeSummary("metrics_requests_seconds", "Latency of metric requests.").WithLabelValues()
scrapeSeconds = makeSummary("relay_scrape_seconds", "Latency of metric scrapes from remote relays.", "result")
relayUptime = makeGauge("relay_uptime", "Uptime of relay", "relay")
relayPendingSessionKeys = makeGauge("relay_pending_session_keys", "Number of pending session keys (two keys per session, one per each side of the connection)", "relay")
relayActiveSessions = makeGauge("relay_active_sessions", "Number of sessions that are happening, a session contains two parties", "relay")
relayConnections = makeGauge("relay_connections", "Number of devices connected to the relay", "relay")
relayProxies = makeGauge("relay_proxies", "Number of active proxy routines sending data between peers (two proxies per session, one for each way)", "relay")
relayBytesProxied = makeGauge("relay_bytes_proxied", "Number of bytes proxied by the relay", "relay")
relayGoRoutines = makeGauge("relay_go_routines", "Number of Go routines in the process", "relay")
relaySessionRate = makeGauge("relay_session_rate", "Rate applied per session", "relay")
relayGlobalRate = makeGauge("relay_global_rate", "Global rate applied on the whole relay", "relay")
relayBuildInfo = makeGauge("relay_build_info", "Build information about a relay", "relay", "go_version", "go_os", "go_arch")
relayLocationInfo = makeGauge("relay_location_info", "Location information about a relay", "relay", "city", "country", "continent")
lastStats = make(map[string]stats)
)
func makeGauge(name string, help string, labels ...string) *prometheus.GaugeVec {
gauge := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: "syncthing",
Subsystem: "relaypoolsrv",
Name: name,
Help: help,
},
labels,
)
prometheus.MustRegister(gauge)
return gauge
}
func makeSummary(name string, help string, labels ...string) *prometheus.SummaryVec {
summary := prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: "syncthing",
Subsystem: "relaypoolsrv",
Name: name,
Help: help,
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
},
labels,
)
prometheus.MustRegister(summary)
return summary
}
func makeCounter(name string, help string, labels ...string) *prometheus.CounterVec {
counter := prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "syncthing",
Subsystem: "relaypoolsrv",
Name: name,
Help: help,
},
labels,
)
prometheus.MustRegister(counter)
return counter
}
func statsRefresher(interval time.Duration) {
ticker := time.NewTicker(interval)
for range ticker.C {
refreshStats()
}
}
type statsFetchResult struct {
relay *relay
stats *stats
}
func refreshStats() {
mut.RLock()
relays := append(permanentRelays, knownRelays...)
mut.RUnlock()
now := time.Now()
wg := sync.NewWaitGroup()
results := make(chan statsFetchResult, len(relays))
for _, rel := range relays {
wg.Add(1)
go func(rel *relay) {
t0 := time.Now()
stats := fetchStats(rel)
duration := time.Since(t0).Seconds()
result := "success"
if stats == nil {
result = "failed"
}
scrapeSeconds.WithLabelValues(result).Observe(duration)
results <- statsFetchResult{
relay: rel,
stats: fetchStats(rel),
}
wg.Done()
}(rel)
}
wg.Wait()
close(results)
mut.Lock()
relayBuildInfo.Reset()
relayLocationInfo.Reset()
for result := range results {
result.relay.StatsRetrieved = now
result.relay.Stats = result.stats
if result.stats == nil {
deleteMetrics(result.relay.uri.Host)
} else {
updateMetrics(result.relay.uri.Host, *result.stats, result.relay.Location)
}
}
mut.Unlock()
}
func fetchStats(relay *relay) *stats {
statusAddr := relay.uri.Query().Get("statusAddr")
if statusAddr == "" {
statusAddr = ":22070"
}
statusHost, statusPort, err := net.SplitHostPort(statusAddr)
if err != nil {
return nil
}
if statusHost == "" {
if host, _, err := net.SplitHostPort(relay.uri.Host); err != nil {
return nil
} else {
statusHost = host
}
}
url := "http://" + net.JoinHostPort(statusHost, statusPort) + "/status"
response, err := statusClient.Get(url)
if err != nil {
return nil
}
var stats stats
if json.NewDecoder(response.Body).Decode(&stats); err != nil {
return nil
}
return &stats
}
func updateMetrics(host string, stats stats, location location) {
if stats.GoVersion != "" || stats.GoOS != "" || stats.GoArch != "" {
relayBuildInfo.WithLabelValues(host, stats.GoVersion, stats.GoOS, stats.GoArch).Add(1)
}
if location.City != "" || location.Country != "" || location.Continent != "" {
relayLocationInfo.WithLabelValues(host, location.City, location.Country, location.Continent).Add(1)
}
if lastStat, ok := lastStats[host]; ok {
stats = mergeStats(stats, lastStat)
}
relayUptime.WithLabelValues(host).Set(float64(stats.UptimeSeconds))
relayPendingSessionKeys.WithLabelValues(host).Set(float64(stats.PendingSessionKeys))
relayActiveSessions.WithLabelValues(host).Set(float64(stats.ActiveSessions))
relayConnections.WithLabelValues(host).Set(float64(stats.Connections))
relayProxies.WithLabelValues(host).Set(float64(stats.Proxies))
relayBytesProxied.WithLabelValues(host).Set(float64(stats.BytesProxied))
relayGoRoutines.WithLabelValues(host).Set(float64(stats.GoRoutines))
relaySessionRate.WithLabelValues(host).Set(float64(stats.Options.SessionRate))
relayGlobalRate.WithLabelValues(host).Set(float64(stats.Options.GlobalRate))
lastStats[host] = stats
}
func deleteMetrics(host string) {
relayUptime.DeleteLabelValues(host)
relayPendingSessionKeys.DeleteLabelValues(host)
relayActiveSessions.DeleteLabelValues(host)
relayConnections.DeleteLabelValues(host)
relayProxies.DeleteLabelValues(host)
relayBytesProxied.DeleteLabelValues(host)
relayGoRoutines.DeleteLabelValues(host)
relaySessionRate.DeleteLabelValues(host)
relayGlobalRate.DeleteLabelValues(host)
delete(lastStats, host)
}
// Due to some unexplainable behaviour, some of the numbers sometimes travel slightly backwards (by less than 1%)
// This happens between scrapes, which is 30s, so this can't be a race.
// This causes prometheus to assume a "rate reset", hence causes phenomenal spikes.
// One of the number that moves backwards is BytesProxied, which atomically increments a counter with numeric value
// returned by net.Conn.Read(). I don't think that can return a negative value, so I have no idea what's going on.
func mergeStats(new stats, old stats) stats {
new.UptimeSeconds = mergeValue(new.UptimeSeconds, old.UptimeSeconds)
new.PendingSessionKeys = mergeValue(new.PendingSessionKeys, old.PendingSessionKeys)
new.ActiveSessions = mergeValue(new.ActiveSessions, old.ActiveSessions)
new.Connections = mergeValue(new.Connections, old.Connections)
new.Proxies = mergeValue(new.Proxies, old.Proxies)
new.BytesProxied = mergeValue(new.BytesProxied, old.BytesProxied)
new.GoRoutines = mergeValue(new.GoRoutines, old.GoRoutines)
new.Options.SessionRate = mergeValue(new.Options.SessionRate, old.Options.SessionRate)
new.Options.GlobalRate = mergeValue(new.Options.GlobalRate, old.Options.GlobalRate)
return new
}
func mergeValue(new, old int) int {
if new >= old {
return new // normal increase
}
if float64(new) > 0.99*float64(old) {
return old // slight backward movement
}
return new // reset (relay restart)
}

View File

@@ -1,21 +0,0 @@
// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
package main
import (
"testing"
)
func TestMerge(t *testing.T) {
if mergeValue(1001, 1000) != 1001 {
t.Error("the computer says no")
}
if mergeValue(999, 1000) != 1000 {
t.Error("the computer says no")
}
if mergeValue(1, 1000) != 1 {
t.Error("the computer says no")
}
}

View File

@@ -104,9 +104,7 @@ func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) {
go messageReader(conn, messages, errors)
pingTicker := time.NewTicker(pingInterval)
defer pingTicker.Stop()
timeoutTicker := time.NewTimer(networkTimeout)
defer timeoutTicker.Stop()
joined := false
for {

View File

@@ -64,13 +64,12 @@ var (
limitCheckTimer *time.Timer
sessionLimitBps int
globalLimitBps int
overLimit int32
descriptorLimit int64
sessionLimiter *rate.Limiter
globalLimiter *rate.Limiter
networkBufferSize int
sessionLimitBps int
globalLimitBps int
overLimit int32
descriptorLimit int64
sessionLimiter *rate.Limiter
globalLimiter *rate.Limiter
statusAddr string
poolAddrs string
@@ -82,16 +81,8 @@ var (
natLease int
natRenewal int
natTimeout int
pprofEnabled bool
)
// httpClient is the HTTP client we use for outbound requests. It has a
// timeout and may get further options set during initialization.
var httpClient = &http.Client{
Timeout: 30 * time.Second,
}
func main() {
log.SetFlags(log.Lshortfile | log.LstdFlags)
@@ -114,8 +105,6 @@ func main() {
flag.IntVar(&natLease, "nat-lease", 60, "NAT lease length in minutes")
flag.IntVar(&natRenewal, "nat-renewal", 30, "NAT renewal frequency in minutes")
flag.IntVar(&natTimeout, "nat-timeout", 10, "NAT discovery timeout in seconds")
flag.BoolVar(&pprofEnabled, "pprof", false, "Enable the built in profiling on the status server")
flag.IntVar(&networkBufferSize, "network-buffer", 2048, "Network buffer size (two of these per proxied connection)")
flag.Parse()
if extAddress == "" {
@@ -135,14 +124,14 @@ func main() {
if err != nil {
log.Fatal(err)
}
if laddr.IP != nil && !laddr.IP.IsUnspecified() {
// We bind to a specific address. Our outgoing HTTP requests should
// also come from that address.
laddr.Port = 0
boundDialer := &net.Dialer{LocalAddr: laddr}
httpClient.Transport = &http.Transport{
DialContext: boundDialer.DialContext,
transport, ok := http.DefaultTransport.(*http.Transport)
if ok {
transport.Dial = (&net.Dialer{
Timeout: 30 * time.Second,
LocalAddr: laddr,
}).Dial
}
}
@@ -166,7 +155,7 @@ func main() {
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
log.Println("Failed to load keypair. Generating one, this might take a while...")
cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv")
cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv", 3072)
if err != nil {
log.Fatalln("Failed to generate X509 key pair:", err)
}

View File

@@ -7,6 +7,7 @@ import (
"encoding/json"
"io/ioutil"
"log"
"net/http"
"net/url"
"time"
)
@@ -26,7 +27,7 @@ func poolHandler(pool string, uri *url.URL, mapping mapping) {
uriCopy.String(),
})
resp, err := httpClient.Post(pool, "application/json", &b)
resp, err := http.Post(pool, "application/json", &b)
if err != nil {
log.Println("Error joining pool", pool, err)
} else if resp.StatusCode == 500 {

View File

@@ -22,7 +22,7 @@ import (
var (
sessionMut = sync.RWMutex{}
activeSessions = make([]*session, 0)
pendingSessions = make(map[string]*session)
pendingSessions = make(map[string]*session, 0)
numProxies int64
bytesProxied int64
)
@@ -189,7 +189,7 @@ done:
// We can end up here in 3 cases:
// 1. Timeout joining, in which case there are potentially entries in pendingSessions
// 2. General session end/timeout, in which case there are entries in activeSessions
// 3. Protocol handler calls dropSession as one of its clients disconnects.
// 3. Protocol handler calls dropSession as one of it's clients disconnects.
sessionMut.Lock()
delete(pendingSessions, string(s.serverkey))
@@ -254,7 +254,7 @@ func (s *session) proxy(c1, c2 net.Conn) error {
atomic.AddInt64(&numProxies, 1)
defer atomic.AddInt64(&numProxies, -1)
buf := make([]byte, networkBufferSize)
buf := make([]byte, 65536)
for {
c1.SetReadDeadline(time.Now().Add(networkTimeout))
n, err := c1.Read(buf)

View File

@@ -6,7 +6,6 @@ import (
"encoding/json"
"log"
"net/http"
"net/http/pprof"
"runtime"
"sync/atomic"
"time"
@@ -19,9 +18,6 @@ func statusService(addr string) {
handler := http.NewServeMux()
handler.HandleFunc("/status", getStatus)
if pprofEnabled {
handler.HandleFunc("/debug/pprof/", pprof.Index)
}
srv := http.Server{
Addr: addr,
@@ -40,10 +36,6 @@ func getStatus(w http.ResponseWriter, r *http.Request) {
sessionMut.Lock()
// This can potentially be double the number of pending sessions, as each session has two keys, one for each side.
status["version"] = Version
status["buildHost"] = BuildHost
status["buildUser"] = BuildUser
status["buildDate"] = BuildDate
status["startTime"] = rc.startTime
status["uptimeSeconds"] = time.Since(rc.startTime) / time.Second
status["numPendingSessionKeys"] = len(pendingSessions)

View File

@@ -24,7 +24,7 @@ func main() {
flag.Parse()
if flag.NArg() < 1 {
log.Print(`Usage:
log.Println(`Usage:
stsigtool <command>
Where command is one of:
@@ -40,7 +40,6 @@ Where command is one of:
verify <signaturefile> <datafile> <pubkeyfile>
- verify a signature, using the specified public key file
`)
}

View File

@@ -37,7 +37,7 @@ func TestAuditService(t *testing.T) {
// This event should not be logged, since we have stopped.
events.Default.Log(events.ConfigSaved, "the third event")
result := buf.String()
result := string(buf.Bytes())
t.Log(result)
if strings.Contains(result, "first event") {

View File

@@ -6,8 +6,8 @@
//+build noupgrade
package build
package main
func init() {
Tags = append(Tags, "noupgrade")
BuildTags = append(BuildTags, "noupgrade")
}

View File

@@ -6,8 +6,8 @@
//+build race
package build
package main
func init() {
Tags = append(Tags, "race")
BuildTags = append(BuildTags, "race")
}

View File

File diff suppressed because it is too large Load Diff

View File

@@ -8,9 +8,7 @@ package main
import (
"bytes"
"crypto/tls"
"encoding/base64"
"fmt"
"net/http"
"strings"
"time"
@@ -20,7 +18,6 @@ import (
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sync"
"golang.org/x/crypto/bcrypt"
ldap "gopkg.in/ldap.v2"
)
var (
@@ -35,9 +32,9 @@ func emitLoginAttempt(success bool, username string) {
})
}
func basicAuthAndSessionMiddleware(cookieName string, guiCfg config.GUIConfiguration, ldapCfg config.LDAPConfiguration, next http.Handler) http.Handler {
func basicAuthAndSessionMiddleware(cookieName string, cfg config.GUIConfiguration, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if guiCfg.IsValidAPIKey(r.Header.Get("X-API-Key")) {
if cfg.IsValidAPIKey(r.Header.Get("X-API-Key")) {
next.ServeHTTP(w, r)
return
}
@@ -80,25 +77,42 @@ func basicAuthAndSessionMiddleware(cookieName string, guiCfg config.GUIConfigura
return
}
// Check if the username is correct, assuming it was sent as UTF-8
username := string(fields[0])
password := string(fields[1])
authOk := auth(username, password, guiCfg, ldapCfg)
if !authOk {
usernameIso := string(iso88591ToUTF8([]byte(username)))
passwordIso := string(iso88591ToUTF8([]byte(password)))
authOk = auth(usernameIso, passwordIso, guiCfg, ldapCfg)
if authOk {
username = usernameIso
}
if username == cfg.User {
goto usernameOK
}
if !authOk {
emitLoginAttempt(false, username)
error()
return
// ... check it again, converting it from assumed ISO-8859-1 to UTF-8
username = string(iso88591ToUTF8(fields[0]))
if username == cfg.User {
goto usernameOK
}
// Neither of the possible interpretations match the configured username
emitLoginAttempt(false, username)
error()
return
usernameOK:
// Check password as given (assumes UTF-8 encoding)
password := fields[1]
if err := bcrypt.CompareHashAndPassword([]byte(cfg.Password), password); err == nil {
goto passwordOK
}
// ... check it again, converting it from assumed ISO-8859-1 to UTF-8
password = iso88591ToUTF8(password)
if err := bcrypt.CompareHashAndPassword([]byte(cfg.Password), password); err == nil {
goto passwordOK
}
// Neither of the attempts to verify the password checked out
emitLoginAttempt(false, username)
error()
return
passwordOK:
sessionid := rand.String(32)
sessionsMut.Lock()
sessions[sessionid] = true
@@ -114,54 +128,6 @@ func basicAuthAndSessionMiddleware(cookieName string, guiCfg config.GUIConfigura
})
}
func auth(username string, password string, guiCfg config.GUIConfiguration, ldapCfg config.LDAPConfiguration) bool {
if guiCfg.AuthMode == config.AuthModeLDAP {
return authLDAP(username, password, ldapCfg)
} else {
return authStatic(username, password, guiCfg.User, guiCfg.Password)
}
}
func authStatic(username string, password string, configUser string, configPassword string) bool {
configPasswordBytes := []byte(configPassword)
passwordBytes := []byte(password)
return bcrypt.CompareHashAndPassword(configPasswordBytes, passwordBytes) == nil && username == configUser
}
func authLDAP(username string, password string, cfg config.LDAPConfiguration) bool {
address := cfg.Address
var connection *ldap.Conn
var err error
if cfg.Transport == config.LDAPTransportTLS {
connection, err = ldap.DialTLS("tcp", address, &tls.Config{InsecureSkipVerify: cfg.InsecureSkipVerify})
} else {
connection, err = ldap.Dial("tcp", address)
}
if err != nil {
l.Warnln("LDAP Dial:", err)
return false
}
if cfg.Transport == config.LDAPTransportStartTLS {
err = connection.StartTLS(&tls.Config{InsecureSkipVerify: cfg.InsecureSkipVerify})
if err != nil {
l.Warnln("LDAP Start TLS:", err)
return false
}
}
defer connection.Close()
err = connection.Bind(fmt.Sprintf(cfg.BindDN, username), password)
if err != nil {
l.Warnln("LDAP Bind:", err)
return false
}
return true
}
// Convert an ISO-8859-1 encoded byte string to UTF-8. Works by the
// principle that ISO-8859-1 bytes are equivalent to unicode code points,
// that a rune slice is a list of code points, and that stringifying a slice

View File

@@ -1,40 +0,0 @@
// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"testing"
"golang.org/x/crypto/bcrypt"
)
var passwordHashBytes []byte
func init() {
passwordHashBytes, _ = bcrypt.GenerateFromPassword([]byte("pass"), 0)
}
func TestStaticAuthOK(t *testing.T) {
ok := authStatic("user", "pass", "user", string(passwordHashBytes))
if !ok {
t.Fatalf("should pass auth")
}
}
func TestSimpleAuthUsernameFail(t *testing.T) {
ok := authStatic("userWRONG", "pass", "user", string(passwordHashBytes))
if ok {
t.Fatalf("should fail auth")
}
}
func TestStaticAuthPasswordFail(t *testing.T) {
ok := authStatic("user", "passWRONG", "user", string(passwordHashBytes))
if ok {
t.Fatalf("should fail auth")
}
}

View File

@@ -14,7 +14,6 @@ import (
"strings"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/locations"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/rand"
"github.com/syncthing/syncthing/lib/sync"
@@ -116,7 +115,7 @@ func saveCsrfTokens() {
// We're ignoring errors in here. It's not super critical and there's
// nothing relevant we can do about them anyway...
name := locations.Get(locations.CsrfTokens)
name := locations[locCsrfTokens]
f, err := osutil.CreateAtomic(name)
if err != nil {
return
@@ -130,7 +129,7 @@ func saveCsrfTokens() {
}
func loadCsrfTokens() {
f, err := os.Open(locations.Get(locations.CsrfTokens))
f, err := os.Open(locations[locCsrfTokens])
if err != nil {
return
}

View File

@@ -16,7 +16,6 @@ import (
"os"
"path/filepath"
"strings"
"time"
"github.com/syncthing/syncthing/lib/auto"
"github.com/syncthing/syncthing/lib/config"
@@ -72,8 +71,6 @@ func (s *staticsServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
func (s *staticsServer) serveAsset(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Cache-Control", "no-cache, must-revalidate")
file := r.URL.Path
if file[0] == '/' {
@@ -92,10 +89,6 @@ func (s *staticsServer) serveAsset(w http.ResponseWriter, r *http.Request) {
if s.assetDir != "" {
p := filepath.Join(s.assetDir, theme, filepath.FromSlash(file))
if _, err := os.Stat(p); err == nil {
mtype := s.mimeTypeForFile(file)
if len(mtype) != 0 {
w.Header().Set("Content-Type", mtype)
}
http.ServeFile(w, r, p)
return
}
@@ -108,10 +101,6 @@ func (s *staticsServer) serveAsset(w http.ResponseWriter, r *http.Request) {
if s.assetDir != "" {
p := filepath.Join(s.assetDir, config.DefaultTheme, filepath.FromSlash(file))
if _, err := os.Stat(p); err == nil {
mtype := s.mimeTypeForFile(file)
if len(mtype) != 0 {
w.Header().Set("Content-Type", mtype)
}
http.ServeFile(w, r, p)
return
}
@@ -125,26 +114,6 @@ func (s *staticsServer) serveAsset(w http.ResponseWriter, r *http.Request) {
}
}
etag := fmt.Sprintf("%d", auto.Generated)
modified := time.Unix(auto.Generated, 0).UTC()
w.Header().Set("Last-Modified", modified.Format(http.TimeFormat))
w.Header().Set("Etag", etag)
if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil {
if modified.Equal(t) || modified.Before(t) {
w.WriteHeader(http.StatusNotModified)
return
}
}
if match := r.Header.Get("If-None-Match"); match != "" {
if strings.Contains(match, etag) {
w.WriteHeader(http.StatusNotModified)
return
}
}
mtype := s.mimeTypeForFile(file)
if len(mtype) != 0 {
w.Header().Set("Content-Type", mtype)
@@ -172,17 +141,17 @@ func (s *staticsServer) serveThemes(w http.ResponseWriter, r *http.Request) {
func (s *staticsServer) mimeTypeForFile(file string) string {
// We use a built in table of the common types since the system
// TypeByExtension might be unreliable. But if we don't know, we delegate
// to the system. All our files are UTF-8.
// to the system.
ext := filepath.Ext(file)
switch ext {
case ".htm", ".html":
return "text/html; charset=utf-8"
return "text/html"
case ".css":
return "text/css; charset=utf-8"
return "text/css"
case ".js":
return "application/javascript; charset=utf-8"
return "application/javascript"
case ".json":
return "application/json; charset=utf-8"
return "application/json"
case ".png":
return "image/png"
case ".ttf":
@@ -190,7 +159,7 @@ func (s *staticsServer) mimeTypeForFile(file string) string {
case ".woff":
return "application/x-font-woff"
case ".svg":
return "image/svg+xml; charset=utf-8"
return "image/svg+xml"
default:
return mime.TypeByExtension(ext)
}

View File

@@ -16,8 +16,6 @@ import (
"net"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
@@ -26,7 +24,6 @@ import (
"github.com/d4l3k/messagediff"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/fs"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sync"
"github.com/thejerf/suture"
@@ -77,9 +74,7 @@ func TestStopAfterBrokenConfig(t *testing.T) {
srv := newAPIService(protocol.LocalDeviceID, w, "../../test/h1/https-cert.pem", "../../test/h1/https-key.pem", "", nil, nil, nil, nil, nil, nil, nil, nil)
srv.started = make(chan string)
sup := suture.New("test", suture.Spec{
PassThroughPanics: true,
})
sup := suture.NewSimple("test")
sup.Add(srv)
sup.ServeBackground()
@@ -489,9 +484,7 @@ func startHTTP(cfg *mockedConfig) (string, error) {
svc.started = addrChan
// Actually start the API service
supervisor := suture.New("API test", suture.Spec{
PassThroughPanics: true,
})
supervisor := suture.NewSimple("API test")
supervisor.Add(svc)
supervisor.ServeBackground()
@@ -846,24 +839,16 @@ func TestAddressIsLocalhost(t *testing.T) {
// These are all valid localhost addresses
{"localhost", true},
{"LOCALHOST", true},
{"localhost.", true},
{"::1", true},
{"127.0.0.1", true},
{"127.23.45.56", true},
{"localhost:8080", true},
{"LOCALHOST:8000", true},
{"localhost.:8080", true},
{"[::1]:8080", true},
{"127.0.0.1:8080", true},
{"127.23.45.56:8080", true},
// These are all non-localhost addresses
{"example.com", false},
{"example.com:8080", false},
{"localhost.com", false},
{"localhost.com:8080", false},
{"www.localhost", false},
{"www.localhost:8080", false},
{"192.0.2.10", false},
{"192.0.2.10:8080", false},
{"0.0.0.0", false},
@@ -972,87 +957,3 @@ func TestEventMasks(t *testing.T) {
t.Errorf("should have returned a valid, non-default event sub")
}
}
func TestBrowse(t *testing.T) {
pathSep := string(os.PathSeparator)
tmpDir, err := ioutil.TempDir("", "syncthing")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpDir)
if err := os.Mkdir(filepath.Join(tmpDir, "dir"), 0755); err != nil {
t.Fatal(err)
}
if err := ioutil.WriteFile(filepath.Join(tmpDir, "file"), []byte("hello"), 0644); err != nil {
t.Fatal(err)
}
if err := os.Mkdir(filepath.Join(tmpDir, "MiXEDCase"), 0755); err != nil {
t.Fatal(err)
}
// We expect completion to return the full path to the completed
// directory, with an ending slash.
dirPath := filepath.Join(tmpDir, "dir") + pathSep
mixedCaseDirPath := filepath.Join(tmpDir, "MiXEDCase") + pathSep
cases := []struct {
current string
returns []string
}{
// The direcotory without slash is completed to one with slash.
{tmpDir, []string{tmpDir + pathSep}},
// With slash it's completed to its contents.
// Dirs are given pathSeps.
// Files are not returned.
{tmpDir + pathSep, []string{mixedCaseDirPath, dirPath}},
// Globbing is automatic based on prefix.
{tmpDir + pathSep + "d", []string{dirPath}},
{tmpDir + pathSep + "di", []string{dirPath}},
{tmpDir + pathSep + "dir", []string{dirPath}},
{tmpDir + pathSep + "f", nil},
{tmpDir + pathSep + "q", nil},
// Globbing is case-insensitve
{tmpDir + pathSep + "mixed", []string{mixedCaseDirPath}},
}
for _, tc := range cases {
ret := browseFiles(tc.current, fs.FilesystemTypeBasic)
if !equalStrings(ret, tc.returns) {
t.Errorf("browseFiles(%q) => %q, expected %q", tc.current, ret, tc.returns)
}
}
}
func TestPrefixMatch(t *testing.T) {
cases := []struct {
s string
prefix string
expected int
}{
{"aaaA", "aaa", matchExact},
{"AAAX", "BBB", noMatch},
{"AAAX", "aAa", matchCaseIns},
{"äÜX", "äü", matchCaseIns},
}
for _, tc := range cases {
ret := checkPrefixMatch(tc.s, tc.prefix)
if ret != tc.expected {
t.Errorf("checkPrefixMatch(%q, %q) => %v, expected %v", tc.s, tc.prefix, ret, tc.expected)
}
}
}
func equalStrings(a, b []string) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}

View File

@@ -49,7 +49,7 @@ func saveHeapProfiles(rate int) {
panic(err)
}
os.Remove(name) // Error deliberately ignored
_ = os.Remove(name) // Error deliberately ignored
err = os.Rename(name+".tmp", name)
if err != nil {
panic(err)

125
cmd/syncthing/locations.go Normal file
View File

@@ -0,0 +1,125 @@
// Copyright (C) 2015 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"os"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/syncthing/syncthing/lib/osutil"
)
type locationEnum string
// Use strings as keys to make printout and serialization of the locations map
// more meaningful.
const (
locConfigFile locationEnum = "config"
locCertFile = "certFile"
locKeyFile = "keyFile"
locHTTPSCertFile = "httpsCertFile"
locHTTPSKeyFile = "httpsKeyFile"
locDatabase = "database"
locLogFile = "logFile"
locCsrfTokens = "csrfTokens"
locPanicLog = "panicLog"
locAuditLog = "auditLog"
locGUIAssets = "GUIAssets"
locDefFolder = "defFolder"
)
// Platform dependent directories
var baseDirs = map[string]string{
"config": defaultConfigDir(), // Overridden by -home flag
"home": homeDir(), // User's home directory, *not* -home flag
}
// Use the variables from baseDirs here
var locations = map[locationEnum]string{
locConfigFile: "${config}/config.xml",
locCertFile: "${config}/cert.pem",
locKeyFile: "${config}/key.pem",
locHTTPSCertFile: "${config}/https-cert.pem",
locHTTPSKeyFile: "${config}/https-key.pem",
locDatabase: "${config}/index-v0.14.0.db",
locLogFile: "${config}/syncthing.log", // -logfile on Windows
locCsrfTokens: "${config}/csrftokens.txt",
locPanicLog: "${config}/panic-${timestamp}.log",
locAuditLog: "${config}/audit-${timestamp}.log",
locGUIAssets: "${config}/gui",
locDefFolder: "${home}/Sync",
}
// expandLocations replaces the variables in the location map with actual
// directory locations.
func expandLocations() error {
for key, dir := range locations {
for varName, value := range baseDirs {
dir = strings.Replace(dir, "${"+varName+"}", value, -1)
}
var err error
dir, err = osutil.ExpandTilde(dir)
if err != nil {
return err
}
locations[key] = dir
}
return nil
}
// defaultConfigDir returns the default configuration directory, as figured
// out by various the environment variables present on each platform, or dies
// trying.
func defaultConfigDir() string {
switch runtime.GOOS {
case "windows":
if p := os.Getenv("LocalAppData"); p != "" {
return filepath.Join(p, "Syncthing")
}
return filepath.Join(os.Getenv("AppData"), "Syncthing")
case "darwin":
dir, err := osutil.ExpandTilde("~/Library/Application Support/Syncthing")
if err != nil {
l.Fatalln(err)
}
return dir
default:
if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
return filepath.Join(xdgCfg, "syncthing")
}
dir, err := osutil.ExpandTilde("~/.config/syncthing")
if err != nil {
l.Fatalln(err)
}
return dir
}
}
// homeDir returns the user's home directory, or dies trying.
func homeDir() string {
home, err := osutil.ExpandTilde("~")
if err != nil {
l.Fatalln(err)
}
return home
}
func timestampedLoc(key locationEnum) string {
// We take the roundtrip via "${timestamp}" instead of passing the path
// directly through time.Format() to avoid issues when the path we are
// expanding contains numbers; otherwise for example
// /home/user2006/.../panic-20060102-150405.log would get both instances of
// 2006 replaced by 2015...
tpl := locations[key]
now := time.Now().Format("20060102-150405")
return strings.Replace(tpl, "${timestamp}", now, -1)
}

View File

File diff suppressed because it is too large Load Diff

View File

@@ -36,3 +36,29 @@ func TestShortIDCheck(t *testing.T) {
t.Error("Should have gotten an error")
}
}
func TestAllowedVersions(t *testing.T) {
testcases := []struct {
ver string
allowed bool
}{
{"v0.13.0", true},
{"v0.12.11+22-gabcdef0", true},
{"v0.13.0-beta0", true},
{"v0.13.0-beta47", true},
{"v0.13.0-beta47+1-gabcdef0", true},
{"v0.13.0-beta.0", true},
{"v0.13.0-beta.47", true},
{"v0.13.0-beta.0+1-gabcdef0", true},
{"v0.13.0-beta.47+1-gabcdef0", true},
{"v0.13.0-some-weird-but-allowed-tag", true},
{"v0.13.0-allowed.to.do.this", true},
{"v0.13.0+not.allowed.to.do.this", false},
}
for i, c := range testcases {
if allowed := allowedVersionExp.MatchString(c.ver); allowed != c.allowed {
t.Errorf("%d: incorrect result %v != %v for %q", i, allowed, c.allowed, c.ver)
}
}
}

View File

@@ -24,10 +24,6 @@ func (c *mockedConfig) ListenAddresses() []string {
return nil
}
func (c *mockedConfig) LDAP() config.LDAPConfiguration {
return config.LDAPConfiguration{}
}
func (c *mockedConfig) RawCopy() config.Configuration {
cfg := config.Configuration{}
util.SetDefaults(&cfg.Options)
@@ -38,14 +34,12 @@ func (c *mockedConfig) Options() config.OptionsConfiguration {
return config.OptionsConfiguration{}
}
func (c *mockedConfig) Replace(cfg config.Configuration) (config.Waiter, error) {
return noopWaiter{}, nil
func (c *mockedConfig) Replace(cfg config.Configuration) error {
return nil
}
func (c *mockedConfig) Subscribe(cm config.Committer) {}
func (c *mockedConfig) Unsubscribe(cm config.Committer) {}
func (c *mockedConfig) Folders() map[string]config.FolderConfiguration {
return nil
}
@@ -54,12 +48,12 @@ func (c *mockedConfig) Devices() map[protocol.DeviceID]config.DeviceConfiguratio
return nil
}
func (c *mockedConfig) SetDevice(config.DeviceConfiguration) (config.Waiter, error) {
return noopWaiter{}, nil
func (c *mockedConfig) SetDevice(config.DeviceConfiguration) error {
return nil
}
func (c *mockedConfig) SetDevices([]config.DeviceConfiguration) (config.Waiter, error) {
return noopWaiter{}, nil
func (c *mockedConfig) SetDevices([]config.DeviceConfiguration) error {
return nil
}
func (c *mockedConfig) Save() error {
@@ -69,59 +63,3 @@ func (c *mockedConfig) Save() error {
func (c *mockedConfig) RequiresRestart() bool {
return false
}
func (c *mockedConfig) AddOrUpdatePendingDevice(device protocol.DeviceID, name, address string) {}
func (c *mockedConfig) AddOrUpdatePendingFolder(id, label string, device protocol.DeviceID) {}
func (m *mockedConfig) MyName() string {
return ""
}
func (m *mockedConfig) ConfigPath() string {
return ""
}
func (m *mockedConfig) SetGUI(gui config.GUIConfiguration) (config.Waiter, error) {
return noopWaiter{}, nil
}
func (m *mockedConfig) SetOptions(opts config.OptionsConfiguration) (config.Waiter, error) {
return noopWaiter{}, nil
}
func (m *mockedConfig) Folder(id string) (config.FolderConfiguration, bool) {
return config.FolderConfiguration{}, false
}
func (m *mockedConfig) FolderList() []config.FolderConfiguration {
return nil
}
func (m *mockedConfig) SetFolder(fld config.FolderConfiguration) (config.Waiter, error) {
return noopWaiter{}, nil
}
func (m *mockedConfig) Device(id protocol.DeviceID) (config.DeviceConfiguration, bool) {
return config.DeviceConfiguration{}, false
}
func (m *mockedConfig) RemoveDevice(id protocol.DeviceID) (config.Waiter, error) {
return noopWaiter{}, nil
}
func (m *mockedConfig) IgnoredDevice(id protocol.DeviceID) bool {
return false
}
func (m *mockedConfig) IgnoredFolder(device protocol.DeviceID, folder string) bool {
return false
}
func (m *mockedConfig) GlobalDiscoveryServers() []string {
return nil
}
type noopWaiter struct{}
func (noopWaiter) Wait() {}

View File

@@ -11,11 +11,3 @@ type mockedConnections struct{}
func (m *mockedConnections) Status() map[string]interface{} {
return nil
}
func (m *mockedConnections) NATType() string {
return ""
}
func (m *mockedConnections) Serve() {}
func (m *mockedConnections) Stop() {}

View File

@@ -44,7 +44,7 @@ func (m *mockedCachingMux) Cache() map[protocol.DeviceID]discover.CacheEntry {
// from events.CachingMux
func (m *mockedCachingMux) Add(finder discover.Finder, cacheTime, negCacheTime time.Duration) {
func (m *mockedCachingMux) Add(finder discover.Finder, cacheTime, negCacheTime time.Duration, priority int) {
}
func (m *mockedCachingMux) ChildErrors() map[string]error {

View File

@@ -7,16 +7,12 @@
package main
import (
"net"
"time"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/connections"
"github.com/syncthing/syncthing/lib/db"
"github.com/syncthing/syncthing/lib/model"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/stats"
"github.com/syncthing/syncthing/lib/versioner"
)
type mockedModel struct{}
@@ -31,14 +27,8 @@ func (m *mockedModel) Completion(device protocol.DeviceID, folder string) model.
func (m *mockedModel) Override(folder string) {}
func (m *mockedModel) Revert(folder string) {}
func (m *mockedModel) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated) {
return nil, nil, nil
}
func (m *mockedModel) RemoteNeedFolderFiles(device protocol.DeviceID, folder string, page, perpage int) ([]db.FileInfoTruncated, error) {
return nil, nil
func (m *mockedModel) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfoTruncated, []db.FileInfoTruncated, []db.FileInfoTruncated, int) {
return nil, nil, nil, 0
}
func (m *mockedModel) NeedSize(folder string) db.Counts {
@@ -68,7 +58,7 @@ func (m *mockedModel) CurrentGlobalFile(folder string, file string) (protocol.Fi
func (m *mockedModel) ResetFolder(folder string) {
}
func (m *mockedModel) Availability(folder string, file protocol.FileInfo, block protocol.BlockInfo) []model.Availability {
func (m *mockedModel) Availability(folder, file string, version protocol.Vector, block protocol.BlockInfo) []model.Availability {
return nil
}
@@ -80,14 +70,6 @@ func (m *mockedModel) SetIgnores(folder string, content []string) error {
return nil
}
func (m *mockedModel) GetFolderVersions(folder string) (map[string][]versioner.FileVersion, error) {
return nil, nil
}
func (m *mockedModel) RestoreFolderVersions(folder string, versions map[string]time.Time) (map[string]string, error) {
return nil, nil
}
func (m *mockedModel) PauseDevice(device protocol.DeviceID) {
}
@@ -109,8 +91,8 @@ func (m *mockedModel) ScanFolderSubdirs(folder string, subs []string) error {
func (m *mockedModel) BringToFront(folder, file string) {}
func (m *mockedModel) Connection(deviceID protocol.DeviceID) (connections.Connection, bool) {
return nil, false
func (m *mockedModel) ConnectedTo(deviceID protocol.DeviceID) bool {
return false
}
func (m *mockedModel) GlobalSize(folder string) db.Counts {
@@ -121,10 +103,6 @@ func (m *mockedModel) LocalSize(folder string) db.Counts {
return db.Counts{}
}
func (m *mockedModel) ReceiveOnlyChangedSize(folder string) db.Counts {
return db.Counts{}
}
func (m *mockedModel) CurrentSequence(folder string) (int64, bool) {
return 0, false
}
@@ -136,54 +114,3 @@ func (m *mockedModel) RemoteSequence(folder string) (int64, bool) {
func (m *mockedModel) State(folder string) (string, time.Time, error) {
return "", time.Time{}, nil
}
func (m *mockedModel) UsageReportingStats(version int, preview bool) map[string]interface{} {
return nil
}
func (m *mockedModel) FolderErrors(folder string) ([]model.FileError, error) {
return nil, nil
}
func (m *mockedModel) WatchError(folder string) error {
return nil
}
func (m *mockedModel) LocalChangedFiles(folder string, page, perpage int) []db.FileInfoTruncated {
return nil
}
func (m *mockedModel) Serve() {}
func (m *mockedModel) Stop() {}
func (m *mockedModel) Index(deviceID protocol.DeviceID, folder string, files []protocol.FileInfo) {}
func (m *mockedModel) IndexUpdate(deviceID protocol.DeviceID, folder string, files []protocol.FileInfo) {
}
func (m *mockedModel) Request(deviceID protocol.DeviceID, folder, name string, size int32, offset int64, hash []byte, weakHash uint32, fromTemporary bool) (protocol.RequestResponse, error) {
return nil, nil
}
func (m *mockedModel) ClusterConfig(deviceID protocol.DeviceID, config protocol.ClusterConfig) {}
func (m *mockedModel) Closed(conn protocol.Connection, err error) {}
func (m *mockedModel) DownloadProgress(deviceID protocol.DeviceID, folder string, updates []protocol.FileDownloadProgressUpdate) {
}
func (m *mockedModel) AddConnection(conn connections.Connection, hello protocol.HelloResult) {}
func (m *mockedModel) OnHello(protocol.DeviceID, net.Addr, protocol.HelloResult) error {
return nil
}
func (m *mockedModel) GetHello(protocol.DeviceID) protocol.HelloIntf {
return nil
}
func (m *mockedModel) AddFolder(cfg config.FolderConfiguration) {}
func (m *mockedModel) RestartFolder(from, to config.FolderConfiguration) {}
func (m *mockedModel) StartFolder(folder string) {}
func (m *mockedModel) StartDeadlockDetector(timeout time.Duration) {}

View File

@@ -17,7 +17,6 @@ import (
"syscall"
"time"
"github.com/syncthing/syncthing/lib/locations"
"github.com/syncthing/syncthing/lib/osutil"
"github.com/syncthing/syncthing/lib/sync"
)
@@ -85,18 +84,18 @@ func monitorMain(runtimeOptions RuntimeOptions) {
stderr, err := cmd.StderrPipe()
if err != nil {
panic(err)
l.Fatalln("stderr:", err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
panic(err)
l.Fatalln("stdout:", err)
}
l.Infoln("Starting syncthing")
err = cmd.Start()
if err != nil {
panic(err)
l.Fatalln(err)
}
stdoutMut.Lock()
@@ -128,7 +127,7 @@ func monitorMain(runtimeOptions RuntimeOptions) {
select {
case s := <-stopSign:
l.Infof("Signal %d received; exiting", s)
cmd.Process.Signal(sigTerm)
cmd.Process.Kill()
<-exit
return
@@ -199,7 +198,7 @@ func copyStderr(stderr io.Reader, dst io.Writer) {
}
if strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:") {
panicFd, err = os.Create(locations.GetTimestamped(locations.PanicLog))
panicFd, err = os.Create(timestampedLoc(locPanicLog))
if err != nil {
l.Warnln("Create panic log:", err)
continue
@@ -419,10 +418,10 @@ func (f *autoclosedFile) closerLoop() {
func childEnv() []string {
var env []string
for _, str := range os.Environ() {
if strings.HasPrefix(str, "STNORESTART=") {
if strings.HasPrefix("STNORESTART=", str) {
continue
}
if strings.HasPrefix(str, "STMONITORED=") {
if strings.HasPrefix("STMONITORED=", str) {
continue
}
env = append(env, str)

View File

@@ -1,88 +0,0 @@
// Copyright (C) 2017 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"io/ioutil"
"os"
"path/filepath"
"testing"
"time"
)
func TestAutoClosedFile(t *testing.T) {
os.RemoveAll("_autoclose")
defer os.RemoveAll("_autoclose")
os.Mkdir("_autoclose", 0755)
file := filepath.FromSlash("_autoclose/tmp")
data := []byte("hello, world\n")
// An autoclosed file that closes very quickly
ac := newAutoclosedFile(file, time.Millisecond, time.Millisecond)
// Write some data.
if _, err := ac.Write(data); err != nil {
t.Fatal(err)
}
// Wait for it to close
start := time.Now()
for {
time.Sleep(time.Millisecond)
ac.mut.Lock()
fd := ac.fd
ac.mut.Unlock()
if fd == nil {
break
}
if time.Since(start) > time.Second {
t.Fatal("File should have been closed after first write")
}
}
// Write more data, which should be an append.
if _, err := ac.Write(data); err != nil {
t.Fatal(err)
}
// Close.
if err := ac.Close(); err != nil {
t.Fatal(err)
}
// The file should have both writes in it.
bs, err := ioutil.ReadFile(file)
if err != nil {
t.Fatal(err)
}
if len(bs) != 2*len(data) {
t.Fatalf("Writes failed, expected %d bytes, not %d", 2*len(data), len(bs))
}
// Open the file again.
ac = newAutoclosedFile(file, time.Second, time.Second)
// Write something
if _, err := ac.Write(data); err != nil {
t.Fatal(err)
}
// It should now contain only one write, because the first open
// should be a truncate.
bs, err = ioutil.ReadFile(file)
if err != nil {
t.Fatal(err)
}
if len(bs) != len(data) {
t.Fatalf("Write failed, expected %d bytes, not %d", len(data), len(bs))
}
// Close.
if err := ac.Close(); err != nil {
t.Fatal(err)
}
}

View File

@@ -38,10 +38,7 @@ func savePerfStats(file string) {
t0 := time.Now()
for t := range time.NewTicker(250 * time.Millisecond).C {
if err := syscall.Getrusage(syscall.RUSAGE_SELF, &rusage); err != nil {
continue
}
syscall.Getrusage(syscall.RUSAGE_SELF, &rusage)
curTime := time.Now().UnixNano()
timeDiff := curTime - prevTime
curUsage := rusage.Utime.Nano() + rusage.Stime.Nano()

View File

@@ -9,9 +9,7 @@ package main
import (
"time"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/events"
"github.com/syncthing/syncthing/lib/model"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/sync"
"github.com/thejerf/suture"
@@ -22,8 +20,8 @@ import (
type folderSummaryService struct {
*suture.Supervisor
cfg config.Wrapper
model model.Model
cfg configIntf
model modelIntf
stop chan struct{}
immediate chan string
@@ -36,11 +34,9 @@ type folderSummaryService struct {
lastEventReqMut sync.Mutex
}
func newFolderSummaryService(cfg config.Wrapper, m model.Model) *folderSummaryService {
func newFolderSummaryService(cfg configIntf, m modelIntf) *folderSummaryService {
service := &folderSummaryService{
Supervisor: suture.New("folderSummaryService", suture.Spec{
PassThroughPanics: true,
}),
Supervisor: suture.NewSimple("folderSummaryService"),
cfg: cfg,
model: m,
stop: make(chan struct{}),
@@ -64,7 +60,7 @@ func (c *folderSummaryService) Stop() {
// listenForUpdates subscribes to the event bus and makes note of folders that
// need their data recalculated.
func (c *folderSummaryService) listenForUpdates() {
sub := events.Default.Subscribe(events.LocalIndexUpdated | events.RemoteIndexUpdated | events.StateChanged | events.RemoteDownloadProgress | events.DeviceConnected | events.FolderWatchStateChanged)
sub := events.Default.Subscribe(events.LocalIndexUpdated | events.RemoteIndexUpdated | events.StateChanged | events.RemoteDownloadProgress | events.DeviceConnected)
defer events.Default.Unsubscribe(sub)
for {
@@ -109,14 +105,14 @@ func (c *folderSummaryService) listenForUpdates() {
// c.immediate must be nonblocking so that we can continue
// handling events.
c.foldersMut.Lock()
select {
case c.immediate <- folder:
c.foldersMut.Lock()
delete(c.folders, folder)
c.foldersMut.Unlock()
default:
c.folders[folder] = struct{}{}
}
c.foldersMut.Unlock()
}
default:
@@ -191,10 +187,7 @@ func (c *folderSummaryService) foldersToHandle() []string {
func (c *folderSummaryService) sendSummary(folder string) {
// The folder summary contains how many bytes, files etc
// are in the folder and how in sync we are.
data, err := folderSummary(c.cfg, c.model, folder)
if err != nil {
return
}
data := folderSummary(c.cfg, c.model, folder)
events.Default.Log(events.FolderSummary, map[string]interface{}{
"folder": folder,
"summary": data,
@@ -205,17 +198,21 @@ func (c *folderSummaryService) sendSummary(folder string) {
// We already know about ourselves.
continue
}
if _, ok := c.model.Connection(devCfg.DeviceID); !ok {
if !c.model.ConnectedTo(devCfg.DeviceID) {
// We're not interested in disconnected devices.
continue
}
// Get completion percentage of this folder for the
// remote device.
comp := jsonCompletion(c.model.Completion(devCfg.DeviceID, folder))
comp["folder"] = folder
comp["device"] = devCfg.DeviceID.String()
events.Default.Log(events.FolderCompletion, comp)
comp := c.model.Completion(devCfg.DeviceID, folder)
events.Default.Log(events.FolderCompletion, map[string]interface{}{
"folder": folder,
"device": devCfg.DeviceID.String(),
"completion": comp.CompletionPct,
"needBytes": comp.NeedBytes,
"globalBytes": comp.GlobalBytes,
})
}
}

View File

@@ -1,47 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"archive/zip"
"io"
"github.com/syncthing/syncthing/lib/config"
)
// getRedactedConfig redacting some parts of config
func getRedactedConfig(s *apiService) config.Configuration {
rawConf := s.cfg.RawCopy()
rawConf.GUI.APIKey = "REDACTED"
if rawConf.GUI.Password != "" {
rawConf.GUI.Password = "REDACTED"
}
if rawConf.GUI.User != "" {
rawConf.GUI.User = "REDACTED"
}
return rawConf
}
// writeZip writes a zip file containing the given entries
func writeZip(writer io.Writer, files []fileEntry) error {
zipWriter := zip.NewWriter(writer)
defer zipWriter.Close()
for _, file := range files {
zipFile, err := zipWriter.Create(file.name)
if err != nil {
return err
}
_, err = zipFile.Write(file.data)
if err != nil {
return err
}
}
return zipWriter.Close()
}

View File

@@ -12,38 +12,82 @@ import (
"crypto/rand"
"crypto/tls"
"encoding/json"
"net"
"fmt"
"net/http"
"runtime"
"sort"
"strings"
"sync"
"time"
"github.com/syncthing/syncthing/lib/build"
"github.com/syncthing/syncthing/lib/config"
"github.com/syncthing/syncthing/lib/connections"
"github.com/syncthing/syncthing/lib/dialer"
"github.com/syncthing/syncthing/lib/model"
"github.com/syncthing/syncthing/lib/protocol"
"github.com/syncthing/syncthing/lib/scanner"
"github.com/syncthing/syncthing/lib/upgrade"
"github.com/thejerf/suture"
)
// Current version number of the usage report, for acceptance purposes. If
// fields are added or changed this integer must be incremented so that users
// are prompted for acceptance of the new report.
const usageReportVersion = 3
const usageReportVersion = 2
type usageReportingManager struct {
cfg *config.Wrapper
model *model.Model
sup *suture.Supervisor
}
func newUsageReportingManager(cfg *config.Wrapper, m *model.Model) *usageReportingManager {
mgr := &usageReportingManager{
cfg: cfg,
model: m,
}
// Start UR if it's enabled.
mgr.CommitConfiguration(config.Configuration{}, cfg.RawCopy())
// Listen to future config changes so that we can start and stop as
// appropriate.
cfg.Subscribe(mgr)
return mgr
}
func (m *usageReportingManager) VerifyConfiguration(from, to config.Configuration) error {
return nil
}
func (m *usageReportingManager) CommitConfiguration(from, to config.Configuration) bool {
if to.Options.URAccepted >= usageReportVersion && m.sup == nil {
// Usage reporting was turned on; lets start it.
service := newUsageReportingService(m.cfg, m.model)
m.sup = suture.NewSimple("usageReporting")
m.sup.Add(service)
m.sup.ServeBackground()
} else if to.Options.URAccepted < usageReportVersion && m.sup != nil {
// Usage reporting was turned off
m.sup.Stop()
m.sup = nil
}
return true
}
func (m *usageReportingManager) String() string {
return fmt.Sprintf("usageReportingManager@%p", m)
}
// reportData returns the data to be sent in a usage report. It's used in
// various places, so not part of the usageReportingManager object.
func reportData(cfg config.Wrapper, m model.Model, connectionsService connections.Service, version int, preview bool) map[string]interface{} {
func reportData(cfg configIntf, m modelIntf) map[string]interface{} {
opts := cfg.Options()
res := make(map[string]interface{})
res["urVersion"] = version
res["urVersion"] = usageReportVersion
res["uniqueID"] = opts.URUniqueID
res["version"] = build.Version
res["longVersion"] = build.LongVersion
res["version"] = Version
res["longVersion"] = LongVersion
res["platform"] = runtime.GOOS + "-" + runtime.GOARCH
res["numFolders"] = len(cfg.Folders())
res["numDevices"] = len(cfg.Devices())
@@ -52,10 +96,10 @@ func reportData(cfg config.Wrapper, m model.Model, connectionsService connection
var totBytes, maxBytes int64
for folderID := range cfg.Folders() {
global := m.GlobalSize(folderID)
totFiles += int(global.Files)
totFiles += global.Files
totBytes += global.Bytes
if int(global.Files) > maxFiles {
maxFiles = int(global.Files)
if global.Files > maxFiles {
maxFiles = global.Files
}
if global.Bytes > maxBytes {
maxBytes = global.Bytes
@@ -81,9 +125,7 @@ func reportData(cfg config.Wrapper, m model.Model, connectionsService connection
var rescanIntvs []int
folderUses := map[string]int{
"sendonly": 0,
"sendreceive": 0,
"receiveonly": 0,
"readonly": 0,
"ignorePerms": 0,
"ignoreDelete": 0,
"autoNormalize": 0,
@@ -95,13 +137,8 @@ func reportData(cfg config.Wrapper, m model.Model, connectionsService connection
for _, cfg := range cfg.Folders() {
rescanIntvs = append(rescanIntvs, cfg.RescanIntervalS)
switch cfg.Type {
case config.FolderTypeSendOnly:
folderUses["sendonly"]++
case config.FolderTypeSendReceive:
folderUses["sendreceive"]++
case config.FolderTypeReceiveOnly:
folderUses["receiveonly"]++
if cfg.Type == config.FolderTypeSendOnly {
folderUses["readonly"]++
}
if cfg.IgnorePerms {
folderUses["ignorePerms"]++
@@ -190,168 +227,27 @@ func reportData(cfg config.Wrapper, m model.Model, connectionsService connection
res["upgradeAllowedAuto"] = !(upgrade.DisabledByCompilation || noUpgradeFromEnv) && opts.AutoUpgradeIntervalH > 0
res["upgradeAllowedPre"] = !(upgrade.DisabledByCompilation || noUpgradeFromEnv) && opts.AutoUpgradeIntervalH > 0 && opts.UpgradeToPreReleases
if version >= 3 {
res["uptime"] = int(time.Since(startTime).Seconds())
res["natType"] = connectionsService.NATType()
res["alwaysLocalNets"] = len(opts.AlwaysLocalNets) > 0
res["cacheIgnoredFiles"] = opts.CacheIgnoredFiles
res["overwriteRemoteDeviceNames"] = opts.OverwriteRemoteDevNames
res["progressEmitterEnabled"] = opts.ProgressUpdateIntervalS > -1
res["customDefaultFolderPath"] = opts.DefaultFolderPath != "~"
res["customTrafficClass"] = opts.TrafficClass != 0
res["customTempIndexMinBlocks"] = opts.TempIndexMinBlocks != 10
res["temporariesDisabled"] = opts.KeepTemporariesH == 0
res["temporariesCustom"] = opts.KeepTemporariesH != 24
res["limitBandwidthInLan"] = opts.LimitBandwidthInLan
res["customReleaseURL"] = opts.ReleasesURL != "https://upgrades.syncthing.net/meta.json"
res["restartOnWakeup"] = opts.RestartOnWakeup
folderUsesV3 := map[string]int{
"scanProgressDisabled": 0,
"conflictsDisabled": 0,
"conflictsUnlimited": 0,
"conflictsOther": 0,
"disableSparseFiles": 0,
"disableTempIndexes": 0,
"alwaysWeakHash": 0,
"customWeakHashThreshold": 0,
"fsWatcherEnabled": 0,
}
pullOrder := make(map[string]int)
filesystemType := make(map[string]int)
var fsWatcherDelays []int
for _, cfg := range cfg.Folders() {
if cfg.ScanProgressIntervalS < 0 {
folderUsesV3["scanProgressDisabled"]++
}
if cfg.MaxConflicts == 0 {
folderUsesV3["conflictsDisabled"]++
} else if cfg.MaxConflicts < 0 {
folderUsesV3["conflictsUnlimited"]++
} else {
folderUsesV3["conflictsOther"]++
}
if cfg.DisableSparseFiles {
folderUsesV3["disableSparseFiles"]++
}
if cfg.DisableTempIndexes {
folderUsesV3["disableTempIndexes"]++
}
if cfg.WeakHashThresholdPct < 0 {
folderUsesV3["alwaysWeakHash"]++
} else if cfg.WeakHashThresholdPct != 25 {
folderUsesV3["customWeakHashThreshold"]++
}
if cfg.FSWatcherEnabled {
folderUsesV3["fsWatcherEnabled"]++
}
pullOrder[cfg.Order.String()]++
filesystemType[cfg.FilesystemType.String()]++
fsWatcherDelays = append(fsWatcherDelays, cfg.FSWatcherDelayS)
}
sort.Ints(fsWatcherDelays)
folderUsesV3Interface := map[string]interface{}{
"pullOrder": pullOrder,
"filesystemType": filesystemType,
"fsWatcherDelays": fsWatcherDelays,
}
for key, value := range folderUsesV3 {
folderUsesV3Interface[key] = value
}
res["folderUsesV3"] = folderUsesV3Interface
guiCfg := cfg.GUI()
// Anticipate multiple GUI configs in the future, hence store counts.
guiStats := map[string]int{
"enabled": 0,
"useTLS": 0,
"useAuth": 0,
"insecureAdminAccess": 0,
"debugging": 0,
"insecureSkipHostCheck": 0,
"insecureAllowFrameLoading": 0,
"listenLocal": 0,
"listenUnspecified": 0,
}
theme := make(map[string]int)
if guiCfg.Enabled {
guiStats["enabled"]++
if guiCfg.UseTLS() {
guiStats["useTLS"]++
}
if len(guiCfg.User) > 0 && len(guiCfg.Password) > 0 {
guiStats["useAuth"]++
}
if guiCfg.InsecureAdminAccess {
guiStats["insecureAdminAccess"]++
}
if guiCfg.Debugging {
guiStats["debugging"]++
}
if guiCfg.InsecureSkipHostCheck {
guiStats["insecureSkipHostCheck"]++
}
if guiCfg.InsecureAllowFrameLoading {
guiStats["insecureAllowFrameLoading"]++
}
addr, err := net.ResolveTCPAddr("tcp", guiCfg.Address())
if err == nil {
if addr.IP.IsLoopback() {
guiStats["listenLocal"]++
} else if addr.IP.IsUnspecified() {
guiStats["listenUnspecified"]++
}
}
theme[guiCfg.Theme]++
}
guiStatsInterface := map[string]interface{}{
"theme": theme,
}
for key, value := range guiStats {
guiStatsInterface[key] = value
}
res["guiStats"] = guiStatsInterface
}
for key, value := range m.UsageReportingStats(version, preview) {
res[key] = value
}
return res
}
type usageReportingService struct {
cfg config.Wrapper
model model.Model
connectionsService connections.Service
forceRun chan struct{}
stop chan struct{}
stopped chan struct{}
stopMut sync.RWMutex
cfg *config.Wrapper
model *model.Model
stop chan struct{}
}
func newUsageReportingService(cfg config.Wrapper, model model.Model, connectionsService connections.Service) *usageReportingService {
svc := &usageReportingService{
cfg: cfg,
model: model,
connectionsService: connectionsService,
forceRun: make(chan struct{}),
stop: make(chan struct{}),
stopped: make(chan struct{}),
func newUsageReportingService(cfg *config.Wrapper, model *model.Model) *usageReportingService {
return &usageReportingService{
cfg: cfg,
model: model,
stop: make(chan struct{}),
}
close(svc.stopped) // Not yet running, dont block on Stop()
cfg.Subscribe(svc)
return svc
}
func (s *usageReportingService) sendUsageReport() error {
d := reportData(s.cfg, s.model, s.connectionsService, s.cfg.Options().URAccepted, false)
d := reportData(s.cfg, s.model)
var b bytes.Buffer
if err := json.NewEncoder(&b).Encode(d); err != nil {
return err
}
json.NewEncoder(&b).Encode(d)
client := &http.Client{
Transport: &http.Transport{
@@ -367,66 +263,33 @@ func (s *usageReportingService) sendUsageReport() error {
}
func (s *usageReportingService) Serve() {
s.stopMut.Lock()
s.stop = make(chan struct{})
s.stopped = make(chan struct{})
s.stopMut.Unlock()
t := time.NewTimer(time.Duration(s.cfg.Options().URInitialDelayS) * time.Second)
s.stopMut.RLock()
defer func() {
close(s.stopped)
s.stopMut.RUnlock()
}()
l.Infoln("Starting usage reporting")
defer l.Infoln("Stopping usage reporting")
t := time.NewTimer(time.Duration(s.cfg.Options().URInitialDelayS) * time.Second) // time to initial report at start
for {
select {
case <-s.stop:
return
case <-s.forceRun:
t.Reset(0)
case <-t.C:
if s.cfg.Options().URAccepted >= 2 {
err := s.sendUsageReport()
if err != nil {
l.Infoln("Usage report:", err)
} else {
l.Infof("Sent usage report (version %d)", s.cfg.Options().URAccepted)
}
err := s.sendUsageReport()
if err != nil {
l.Infoln("Usage report:", err)
}
t.Reset(24 * time.Hour) // next report tomorrow
}
}
}
func (s *usageReportingService) VerifyConfiguration(from, to config.Configuration) error {
return nil
}
func (s *usageReportingService) CommitConfiguration(from, to config.Configuration) bool {
if from.Options.URAccepted != to.Options.URAccepted || from.Options.URUniqueID != to.Options.URUniqueID || from.Options.URURL != to.Options.URURL {
s.stopMut.RLock()
select {
case s.forceRun <- struct{}{}:
case <-s.stop:
}
s.stopMut.RUnlock()
}
return true
}
func (s *usageReportingService) Stop() {
s.stopMut.RLock()
close(s.stop)
<-s.stopped
s.stopMut.RUnlock()
}
func (*usageReportingService) String() string {
return "usageReportingService"
}
// cpuBench returns CPU performance as a measure of single threaded SHA-256 MiB/s
func cpuBench(iterations int, duration time.Duration, useWeakHash bool) float64 {
dataSize := 16 * protocol.MinBlockSize
dataSize := 16 * protocol.BlockSize
bs := make([]byte, dataSize)
rand.Reader.Read(bs)
@@ -447,7 +310,7 @@ func cpuBenchOnce(duration time.Duration, useWeakHash bool, bs []byte) float64 {
b := 0
for time.Since(t0) < duration {
r := bytes.NewReader(bs)
blocksResult, _ = scanner.Blocks(context.TODO(), r, protocol.MinBlockSize, int64(len(bs)), nil, useWeakHash)
blocksResult, _ = scanner.Blocks(context.TODO(), r, protocol.BlockSize, int64(len(bs)), nil, useWeakHash)
b += len(bs)
}
d := time.Since(t0)

View File

@@ -105,7 +105,7 @@ func (s *verboseService) formatEvent(ev events.Event) string {
return fmt.Sprintf("Device %v sent an index update for %q with %d items", data["device"], data["folder"], data["items"])
case events.DeviceRejected:
data := ev.Data.(map[string]string)
data := ev.Data.(map[string]interface{})
return fmt.Sprintf("Rejected connection from device %v at %v", data["device"], data["address"])
case events.FolderRejected:

View File

@@ -1,330 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"database/sql"
"log"
"os"
"time"
_ "github.com/lib/pq"
)
var dbConn = getEnvDefault("UR_DB_URL", "postgres://user:password@localhost/ur?sslmode=disable")
func getEnvDefault(key, def string) string {
if val := os.Getenv(key); val != "" {
return val
}
return def
}
func main() {
log.SetFlags(log.Ltime | log.Ldate)
log.SetOutput(os.Stdout)
db, err := sql.Open("postgres", dbConn)
if err != nil {
log.Fatalln("database:", err)
}
err = setupDB(db)
if err != nil {
log.Fatalln("database:", err)
}
for {
runAggregation(db)
// Sleep until one minute past next midnight
sleepUntilNext(24*time.Hour, 1*time.Minute)
}
}
func runAggregation(db *sql.DB) {
since := maxIndexedDay(db, "VersionSummary")
log.Println("Aggregating VersionSummary data since", since)
rows, err := aggregateVersionSummary(db, since)
if err != nil {
log.Fatalln("aggregate:", err)
}
log.Println("Inserted", rows, "rows")
log.Println("Aggregating UserMovement data")
rows, err = aggregateUserMovement(db)
if err != nil {
log.Fatalln("aggregate:", err)
}
log.Println("Inserted", rows, "rows")
log.Println("Aggregating Performance data")
since = maxIndexedDay(db, "Performance")
rows, err = aggregatePerformance(db, since)
if err != nil {
log.Fatalln("aggregate:", err)
}
log.Println("Inserted", rows, "rows")
log.Println("Aggregating BlockStats data")
since = maxIndexedDay(db, "BlockStats")
rows, err = aggregateBlockStats(db, since)
if err != nil {
log.Fatalln("aggregate:", err)
}
log.Println("Inserted", rows, "rows")
}
func sleepUntilNext(intv, margin time.Duration) {
now := time.Now().UTC()
next := now.Truncate(intv).Add(intv).Add(margin)
log.Println("Sleeping until", next)
time.Sleep(next.Sub(now))
}
func setupDB(db *sql.DB) error {
_, err := db.Exec(`CREATE TABLE IF NOT EXISTS VersionSummary (
Day TIMESTAMP NOT NULL,
Version VARCHAR(8) NOT NULL,
Count INTEGER NOT NULL
)`)
if err != nil {
return err
}
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS UserMovement (
Day TIMESTAMP NOT NULL,
Added INTEGER NOT NULL,
Bounced INTEGER NOT NULL,
Removed INTEGER NOT NULL
)`)
if err != nil {
return err
}
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS Performance (
Day TIMESTAMP NOT NULL,
TotFiles INTEGER NOT NULL,
TotMiB INTEGER NOT NULL,
SHA256Perf DOUBLE PRECISION NOT NULL,
MemorySize INTEGER NOT NULL,
MemoryUsageMiB INTEGER NOT NULL
)`)
if err != nil {
return err
}
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS BlockStats (
Day TIMESTAMP NOT NULL,
Reports INTEGER NOT NULL,
Total INTEGER NOT NULL,
Renamed INTEGER NOT NULL,
Reused INTEGER NOT NULL,
Pulled INTEGER NOT NULL,
CopyOrigin INTEGER NOT NULL,
CopyOriginShifted INTEGER NOT NULL,
CopyElsewhere INTEGER NOT NULL
)`)
if err != nil {
return err
}
var t string
row := db.QueryRow(`SELECT 'UniqueDayVersionIndex'::regclass`)
if err := row.Scan(&t); err != nil {
_, err = db.Exec(`CREATE UNIQUE INDEX UniqueDayVersionIndex ON VersionSummary (Day, Version)`)
}
row = db.QueryRow(`SELECT 'VersionDayIndex'::regclass`)
if err := row.Scan(&t); err != nil {
_, err = db.Exec(`CREATE INDEX VersionDayIndex ON VersionSummary (Day)`)
}
row = db.QueryRow(`SELECT 'MovementDayIndex'::regclass`)
if err := row.Scan(&t); err != nil {
_, err = db.Exec(`CREATE INDEX MovementDayIndex ON UserMovement (Day)`)
}
row = db.QueryRow(`SELECT 'PerformanceDayIndex'::regclass`)
if err := row.Scan(&t); err != nil {
_, err = db.Exec(`CREATE INDEX PerformanceDayIndex ON Performance (Day)`)
}
row = db.QueryRow(`SELECT 'BlockStatsDayIndex'::regclass`)
if err := row.Scan(&t); err != nil {
_, err = db.Exec(`CREATE INDEX BlockStatsDayIndex ON BlockStats (Day)`)
}
return err
}
func maxIndexedDay(db *sql.DB, table string) time.Time {
var t time.Time
row := db.QueryRow("SELECT MAX(Day) FROM " + table)
err := row.Scan(&t)
if err != nil {
return time.Time{}
}
return t
}
func aggregateVersionSummary(db *sql.DB, since time.Time) (int64, error) {
res, err := db.Exec(`INSERT INTO VersionSummary (
SELECT
DATE_TRUNC('day', Received) AS Day,
SUBSTRING(Version FROM '^v\d.\d+') AS Ver,
COUNT(*) AS Count
FROM Reports
WHERE
DATE_TRUNC('day', Received) > $1
AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
AND Version like 'v_.%'
GROUP BY Day, Ver
);
`, since)
if err != nil {
return 0, err
}
return res.RowsAffected()
}
func aggregateUserMovement(db *sql.DB) (int64, error) {
rows, err := db.Query(`SELECT
DATE_TRUNC('day', Received) AS Day,
UniqueID
FROM Reports
WHERE
DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
AND Version like 'v_.%'
ORDER BY Day
`)
if err != nil {
return 0, err
}
defer rows.Close()
firstSeen := make(map[string]time.Time)
lastSeen := make(map[string]time.Time)
var minTs time.Time
minTs = minTs.In(time.UTC)
for rows.Next() {
var ts time.Time
var id string
if err := rows.Scan(&ts, &id); err != nil {
return 0, err
}
if minTs.IsZero() {
minTs = ts
}
if _, ok := firstSeen[id]; !ok {
firstSeen[id] = ts
}
lastSeen[id] = ts
}
type sumRow struct {
day time.Time
added int
removed int
bounced int
}
var sumRows []sumRow
for t := minTs; t.Before(time.Now().Truncate(24 * time.Hour)); t = t.AddDate(0, 0, 1) {
var added, removed, bounced int
old := t.Before(time.Now().AddDate(0, 0, -30))
for id, first := range firstSeen {
last := lastSeen[id]
if first.Equal(t) && last.Equal(t) && old {
bounced++
continue
}
if first.Equal(t) {
added++
}
if last == t && old {
removed++
}
}
sumRows = append(sumRows, sumRow{t, added, removed, bounced})
}
tx, err := db.Begin()
if err != nil {
return 0, err
}
if _, err := tx.Exec("DELETE FROM UserMovement"); err != nil {
tx.Rollback()
return 0, err
}
for _, r := range sumRows {
if _, err := tx.Exec("INSERT INTO UserMovement (Day, Added, Removed, Bounced) VALUES ($1, $2, $3, $4)", r.day, r.added, r.removed, r.bounced); err != nil {
tx.Rollback()
return 0, err
}
}
return int64(len(sumRows)), tx.Commit()
}
func aggregatePerformance(db *sql.DB, since time.Time) (int64, error) {
res, err := db.Exec(`INSERT INTO Performance (
SELECT
DATE_TRUNC('day', Received) AS Day,
AVG(TotFiles) As TotFiles,
AVG(TotMiB) As TotMiB,
AVG(SHA256Perf) As SHA256Perf,
AVG(MemorySize) As MemorySize,
AVG(MemoryUsageMiB) As MemoryUsageMiB
FROM Reports
WHERE
DATE_TRUNC('day', Received) > $1
AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
AND Version like 'v_.%'
GROUP BY Day
);
`, since)
if err != nil {
return 0, err
}
return res.RowsAffected()
}
func aggregateBlockStats(db *sql.DB, since time.Time) (int64, error) {
// Filter out anything prior 0.14.41 as that has sum aggregations which
// made no sense.
res, err := db.Exec(`INSERT INTO BlockStats (
SELECT
DATE_TRUNC('day', Received) AS Day,
COUNT(1) As Reports,
SUM(BlocksTotal) AS Total,
SUM(BlocksRenamed) AS Renamed,
SUM(BlocksReused) AS Reused,
SUM(BlocksPulled) AS Pulled,
SUM(BlocksCopyOrigin) AS CopyOrigin,
SUM(BlocksCopyOriginShifted) AS CopyOriginShifted,
SUM(BlocksCopyElsewhere) AS CopyElsewhere
FROM Reports
WHERE
DATE_TRUNC('day', Received) > $1
AND DATE_TRUNC('day', Received) < DATE_TRUNC('day', NOW())
AND ReportVersion = 3
AND Version like 'v_.%'
AND Version NOT LIKE 'v0.14.40%'
AND Version NOT LIKE 'v0.14.39%'
AND Version NOT LIKE 'v0.14.38%'
GROUP BY Day
);
`, since)
if err != nil {
return 0, err
}
return res.RowsAffected()
}

View File

@@ -1,240 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"regexp"
"sort"
"strconv"
"strings"
)
type analytic struct {
Key string
Count int
Percentage float64
Items []analytic `json:",omitempty"`
}
type analyticList []analytic
func (l analyticList) Less(a, b int) bool {
if l[a].Key == "Others" {
return true
}
if l[b].Key == "Others" {
return false
}
return l[b].Count < l[a].Count // inverse
}
func (l analyticList) Swap(a, b int) {
l[a], l[b] = l[b], l[a]
}
func (l analyticList) Len() int {
return len(l)
}
// Returns a list of frequency analytics for a given list of strings.
func analyticsFor(ss []string, cutoff int) []analytic {
m := make(map[string]int)
t := 0
for _, s := range ss {
m[s]++
t++
}
l := make([]analytic, 0, len(m))
for k, c := range m {
l = append(l, analytic{
Key: k,
Count: c,
Percentage: 100 * float64(c) / float64(t),
})
}
sort.Sort(analyticList(l))
if cutoff > 0 && len(l) > cutoff {
c := 0
for _, i := range l[cutoff:] {
c += i.Count
}
l = append(l[:cutoff], analytic{
Key: "Others",
Count: c,
Percentage: 100 * float64(c) / float64(t),
})
}
return l
}
// Find the points at which certain penetration levels are met
func penetrationLevels(as []analytic, points []float64) []analytic {
sort.Slice(as, func(a, b int) bool {
return versionLess(as[b].Key, as[a].Key)
})
var res []analytic
idx := 0
sum := 0.0
for _, a := range as {
sum += a.Percentage
if sum >= points[idx] {
a.Count = int(points[idx])
a.Percentage = sum
res = append(res, a)
idx++
if idx == len(points) {
break
}
}
}
return res
}
func statsForInts(data []int) [4]float64 {
var res [4]float64
if len(data) == 0 {
return res
}
sort.Ints(data)
res[0] = float64(data[int(float64(len(data))*0.05)])
res[1] = float64(data[len(data)/2])
res[2] = float64(data[int(float64(len(data))*0.95)])
res[3] = float64(data[len(data)-1])
return res
}
func statsForFloats(data []float64) [4]float64 {
var res [4]float64
if len(data) == 0 {
return res
}
sort.Float64s(data)
res[0] = data[int(float64(len(data))*0.05)]
res[1] = data[len(data)/2]
res[2] = data[int(float64(len(data))*0.95)]
res[3] = data[len(data)-1]
return res
}
func group(by func(string) string, as []analytic, perGroup int) []analytic {
var res []analytic
next:
for _, a := range as {
group := by(a.Key)
for i := range res {
if res[i].Key == group {
res[i].Count += a.Count
res[i].Percentage += a.Percentage
if len(res[i].Items) < perGroup {
res[i].Items = append(res[i].Items, a)
}
continue next
}
}
res = append(res, analytic{
Key: group,
Count: a.Count,
Percentage: a.Percentage,
Items: []analytic{a},
})
}
sort.Sort(analyticList(res))
return res
}
func byVersion(s string) string {
parts := strings.Split(s, ".")
if len(parts) >= 2 {
return strings.Join(parts[:2], ".")
}
return s
}
func byPlatform(s string) string {
parts := strings.Split(s, "-")
if len(parts) >= 2 {
return parts[0]
}
return s
}
var numericGoVersion = regexp.MustCompile(`^go[0-9]\.[0-9]+`)
func byCompiler(s string) string {
if m := numericGoVersion.FindString(s); m != "" {
return m
}
return "Other"
}
func versionLess(a, b string) bool {
arel, apre := versionParts(a)
brel, bpre := versionParts(b)
minlen := len(arel)
if l := len(brel); l < minlen {
minlen = l
}
for i := 0; i < minlen; i++ {
if arel[i] != brel[i] {
return arel[i] < brel[i]
}
}
// Longer version is newer, when the preceding parts are equal
if len(arel) != len(brel) {
return len(arel) < len(brel)
}
if apre != bpre {
// "(+dev)" versions are ahead
if apre == plusStr {
return false
}
if bpre == plusStr {
return true
}
return apre < bpre
}
// don't actually care how the prerelease stuff compares for our purposes
return false
}
// Split a version as returned from transformVersion into parts.
// "1.2.3-beta.2" -> []int{1, 2, 3}, "beta.2"}
func versionParts(v string) ([]int, string) {
parts := strings.SplitN(v[1:], " ", 2) // " (+dev)" versions
if len(parts) == 1 {
parts = strings.SplitN(parts[0], "-", 2) // "-rc.1" type versions
}
fields := strings.Split(parts[0], ".")
release := make([]int, len(fields))
for i, s := range fields {
v, _ := strconv.Atoi(s)
release[i] = v
}
var prerelease string
if len(parts) > 1 {
prerelease = parts[1]
}
return release, prerelease
}

View File

@@ -1,31 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import "testing"
func TestCompilerRe(t *testing.T) {
tests := [][3]string{
{`syncthing v0.11.0 (xgcc (Ubuntu 4.9.3-0ubuntu4) 4.9.3 linux-amd64 default) niklas@Niklas-Netbook 2015-04-26 13:15:08 UTC`, "xgcc (Ubuntu 4.9.3-0ubuntu4) 4.9.3", "niklas@Niklas-Netbook"},
{`syncthing v0.12.0-rc5 "Beryllium Bedbug" (go1.4.2 linux-arm android) unknown-user@Felix-T420 2015-10-22 18:32:15 UTC`, "go1.4.2", "unknown-user@Felix-T420"},
{`syncthing v0.13.0-beta.0+39-ge267bf3 "Copper Cockroach" (go1.4.2 linux-amd64) portage@slevermann.de 2016-01-20 08:41:52 UTC`, "go1.4.2", "portage@slevermann.de"},
}
for _, tc := range tests {
m := compilerRe.FindStringSubmatch(tc[0])
if len(m) != 3 {
t.Errorf("Regexp didn't match %q", tc[0])
continue
}
if m[1] != tc[1] {
t.Errorf("Compiler %q != %q", m[1], tc[1])
}
if m[2] != tc[2] {
t.Errorf("Builder %q != %q", m[2], tc[2])
}
}
}

View File

@@ -1,128 +0,0 @@
// Copyright (C) 2018 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.
package main
import (
"bytes"
"fmt"
"strings"
)
type NumberType int
const (
NumberMetric NumberType = iota
NumberBinary
NumberDuration
)
func number(ntype NumberType, v float64) string {
if ntype == NumberDuration {
return duration(v)
} else if ntype == NumberBinary {
return binary(v)
} else {
return metric(v)
}
}
type suffix struct {
Suffix string
Multiplier float64
}
var metricSuffixes = []suffix{
{"G", 1e9},
{"M", 1e6},
{"k", 1e3},
}
var binarySuffixes = []suffix{
{"Gi", 1 << 30},
{"Mi", 1 << 20},
{"Ki", 1 << 10},
}
var durationSuffix = []suffix{
{"year", 365 * 24 * 60 * 60},
{"month", 30 * 24 * 60 * 60},
{"day", 24 * 60 * 60},
{"hour", 60 * 60},
{"minute", 60},
{"second", 1},
}
func metric(v float64) string {
return withSuffix(v, metricSuffixes, false)
}
func binary(v float64) string {
return withSuffix(v, binarySuffixes, false)
}
func duration(v float64) string {
return withSuffix(v, durationSuffix, true)
}
func withSuffix(v float64, ps []suffix, pluralize bool) string {
for _, p := range ps {
if v >= p.Multiplier {
suffix := p.Suffix
if pluralize && v/p.Multiplier != 1.0 {
suffix += "s"
}
// If the number only has decimal zeroes, strip em off.
num := strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.1f", v/p.Multiplier), "0"), ".")
return fmt.Sprintf("%s %s", num, suffix)
}
}
return strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.1f", v), "0"), ".")
}
// commatize returns a number with sep as thousands separators. Handles
// integers and plain floats.
func commatize(sep, s string) string {
// If no dot, don't do anything.
if !strings.ContainsRune(s, '.') {
return s
}
var b bytes.Buffer
fs := strings.SplitN(s, ".", 2)
l := len(fs[0])
for i := range fs[0] {
b.Write([]byte{s[i]})
if i < l-1 && (l-i)%3 == 1 {
b.WriteString(sep)
}
}
if len(fs) > 1 && len(fs[1]) > 0 {
b.WriteString(".")
b.WriteString(fs[1])
}
return b.String()
}
func proportion(m map[string]int, count int) float64 {
total := 0
isMax := true
for _, n := range m {
total += n
if n > count {
isMax = false
}
}
pct := (100 * float64(count)) / float64(total)
// To avoid rounding errors in the template, surpassing 100% and breaking
// the progress bars.
if isMax && len(m) > 1 && count != total {
pct -= 0.01
}
return pct
}

View File

File diff suppressed because it is too large Load Diff

View File

Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.4 KiB

Some files were not shown because too many files have changed in this diff Show More