mirror of
https://github.com/syncthing/syncthing.git
synced 2026-08-07 05:43:24 -04:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f0473fde17 | ||
|
|
e2980a5210 | ||
|
|
764da14440 | ||
|
|
7da6c627fe | ||
|
|
0a092b5b7f | ||
|
|
9f8af2327d | ||
|
|
345e24142e | ||
|
|
0fdd03ddee | ||
|
|
a0fa288cb6 | ||
|
|
70bac24832 | ||
|
|
1df40fbdeb | ||
|
|
91e9ffff85 | ||
|
|
853df14e2f | ||
|
|
e17a772bb6 | ||
|
|
90e027d9a4 | ||
|
|
fdc9a5d8b0 | ||
|
|
543891a0a0 | ||
|
|
06921443fc |
No files matched your search
@@ -1,5 +0,0 @@
|
|||||||
coverage:
|
|
||||||
range: "40...100"
|
|
||||||
|
|
||||||
ignore:
|
|
||||||
- "**.pb.go"
|
|
||||||
+2
-1
@@ -2,7 +2,8 @@
|
|||||||
* text=auto
|
* text=auto
|
||||||
|
|
||||||
# Except the dependencies, which we leave alone
|
# Except the dependencies, which we leave alone
|
||||||
vendor/** -text=auto
|
Godeps/** -text=auto
|
||||||
|
|
||||||
# Diffs on these files are meaningless
|
# Diffs on these files are meaningless
|
||||||
|
gui.files.go -diff
|
||||||
*.svg -diff
|
*.svg -diff
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
/AUTHORS @calmh
|
|
||||||
/*.md @calmh
|
|
||||||
@@ -1,42 +0,0 @@
|
|||||||
### DO NOT REPORT SECURITY ISSUES IN THIS ISSUE TRACKER
|
|
||||||
|
|
||||||
Instead, contact security@syncthing.net directly - see
|
|
||||||
https://syncthing.net/security.html for more information.
|
|
||||||
|
|
||||||
### DO NOT POST SUPPORT REQUESTS OR GENERAL QUESTIONS IN THIS ISSUE TRACKER
|
|
||||||
|
|
||||||
Please use the forum at https://forum.syncthing.net/ where a large number of
|
|
||||||
helpful people hang out. This issue tracker is for reporting bugs or feature
|
|
||||||
requests directly to the developers. Worst case you might get a short
|
|
||||||
"that's a bug, please report it on GitHub" response on the forum, in which
|
|
||||||
case we thank you for your patience and following our advice. :)
|
|
||||||
|
|
||||||
### Please use the correct issue tracker
|
|
||||||
|
|
||||||
If your problem relates to a Syncthing wrapper or [sub-project](https://github.com/syncthing) such as [Syncthing for Android](https://github.com/syncthing/syncthing-android/issues), [SyncTrayzor](https://github.com/canton7/synctrayzor) or the [documentation](https://github.com/syncthing/docs/issues), please use their respective issue trackers.
|
|
||||||
|
|
||||||
### Does your log mention database corruption?
|
|
||||||
|
|
||||||
If your Syncthing log reports panics because of database corruption it is most likely a fault with your system's storage or memory. Affected log entries will contain lines starting with `panic: leveldb`. You will need to delete the index database to clear this, by running `syncthing -reset-database`.
|
|
||||||
|
|
||||||
### Please do post actual bug reports and feature requests.
|
|
||||||
|
|
||||||
If your issue is a bug report, replace this boilerplate with a description
|
|
||||||
of the problem, being sure to include at least:
|
|
||||||
|
|
||||||
- what happened,
|
|
||||||
- what you expected to happen instead, and
|
|
||||||
- any steps to reproduce the problem.
|
|
||||||
|
|
||||||
Also fill out the version information below and add log output or
|
|
||||||
screenshots as appropriate.
|
|
||||||
|
|
||||||
If your issue is a feature request, simply replace this template text in
|
|
||||||
its entirety.
|
|
||||||
|
|
||||||
### Version Information
|
|
||||||
|
|
||||||
Syncthing Version: v0.x.y
|
|
||||||
OS Version: Windows 7 / Ubuntu 14.04 / ...
|
|
||||||
Browser Version: (if applicable, for GUI issues)
|
|
||||||
|
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
### Purpose
|
|
||||||
|
|
||||||
Describe the purpose of this change. If there is an existing issue that is
|
|
||||||
resolved by this pull request, ensure that the commit subject is on the form
|
|
||||||
`Some short description (fixes #1234)` where 1234 is the issue number.
|
|
||||||
|
|
||||||
### Testing
|
|
||||||
|
|
||||||
Describe what testing has been done, and how the reviewer can test the change
|
|
||||||
if new tests are not included.
|
|
||||||
|
|
||||||
### Screenshots
|
|
||||||
|
|
||||||
If this is a GUI change, include screenshots of the change. If not, please
|
|
||||||
feel free to just delete this section.
|
|
||||||
|
|
||||||
### Documentation
|
|
||||||
|
|
||||||
If this is a user visible change (including API and protocol changes), add a link here
|
|
||||||
to the corresponding pull request on https://github.com/syncthing/docs or describe
|
|
||||||
the documentation changes necessary.
|
|
||||||
|
|
||||||
## Authorship
|
|
||||||
|
|
||||||
Your name and email will be added automatically to the AUTHORS file
|
|
||||||
based on the commit metadata.
|
|
||||||
|
|
||||||
@@ -1,48 +0,0 @@
|
|||||||
## Reporting a Vulnerability
|
|
||||||
|
|
||||||
If you believe that you've found a Syncthing-related security vulnerability, please report it by sending email to the address security@syncthing.net. The PGP key for security@syncthing.net (B683AD7B76CAB013) below can be used to send encrypted mail or to verify responses received from that address.
|
|
||||||
|
|
||||||
```
|
|
||||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
|
||||||
Version: GnuPG v1
|
|
||||||
|
|
||||||
mQENBFShFlIBCACsW346HYskhKhxrdZMjyU5Ntsjvg6/ogqINDPoL10/oaIP0G+t
|
|
||||||
7zzC0K5Cq29ix43kNNLKTJNyPkdTeaJEcqslMUt6tovjHwoKJ073GP0W3KsNvBRg
|
|
||||||
ffCZOAexGfOsBSL9KHaYGK67Py3TFgtN1H/EmboU1arrLfAMrmqOip++EGqOxjse
|
|
||||||
gH0qk7Mk1TqEC5Xh3NGE7r1UobAlqdUv5E3v7U11NhAdP1zu/XZ/zvP5mwVQJMLv
|
|
||||||
iZyeWGliNI8nEeRjYw+S85f4gq7H2mgoeNBN4WwwK1hhz9qpvCsgPW3XqlExTPI4
|
|
||||||
1vM4PxpiFIuF0zuy2OwsmjrpTCZeBscr4Tj5ABEBAAG0K1N5bmN0aGluZyBTZWN1
|
|
||||||
cml0eSA8c2VjdXJpdHlAc3luY3RoaW5nLm5ldD6JATgEEwECACIFAlShFlICGwMG
|
|
||||||
CwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJELaDrXt2yrATB1UH/jnnIa6DekCA
|
|
||||||
2V36N/2+pFSNLVWeoZQxZ9ne9S7WSubaoK4oCWiuChPSAy5hagnKnNA3a9wrz5iN
|
|
||||||
0hgCA88NnTU18biZW6HX8xWEd3dnY3fX1sG0XdHuKlFria8ByfcrbShf/CttXkEd
|
|
||||||
Y5qPHH9aLIMtksBS1MIsRYrOCHiLNYFCKlbjDDCGT3tuk/yaU7aBAFVPIag54afC
|
|
||||||
zZAIvTIgRruLWkNT/sSEJx9ekhJzO/+pXNbSSHwhoj5OJh7sQjZWwPzNazelk48R
|
|
||||||
+ku8VpB5Wxgk2MnJPf1RxF+7saBAaeAVZsJcPN+Jp7u9LCFelIRn1ISsHbhLhyqL
|
|
||||||
cPXqllltLCKJAhwEEAECAAYFAlShFlsACgkQSfWuwLzlJMcEpA//VvZYHGZgZMM0
|
|
||||||
bBkYnjManYnJkHSQ2pHsjquSFH+fbfq2FxxTk/nQ/IAvBzt8NDTT3ylPOmsl4A8q
|
|
||||||
r8BxsRNENhU7zDQrKC5NtYUrzXhPGo8qfDwkeLyqd2msUvHn7EH3PNgiN2QaFWxE
|
|
||||||
21FqoXIpERKJRgRtv1SYZoMyNGjmT2hta1ZbwEfrPJnzjYhneoUGsRApG7p+uPqe
|
|
||||||
4LRpbG3Fa2vBm/UWUrOe+6jPzvMokjIJbdK0IjXamFAzwYW5fSZaFa94mR6L5f5m
|
|
||||||
X8Dhp66mBRTx7qk6ldEqptvaYGqihaWP1xbDaApQsGHQujtcBYGp+edlkabuW5h7
|
|
||||||
stl/7QTFkEPqHT4ybxEX0rLoFUGq56WUlKp27z40keStaXMgfrsxJtkz66Xs8vU4
|
|
||||||
7qZcLAcPsin+y0toqavtwtM/L7uCMu1yhbFRGJ+JL6saJAqzwS8l7r6E2R7OnVj1
|
|
||||||
BdASgxx1TgzW6ZFW5p1Iy6mtpkBypsnp8s3UcP3GFRnQe9gi1EXHjzuZAUGafe4Q
|
|
||||||
juvJ8t8xIcQMFuAylNIVyXvIWJoehqsVY9EBgVtE4y1SRh8XTT3Tddn8ab5fl7uh
|
|
||||||
HWAY8cRlv6WIOhK8w8oroiYx0SID8jMeEwJBS4DL7qWtJMkDo8ZEJiB+Pkd963MX
|
|
||||||
05QXt33AvJJ9PmbGCHkcH7198tCmA+e5AQ0EVKEWUgEIAPGczHpa6NdxY0pm+tIp
|
|
||||||
btiA6gdPE70pJYgJTKX7siCQ2w770H3CBSKmqEXadr7WnfIgUYIDaSxadeGzB/Mn
|
|
||||||
3SHCYRCqbA7mwu2k4wNNvCEM0xZqFAvaDJ4avlZ5oiMT8IFHKsjC77nkhmfXaIGt
|
|
||||||
hn10H2MFADjvJYul7vR+Ghg1wGhTGWo2u7VVj9BI+AfvnWaouFI0cx2KNWEI/Ocj
|
|
||||||
z6jk8nmC3yOEFQECM/hF4lkAOv9CQUa8UcvAr31trzawmV1iSsKjmVZgqd0N4T8f
|
|
||||||
hUikqUPZGNCRcqEUffTzggIyGPbedFnZw9Di7o1xByxyTrZycemAVqaVGF+9nFLG
|
|
||||||
pccAEQEAAYkBHwQYAQIACQUCVKEWUgIbDAAKCRC2g617dsqwEzrjB/9q0T8A4XUE
|
|
||||||
p0g6xq86jhmh4jlEedxrfXUL3R6ejFtuKMThulxEP0xiQ7xLzBhOnvyxLCsVbjp8
|
|
||||||
0JtJTVCq44UzUiIBuVNRoYG29uXuTUL2UtI27VhjFxMzLDwZL97tbGR6lzdM/+U5
|
|
||||||
9PC+PIvS0yz13z2t3/x3KUOnBgxZnpy9h4AdKwjrNVtnQdGDXSKlJBLb57TFcS94
|
|
||||||
f3roZ5Gpw3AWYSmSSiZWbhks2UNzYSQ84LAKlV1NkktO+qRc/pUqr6IxMjOKc7XP
|
|
||||||
e8u1Nst6fN3GNqZOV+jUYs/fqrJgp1TUWjNTuf22Rl0Idz7XLPJKYFh9W7T/4MbU
|
|
||||||
M7Q8GZuww1rk
|
|
||||||
=No/v
|
|
||||||
-----END PGP PUBLIC KEY BLOCK-----
|
|
||||||
```
|
|
||||||
+3
-13
@@ -1,11 +1,10 @@
|
|||||||
/syncthing
|
syncthing
|
||||||
/stdiscosrv
|
!gui/syncthing
|
||||||
|
!Godeps/_workspace/src/github.com/syncthing
|
||||||
syncthing.exe
|
syncthing.exe
|
||||||
stdiscosrv.exe
|
|
||||||
*.tar.gz
|
*.tar.gz
|
||||||
*.zip
|
*.zip
|
||||||
*.asc
|
*.asc
|
||||||
*.deb
|
|
||||||
.jshintrc
|
.jshintrc
|
||||||
coverage.out
|
coverage.out
|
||||||
files/pidx
|
files/pidx
|
||||||
@@ -15,12 +14,3 @@ coverage.xml
|
|||||||
syncthing.sig
|
syncthing.sig
|
||||||
RELEASE
|
RELEASE
|
||||||
deb
|
deb
|
||||||
lib/auto/gui.files.go
|
|
||||||
snapcraft.yaml
|
|
||||||
prime/
|
|
||||||
snap/
|
|
||||||
parts/
|
|
||||||
stage/
|
|
||||||
*.snap
|
|
||||||
*.bz2
|
|
||||||
/repos
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
linters-settings:
|
|
||||||
maligned:
|
|
||||||
suggest-new: true
|
|
||||||
|
|
||||||
linters:
|
|
||||||
enable-all: true
|
|
||||||
disable:
|
|
||||||
- goimports
|
|
||||||
- depguard
|
|
||||||
- lll
|
|
||||||
- gochecknoinits
|
|
||||||
- gochecknoglobals
|
|
||||||
- gofmt
|
|
||||||
- scopelint
|
|
||||||
- gocyclo
|
|
||||||
- funlen
|
|
||||||
- wsl
|
|
||||||
|
|
||||||
service:
|
|
||||||
golangci-lint-version: 1.21.x
|
|
||||||
prepare:
|
|
||||||
- rm -f go.sum # 1.12 -> 1.13 issues with QUIC-go
|
|
||||||
- GO111MODULE=on go mod vendor
|
|
||||||
- go run build.go assets
|
|
||||||
@@ -1,208 +1,81 @@
|
|||||||
# This is the official list of Syncthing authors for copyright purposes.
|
# This is the official list of Syncthing authors for copyright purposes.
|
||||||
#
|
|
||||||
# THIS FILE IS MOSTLY AUTO GENERATED. IF YOU'VE MADE A COMMIT TO THE
|
|
||||||
# REPOSITORY YOU WILL BE ADDED HERE AUTOMATICALLY WITHOUT THE NEED FOR
|
|
||||||
# ANY MANUAL ACTION.
|
|
||||||
#
|
|
||||||
# That said, you are welcome to correct your name or add a nickname / GitHub
|
|
||||||
# user name as appropriate. The format is:
|
|
||||||
#
|
|
||||||
# Name Name Name (nickname) <email1@example.com> <email2@example.com>
|
|
||||||
#
|
|
||||||
# The in-GUI authors list is periodically automatically updated from the
|
|
||||||
# contents of this file.
|
|
||||||
#
|
|
||||||
|
|
||||||
Aaron Bieber (qbit) <qbit@deftly.net>
|
Aaron Bieber <qbit@deftly.net>
|
||||||
Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com> <adam@proactiveservices.co.uk>
|
Adam Piggott <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com>
|
||||||
Adel Qalieh (adelq) <aqalieh95@gmail.com> <adelq@users.noreply.github.com>
|
Alessandro G. <alessandro.g89@gmail.com>
|
||||||
Alessandro G. (alessandro.g89) <alessandro.g89@gmail.com>
|
Alexander Graf <register-github@alex-graf.de>
|
||||||
Alexander Graf (alex2108) <register-github@alex-graf.de>
|
Anderson Mesquita <andersonvom@gmail.com>
|
||||||
Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
|
Andrew Dunham <andrew@du.nham.ca>
|
||||||
Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
|
Antony Male <antony.male@gmail.com>
|
||||||
andresvia <andres.via@gmail.com>
|
Arthur Axel fREW Schmidt <frew@afoolishmanifesto.com> <frioux@gmail.com>
|
||||||
Andrew Dunham (andrew-d) <andrew@du.nham.ca>
|
Audrius Butkevicius <audrius.butkevicius@gmail.com>
|
||||||
Andrew Rabert (nvllsvm) <ar@nullsum.net> <6550543+nvllsvm@users.noreply.github.com>
|
Bart De Vries <devriesb@gmail.com>
|
||||||
Andrey D (scienmind) <scintertech@cryptolab.net> <scienmind@users.noreply.github.com>
|
Ben Curthoys <ben@bencurthoys.com>
|
||||||
André Colomb (acolomb) <src@andre.colomb.de> <github.com@andre.colomb.de>
|
Ben Schulz <ueomkail@gmail.com> <uok@users.noreply.github.com>
|
||||||
andyleap <andyleap@gmail.com>
|
Ben Sidhom <bsidhom@gmail.com>
|
||||||
Antoine Lamielle (0x010C) <antoine.lamielle@0x010c.fr> <gh@0x010c.fr>
|
Brandon Philips <brandon@ifup.org>
|
||||||
Antony Male (canton7) <antony.male@gmail.com>
|
Brendan Long <self@brendanlong.com>
|
||||||
Aranjedeath <Aranjedeath@users.noreply.github.com>
|
Brian R. Becker <brbecker@gmail.com>
|
||||||
Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
|
Caleb Callaway <enlightened.despot@gmail.com>
|
||||||
Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com> <github@audrius.rocks>
|
Carsten Hagemann <moter8@gmail.com>
|
||||||
BAHADIR YILMAZ <bahadiryilmaz32@gmail.com>
|
Cathryne Linenweaver <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
|
||||||
Bart De Vries (mogwa1) <devriesb@gmail.com>
|
Chris Howie <me@chrishowie.com>
|
||||||
Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
|
Chris Joel <chris@scriptolo.gy>
|
||||||
Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
|
Colin Kennedy <moshen.colin@gmail.com>
|
||||||
Ben Shepherd (benshep) <bjashepherd@gmail.com>
|
Daniel Bergmann <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
|
||||||
Ben Sidhom (bsidhom) <bsidhom@gmail.com>
|
Daniel Martí <mvdan@mvdan.cc>
|
||||||
Benedikt Heine (bebehei) <bebe@bebehei.de>
|
Denis A. <denisva@gmail.com>
|
||||||
Benedikt Morbach <benedikt.morbach@googlemail.com>
|
Dennis Wilson <dw@risu.io>
|
||||||
Benno Fünfstück <benno.fuenfstueck@gmail.com>
|
Dominik Heidler <dominik@heidler.eu>
|
||||||
Benny Ng (tpng) <benny.tpng@gmail.com>
|
Elias Jarlebring <jarlebring@gmail.com>
|
||||||
Boris Rybalkin <ribalkin@gmail.com>
|
Emil Hessman <emil@hessman.se>
|
||||||
Brandon Philips (philips) <brandon@ifup.org>
|
Erik Meitner <e.meitner@willystreet.coop>
|
||||||
Brendan Long (brendanlong) <self@brendanlong.com>
|
Federico Castagnini <federico.castagnini@gmail.com>
|
||||||
Brian R. Becker (brbecker) <brbecker@gmail.com>
|
Felix Ableitner <me@nutomic.com>
|
||||||
Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
|
Felix Unterpaintner <bigbear2nd@gmail.com>
|
||||||
Carsten Hagemann (carstenhag) <moter8@gmail.com> <carsten@chagemann.de>
|
Francois-Xavier Gsell <fxgsell@gmail.com>
|
||||||
Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com> <katrinleinweber@MAC.local>
|
Frank Isemann <frank@isemann.name>
|
||||||
Cedric Staniewski (xduugu) <cedric@gmx.ca>
|
Gilli Sigurdsson <gilli@vx.is>
|
||||||
Chris Howie (cdhowie) <me@chrishowie.com>
|
Jaakko Hannikainen <jgke@jgke.fi>
|
||||||
Chris Joel (cdata) <chris@scriptolo.gy>
|
Jacek Szafarkiewicz <szafar@linux.pl>
|
||||||
Chris Tonkinson <chris@masterbran.ch>
|
Jake Peterson <jake@acogdev.com>
|
||||||
chucic <chucic@seznam.cz>
|
Jakob Borg <jakob@nym.se>
|
||||||
Colin Kennedy (moshen) <moshen.colin@gmail.com>
|
James Patterson <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
|
||||||
Cromefire_ <tim.l@nghorst.net>
|
Jaroslav Malec <dzardacz@gmail.com>
|
||||||
Dale Visser <dale.visser@live.com>
|
Jens Diemer <github.com@jensdiemer.de> <git@jensdiemer.de>
|
||||||
Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
|
Jochen Voss <voss@seehuhn.de>
|
||||||
Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
|
Johan Vromans <jvromans@squirrel.nl>
|
||||||
Daniel Martí (mvdan) <mvdan@mvdan.cc>
|
Karol Różycki <rozycki.karol@gmail.com>
|
||||||
Darshil Chanpura (dtchanpura) <dtchanpura@gmail.com> <dcprime314@gmail.com>
|
Ken'ichi Kamada <kamada@nanohz.org>
|
||||||
David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
|
Kevin Allen <kma1660@gmail.com>
|
||||||
Denis A. (dva) <denisva@gmail.com>
|
Lode Hoste <zillode@zillode.be>
|
||||||
Dennis Wilson (snnd) <dw@risu.io>
|
Lord Landon Agahnim <lordlandon@gmail.com>
|
||||||
dependabot-preview[bot] <dependabot-preview[bot]@users.noreply.github.com>
|
Marc Laporte <marc@marclaporte.com> <marc@laporte.name>
|
||||||
dependabot[bot] <dependabot[bot]@users.noreply.github.com>
|
Marc Pujol <kilburn@la3.org>
|
||||||
derekriemer <derek.riemer@colorado.edu>
|
Marcin Dziadus <dziadus.marcin@gmail.com>
|
||||||
desbma <desbma@users.noreply.github.com>
|
Mateusz Naściszewski <matin1111@wp.pl>
|
||||||
Dmitry Saveliev (dsaveliev) <d.e.saveliev@gmail.com>
|
Matt Burke <mburke@amplify.com> <burkemw3@gmail.com>
|
||||||
Dominik Heidler (asdil12) <dominik@heidler.eu>
|
Michael Jephcote <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
|
||||||
Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
|
Michael Ploujnikov <ploujj@gmail.com>
|
||||||
Elliot Huffman <thelich2@gmail.com>
|
Michael Tilli <pyfisch@gmail.com>
|
||||||
Emil Hessman (ceh) <emil@hessman.se>
|
Nate Morrison <natemorrison@gmail.com>
|
||||||
Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
|
Pascal Jungblut <github@pascalj.com> <mail@pascal-jungblut.com>
|
||||||
Evgeny Kuznetsov <evgeny@kuznetsov.md>
|
Peter Hoeg <peter@speartail.com>
|
||||||
Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
|
Philippe Schommers <philippe@schommers.be>
|
||||||
Felix Ableitner (Nutomic) <me@nutomic.com>
|
Phill Luby <phill.luby@newredo.com>
|
||||||
Felix Unterpaintner (bigbear2nd) <bigbear2nd@gmail.com>
|
Piotr Bejda <piotrb10@gmail.com>
|
||||||
Francois-Xavier Gsell (zukoo) <fxgsell@gmail.com>
|
Ryan Sullivan <kayoticsully@gmail.com>
|
||||||
Frank Isemann (fti7) <frank@isemann.name>
|
Scott Klupfel <kluppy@going2blue.com>
|
||||||
georgespatton <georgespatton@users.noreply.github.com>
|
Sergey Mishin <ralder@yandex.ru>
|
||||||
Gilli Sigurdsson (gillisig) <gilli@vx.is>
|
Stefan Kuntz <stefan.github@gmail.com> <Stefan.github@gmail.com>
|
||||||
Graham Miln (grahammiln) <graham.miln@dssw.co.uk> <graham.miln@miln.eu>
|
Stefan Tatschner <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
|
||||||
Han Boetes <han@boetes.org>
|
Tim Abell <tim@timwise.co.uk>
|
||||||
Harrison Jones (harrisonhjones) <harrisonhjones@users.noreply.github.com>
|
Tobias Nygren <tnn@nygren.pp.se>
|
||||||
Heiko Zuerker (Smiley73) <heiko@zuerker.org>
|
Tomas Cerveny <kozec@kozec.com>
|
||||||
Hugo Locurcio <hugo.locurcio@hugo.pro>
|
Tully Robinson <tully@tojr.org>
|
||||||
Iain Barnett <iainspeed@gmail.com>
|
Tyler Brazier <tyler@tylerbrazier.com>
|
||||||
Ian Johnson (anonymouse64) <ian.johnson@canonical.com> <person.uwsome@gmail.com>
|
Veeti Paananen <veeti.paananen@rojekti.fi>
|
||||||
Iskander Sharipov (Alex) <quasilyte@gmail.com>
|
Victor Buinsky <vix_booja@tut.by>
|
||||||
Jaakko Hannikainen (jgke) <jgke@jgke.fi>
|
Vil Brekin <vilbrekin@gmail.com>
|
||||||
Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
|
William A. Kennington III <william@wkennington.com>
|
||||||
Jake Peterson (acogdev) <jake@acogdev.com>
|
Yannic A. <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>
|
||||||
Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net>
|
|
||||||
James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
|
|
||||||
janost <janost@tuta.io>
|
|
||||||
Jaroslav Malec (dzarda) <dzardacz@gmail.com>
|
|
||||||
jaseg <githubaccount@jaseg.net>
|
|
||||||
Jaya Chithra (jayachithra) <s.k.jayachithra@gmail.com>
|
|
||||||
Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
|
|
||||||
Jerry Jacobs (xor-gate) <jerry.jacobs@xor-gate.org> <xor-gate@users.noreply.github.com>
|
|
||||||
Jochen Voss (seehuhn) <voss@seehuhn.de>
|
|
||||||
Johan Andersson <j@i19.se>
|
|
||||||
Johan Vromans (sciurius) <jvromans@squirrel.nl>
|
|
||||||
John Rinehart (fuzzybear3965) <johnrichardrinehart@gmail.com>
|
|
||||||
Jonas Thelemann <e-mail@jonas-thelemann.de>
|
|
||||||
Jonathan Cross <jcross@gmail.com>
|
|
||||||
Jose Manuel Delicado (jmdaweb) <jmdaweb@hotmail.com> <jmdaweb@users.noreply.github.com>
|
|
||||||
Jörg Thalheim <Mic92@users.noreply.github.com>
|
|
||||||
Kalle Laine <pahakalle@protonmail.com>
|
|
||||||
Karol Różycki (krozycki) <rozycki.karol@gmail.com>
|
|
||||||
Keith Turner <kturner@apache.org>
|
|
||||||
Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
|
|
||||||
Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
|
|
||||||
Kevin Allen (ironmig) <kma1660@gmail.com>
|
|
||||||
Kevin White, Jr. (kwhite17) <kevinwhite1710@gmail.com>
|
|
||||||
klemens <ka7@github.com>
|
|
||||||
Kurt Fitzner (Kudalufi) <kurt@va1der.ca> <kurt.fitzner@gmail.com>
|
|
||||||
Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
|
|
||||||
Laurent Arnoud <laurent@spkdev.net>
|
|
||||||
Laurent Etiemble (letiemble) <laurent.etiemble@gmail.com> <laurent.etiemble@monobjc.net>
|
|
||||||
Leo Arias (elopio) <yo@elopio.net>
|
|
||||||
Liu Siyuan (liusy182) <liusy182@gmail.com> <liusy182@hotmail.com>
|
|
||||||
Lode Hoste (Zillode) <zillode@zillode.be>
|
|
||||||
Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
|
|
||||||
Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
|
|
||||||
Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
|
|
||||||
Marc Pujol (kilburn) <kilburn@la3.org>
|
|
||||||
Marcin Dziadus (marcindziadus) <dziadus.marcin@gmail.com>
|
|
||||||
marco-m <marco.molteni@laposte.net>
|
|
||||||
Mark Pulford (mpx) <mark@kyne.com.au>
|
|
||||||
Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
|
|
||||||
Matic Potočnik <hairyfotr@gmail.com>
|
|
||||||
Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
|
|
||||||
Matt Robenolt <matt@ydekproductions.com>
|
|
||||||
Matteo Ruina <matteo.ruina@gmail.com>
|
|
||||||
Maurizio Tomasi <ziotom78@gmail.com>
|
|
||||||
Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
|
|
||||||
MaximAL <almaximal@ya.ru>
|
|
||||||
Maxime Thirouin <m@moox.io>
|
|
||||||
Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
|
|
||||||
Michael Ploujnikov (plouj) <ploujj@gmail.com>
|
|
||||||
Michael Tilli (pyfisch) <pyfisch@gmail.com>
|
|
||||||
Mike Boone <mike@boonedocks.net>
|
|
||||||
MikeLund <MikeLund@users.noreply.github.com>
|
|
||||||
Mingxuan Lin <gdlmx@users.noreply.github.com>
|
|
||||||
Nate Morrison (nrm21) <natemorrison@gmail.com>
|
|
||||||
Nicholas Rishel (PrototypeNM1) <rishel.nick@gmail.com> <PrototypeNM1@users.noreply.github.com>
|
|
||||||
Nico Stapelbroek <3368018+nstapelbroek@users.noreply.github.com>
|
|
||||||
Nicolas Braud-Santoni <nicolas@braud-santoni.eu>
|
|
||||||
Niels Peter Roest (Niller303) <nielsproest@hotmail.com> <seje.niels@hotmail.com>
|
|
||||||
Nils Jakobi (thunderstorm99) <jakobi.nils@gmail.com>
|
|
||||||
Nitroretro <43112364+Nitroretro@users.noreply.github.com>
|
|
||||||
NoLooseEnds <jon.koslung@gmail.com>
|
|
||||||
otbutz <tbutz@optitool.de>
|
|
||||||
Oyebanji Jacob Mayowa <oyebanji05@gmail.com>
|
|
||||||
Pascal Jungblut (pascalj) <github@pascalj.com> <mail@pascal-jungblut.com>
|
|
||||||
Pawel Palenica (qepasa) <pawelpalenica11@gmail.com>
|
|
||||||
Paweł Rozlach <vespian@users.noreply.github.com>
|
|
||||||
perewa <cavalcante.ten@gmail.com>
|
|
||||||
Peter Badida <KeyWeeUsr@users.noreply.github.com>
|
|
||||||
Peter Dave Hello <hsu@peterdavehello.org>
|
|
||||||
Peter Hoeg (peterhoeg) <peter@speartail.com>
|
|
||||||
Peter Marquardt (wwwutz) <wwwutz@gmail.com> <wwwutz@googlemail.com>
|
|
||||||
Phil Davis <phil.davis@inf.org>
|
|
||||||
Philippe Schommers (filoozoom) <philippe@schommers.be>
|
|
||||||
Phill Luby (pluby) <phill.luby@newredo.com>
|
|
||||||
Pier Paolo Ramon <ramonpierre@gmail.com>
|
|
||||||
Piotr Bejda (piobpl) <piotrb10@gmail.com>
|
|
||||||
Pramodh KP (pramodhkp) <pramodh.p@directi.com> <1507241+pramodhkp@users.noreply.github.com>
|
|
||||||
Richard Hartmann <RichiH@users.noreply.github.com>
|
|
||||||
Robert Carosi (nov1n) <robert@carosi.nl>
|
|
||||||
Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
|
|
||||||
Ross Smith II (rasa) <ross@smithii.com>
|
|
||||||
rubenbe <github-com-00ff86@vandamme.email>
|
|
||||||
Ryan Sullivan (KayoticSully) <kayoticsully@gmail.com>
|
|
||||||
Sacheendra Talluri (sacheendra) <sacheendra.t@gmail.com>
|
|
||||||
Scott Klupfel (kluppy) <kluppy@going2blue.com>
|
|
||||||
Sergey Mishin (ralder) <ralder@yandex.ru>
|
|
||||||
Simon Frei (imsodin) <freisim93@gmail.com>
|
|
||||||
Sly_tom_cat <slytomcat@mail.ru>
|
|
||||||
Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
|
|
||||||
Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org> <stefan@rumpelsepp.org>
|
|
||||||
Suhas Gundimeda (snugghash) <suhas.gundimeda@gmail.com> <snugghash@gmail.com>
|
|
||||||
Taylor Khan (nelsonkhan) <nelsonkhan@gmail.com>
|
|
||||||
Thomas Hipp <thomashipp@gmail.com>
|
|
||||||
Tim Abell (timabell) <tim@timwise.co.uk>
|
|
||||||
Tim Howes (timhowes) <timhowes@berkeley.edu>
|
|
||||||
Tobias Nygren (tnn2) <tnn@nygren.pp.se>
|
|
||||||
Tobias Tom (tobiastom) <t.tom@succont.de>
|
|
||||||
Tom Jakubowski <tom@crystae.net>
|
|
||||||
Tomas Cerveny (kozec) <kozec@kozec.com>
|
|
||||||
Tommy Thorn <tommy-github-email@thorn.ws>
|
|
||||||
Tully Robinson (tojrobinson) <tully@tojr.org>
|
|
||||||
Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
|
|
||||||
Unrud (Unrud) <unrud@openaliasbox.org> <Unrud@users.noreply.github.com>
|
|
||||||
Veeti Paananen (veeti) <veeti.paananen@rojekti.fi>
|
|
||||||
Victor Buinsky (buinsky) <vix_booja@tut.by>
|
|
||||||
Vil Brekin (Vilbrekin) <vilbrekin@gmail.com>
|
|
||||||
Vladimir Rusinov <vrusinov@google.com>
|
|
||||||
wangguoliang <liangcszzu@163.com>
|
|
||||||
William A. Kennington III (wkennington) <william@wkennington.com>
|
|
||||||
Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de> <wulf@weich-kr.de>
|
|
||||||
Xavier O. (damajor) <damajor@gmail.com>
|
|
||||||
xjtdy888 (xjtdy888) <xjtdy888@163.com>
|
|
||||||
Yannic A. (eipiminus1) <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>
|
|
||||||
佛跳墙 <daoquan@qq.com>
|
|
||||||
+74
-55
@@ -1,73 +1,92 @@
|
|||||||
# Contributor Covenant Code of Conduct
|
## Conduct
|
||||||
|
|
||||||
## Our Pledge
|
* We are committed to providing a friendly, safe and welcoming
|
||||||
|
environment for all, regardless of gender, sexual orientation,
|
||||||
|
disability, ethnicity, religion, or similar personal characteristic.
|
||||||
|
|
||||||
In the interest of fostering an open and welcoming environment, we as
|
* On IRC, please avoid using overtly sexual nicknames or other nicknames
|
||||||
contributors and maintainers pledge to making participation in our project and
|
that might detract from a friendly, safe and welcoming environment for
|
||||||
our community a harassment-free experience for everyone, regardless of age, body
|
all.
|
||||||
size, disability, ethnicity, gender identity and expression, level of experience,
|
|
||||||
education, socio-economic status, nationality, personal appearance, race,
|
|
||||||
religion, or sexual identity and orientation.
|
|
||||||
|
|
||||||
## Our Standards
|
* Please be kind and courteous. There's no need to be mean or rude.
|
||||||
|
|
||||||
Examples of behavior that contributes to creating a positive environment
|
* Respect that people have differences of opinion and that every design
|
||||||
include:
|
or implementation choice carries a trade-off and numerous costs. There
|
||||||
|
is seldom a right answer.
|
||||||
|
|
||||||
* Using welcoming and inclusive language
|
* Please keep unstructured critique to a minimum. If you have solid
|
||||||
* Being respectful of differing viewpoints and experiences
|
ideas you want to experiment with, make a fork and see how it works.
|
||||||
* Gracefully accepting constructive criticism
|
|
||||||
* Focusing on what is best for the community
|
|
||||||
* Showing empathy towards other community members
|
|
||||||
|
|
||||||
Examples of unacceptable behavior by participants include:
|
* We will exclude you from interaction if you insult, demean or harass
|
||||||
|
anyone. That is not welcome behaviour. We interpret the term
|
||||||
|
"harassment" as including the definition in the <a
|
||||||
|
href="http://citizencodeofconduct.org/">Citizen Code of Conduct</a>;
|
||||||
|
if you have any lack of clarity about what might be included in that
|
||||||
|
concept, please read their definition. In particular, we don't
|
||||||
|
tolerate behavior that excludes people in socially marginalized
|
||||||
|
groups.
|
||||||
|
|
||||||
* The use of sexualized language or imagery and unwelcome sexual attention or
|
* Private harassment is also unacceptable. No matter who you are, if you
|
||||||
advances
|
feel you have been or are being harassed or made uncomfortable by a
|
||||||
* Trolling, insulting/derogatory comments, and personal or political attacks
|
community member, please contact one of the channel ops or any of the
|
||||||
* Public or private harassment
|
Syncthing core team immediately. Whether you're a regular contributor
|
||||||
* Publishing others' private information, such as a physical or electronic
|
or a newcomer, we care about making this community a safe place for
|
||||||
address, without explicit permission
|
you and we've got your back.
|
||||||
* Other conduct which could reasonably be considered inappropriate in a
|
|
||||||
professional setting
|
|
||||||
|
|
||||||
## Our Responsibilities
|
* Likewise any spamming, trolling, flaming, baiting or other
|
||||||
|
attention-stealing behaviour is not welcome.
|
||||||
|
|
||||||
Project maintainers are responsible for clarifying the standards of acceptable
|
## Moderation
|
||||||
behavior and are expected to take appropriate and fair corrective action in
|
|
||||||
response to any instances of unacceptable behavior.
|
|
||||||
|
|
||||||
Project maintainers have the right and responsibility to remove, edit, or
|
These are the policies for upholding our community's standards of
|
||||||
reject comments, commits, code, wiki edits, issues, and other contributions
|
conduct in our communication channels, most notably in Syncthing-related
|
||||||
that are not aligned to this Code of Conduct, or to ban temporarily or
|
IRC channels and on the web forum.
|
||||||
permanently any contributor for other behaviors that they deem inappropriate,
|
|
||||||
threatening, offensive, or harmful.
|
|
||||||
|
|
||||||
## Scope
|
1. Remarks that violate the Syncthing standards of conduct, including
|
||||||
|
hateful, hurtful, oppressive, or exclusionary remarks, are not
|
||||||
|
allowed. (Cursing is allowed, but never targeting another user, and
|
||||||
|
never in a hateful manner.)
|
||||||
|
|
||||||
This Code of Conduct applies both within project spaces and in public spaces
|
2. Remarks that moderators find inappropriate, whether listed in the
|
||||||
when an individual is representing the project or its community. Examples of
|
code of conduct or not, are also not allowed.
|
||||||
representing a project or community include using an official project e-mail
|
|
||||||
address, posting via an official social media account, or acting as an appointed
|
|
||||||
representative at an online or offline event. Representation of a project may be
|
|
||||||
further defined and clarified by project maintainers.
|
|
||||||
|
|
||||||
## Enforcement
|
3. Moderators will first respond to such remarks with a warning.
|
||||||
|
|
||||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be
|
4. If the warning is unheeded, the user will be "kicked," i.e., kicked
|
||||||
reported by contacting the project team at security@syncthing.net. All
|
out of the communication channel to cool off.
|
||||||
complaints will be reviewed and investigated and will result in a response that
|
|
||||||
is deemed necessary and appropriate to the circumstances. The project team is
|
|
||||||
obligated to maintain confidentiality with regard to the reporter of an incident.
|
|
||||||
Further details of specific enforcement policies may be posted separately.
|
|
||||||
|
|
||||||
Project maintainers who do not follow or enforce the Code of Conduct in good
|
5. If the user comes back and continues to make trouble, they will be
|
||||||
faith may face temporary or permanent repercussions as determined by other
|
banned, i.e., indefinitely excluded.
|
||||||
members of the project's leadership.
|
|
||||||
|
|
||||||
## Attribution
|
6. Moderators may choose at their discretion to un-ban the user if it
|
||||||
|
was a first offense and they offer the offended party a genuine
|
||||||
|
apology.
|
||||||
|
|
||||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
|
7. If a moderator bans someone and you think it was unjustified, please
|
||||||
available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
|
take it up with that moderator, or with a different moderator, **in
|
||||||
|
private**. Complaints about bans in-channel are not allowed.
|
||||||
|
|
||||||
[homepage]: https://www.contributor-covenant.org
|
8. Moderators are held to a higher standard than other community
|
||||||
|
members. If a moderator creates an inappropriate situation, they
|
||||||
|
should expect less leeway than others.
|
||||||
|
|
||||||
|
In the Syncthing community we strive to go the extra step to look out
|
||||||
|
for each other. Don't just aim to be technically unimpeachable, try to
|
||||||
|
be your best self. In particular, avoid flirting with offensive or
|
||||||
|
sensitive issues, particularly if they're off-topic; this all too
|
||||||
|
often leads to unnecessary fights, hurt feelings, and damaged trust;
|
||||||
|
worse, it can drive people away from the community entirely.
|
||||||
|
|
||||||
|
And if someone takes issue with something you said or did, resist the
|
||||||
|
urge to be defensive. Just stop doing what it was they complained about
|
||||||
|
and apologize. Even if you feel you were misinterpreted or unfairly
|
||||||
|
accused, chances are good there was something you could've communicated
|
||||||
|
better — remember that it's your responsibility to make your fellow
|
||||||
|
community members comfortable. Everyone wants to get along and we are
|
||||||
|
all here first and foremost because we want to talk about cool
|
||||||
|
technology. You will find that people will be eager to assume good
|
||||||
|
intent and forgive as long as you earn their trust.
|
||||||
|
|
||||||
|
*Adapted from the [Rust Code of Conduct](https://github.com/rust-lang/rust/wiki/Note-development-policy#conduct)*
|
||||||
|
|
||||||
|
*Adapted from the [Node.js Policy on Trolling](http://blog.izs.me/post/30036893703/policy-on-trolling)*
|
||||||
+7
-18
@@ -33,31 +33,20 @@ latest info on Transifex.
|
|||||||
|
|
||||||
Every contribution is welcome. If you want to contribute but are unsure
|
Every contribution is welcome. If you want to contribute but are unsure
|
||||||
where to start, any open issues are fair game! See the [Contribution
|
where to start, any open issues are fair game! See the [Contribution
|
||||||
Guidelines](https://docs.syncthing.net/dev/contributing.html) for the full
|
Guidelines](http://docs.syncthing.net/dev/contributing.html) for the full
|
||||||
story on committing code.
|
story on committing code.
|
||||||
|
|
||||||
## Contributing Documentation
|
## Contributing Documentation
|
||||||
|
|
||||||
Updates to the [documentation site](https://docs.syncthing.net/) can be
|
Updates to the [documentation site](http://docs.syncthing.net/) can be
|
||||||
made as pull requests on the [documentation
|
made as pull requests on the [documentation
|
||||||
repository](https://github.com/syncthing/docs).
|
repository](https://github.com/syncthing/docs).
|
||||||
|
|
||||||
## Licensing
|
## Licensing
|
||||||
|
|
||||||
All contributions are made available under the same license as the already
|
All contributions are made under the same MPLv2 license as the rest of
|
||||||
existing material being contributed to. For most of the project and unless
|
the project, except documentation, user interface text and translation
|
||||||
otherwise stated this means MPLv2, but there are exceptions:
|
strings which are licensed under the Creative Commons Attribution 4.0
|
||||||
|
International License. You retain the copyright to code you have
|
||||||
- Certain commands (under cmd/...) may have a separate license, indicated by
|
written.
|
||||||
the presence of a LICENSE file in the corresponding directory.
|
|
||||||
|
|
||||||
- The documentation (man/...) is licensed under the Creative Commons
|
|
||||||
Attribution 4.0 International License.
|
|
||||||
|
|
||||||
- Projects under vendor/... are copyright by and licensed from their
|
|
||||||
respective original authors. Contributions should be made to the original
|
|
||||||
project, not here.
|
|
||||||
|
|
||||||
Regardless of the license in effect, you retain the copyright to your
|
|
||||||
contribution.
|
|
||||||
|
|
||||||
-28
@@ -1,28 +0,0 @@
|
|||||||
FROM golang:1.13 AS builder
|
|
||||||
|
|
||||||
WORKDIR /src
|
|
||||||
COPY . .
|
|
||||||
|
|
||||||
ENV CGO_ENABLED=0
|
|
||||||
ENV BUILD_HOST=syncthing.net
|
|
||||||
ENV BUILD_USER=docker
|
|
||||||
RUN rm -f syncthing && go run build.go -no-upgrade build syncthing
|
|
||||||
|
|
||||||
FROM alpine
|
|
||||||
|
|
||||||
EXPOSE 8384 22000 21027/udp
|
|
||||||
|
|
||||||
VOLUME ["/var/syncthing"]
|
|
||||||
|
|
||||||
RUN apk add --no-cache ca-certificates su-exec
|
|
||||||
|
|
||||||
COPY --from=builder /src/syncthing /bin/syncthing
|
|
||||||
COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
|
|
||||||
|
|
||||||
ENV PUID=1000 PGID=1000 HOME=/var/syncthing
|
|
||||||
|
|
||||||
HEALTHCHECK --interval=1m --timeout=10s \
|
|
||||||
CMD nc -z localhost 8384 || exit 1
|
|
||||||
|
|
||||||
ENV STGUIADDRESS=0.0.0.0:8384
|
|
||||||
ENTRYPOINT ["/bin/entrypoint.sh", "/bin/syncthing", "-home", "/var/syncthing/config"]
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
FROM golang:1.13 AS builder
|
|
||||||
|
|
||||||
WORKDIR /src
|
|
||||||
COPY . .
|
|
||||||
|
|
||||||
ENV CGO_ENABLED=0
|
|
||||||
ENV BUILD_HOST=syncthing.net
|
|
||||||
ENV BUILD_USER=docker
|
|
||||||
RUN rm -f stdiscosrv && go run build.go -no-upgrade build stdiscosrv
|
|
||||||
|
|
||||||
FROM alpine
|
|
||||||
|
|
||||||
EXPOSE 19200 8443
|
|
||||||
|
|
||||||
VOLUME ["/var/stdiscosrv"]
|
|
||||||
|
|
||||||
RUN apk add --no-cache ca-certificates su-exec
|
|
||||||
|
|
||||||
COPY --from=builder /src/stdiscosrv /bin/stdiscosrv
|
|
||||||
COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
|
|
||||||
|
|
||||||
ENV PUID=1000 PGID=1000 HOME=/var/stdiscosrv
|
|
||||||
|
|
||||||
HEALTHCHECK --interval=1m --timeout=10s \
|
|
||||||
CMD nc -z localhost 8443 || exit 1
|
|
||||||
|
|
||||||
WORKDIR /var/stdiscosrv
|
|
||||||
ENTRYPOINT ["/bin/entrypoint.sh", "/bin/stdiscosrv"]
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
FROM golang:1.13 AS builder
|
|
||||||
|
|
||||||
WORKDIR /src
|
|
||||||
COPY . .
|
|
||||||
|
|
||||||
ENV CGO_ENABLED=0
|
|
||||||
ENV BUILD_HOST=syncthing.net
|
|
||||||
ENV BUILD_USER=docker
|
|
||||||
RUN rm -f strelaysrv && go run build.go -no-upgrade build strelaysrv
|
|
||||||
|
|
||||||
FROM alpine
|
|
||||||
|
|
||||||
EXPOSE 22067 22070
|
|
||||||
|
|
||||||
VOLUME ["/var/strelaysrv"]
|
|
||||||
|
|
||||||
RUN apk add --no-cache ca-certificates su-exec
|
|
||||||
|
|
||||||
COPY --from=builder /src/strelaysrv /bin/strelaysrv
|
|
||||||
COPY --from=builder /src/script/docker-entrypoint.sh /bin/entrypoint.sh
|
|
||||||
|
|
||||||
ENV PUID=1000 PGID=1000 HOME=/var/strelaysrv
|
|
||||||
|
|
||||||
HEALTHCHECK --interval=1m --timeout=10s \
|
|
||||||
CMD nc -z localhost 22067 || exit 1
|
|
||||||
|
|
||||||
WORKDIR /var/strelaysrv
|
|
||||||
ENTRYPOINT ["/bin/entrypoint.sh", "/bin/strelaysrv"]
|
|
||||||
@@ -1,83 +0,0 @@
|
|||||||
# The Syncthing Goals
|
|
||||||
|
|
||||||
Syncthing is a **continuous file synchronization program**. It synchronizes
|
|
||||||
files between two or more computers. We strive to fulfill the goals below.
|
|
||||||
The goals are listed in order of importance, the most important one being
|
|
||||||
the first.
|
|
||||||
|
|
||||||
> "Syncing files" here is precise. It means we specifically exclude things
|
|
||||||
> that are not files - calendar items, instant messages, and so on. If those
|
|
||||||
> are in fact stored as files on disk, they can of course be synced as
|
|
||||||
> files.
|
|
||||||
|
|
||||||
Syncthing should be:
|
|
||||||
|
|
||||||
### 1. Safe From Data Loss
|
|
||||||
|
|
||||||
Protecting the user's data is paramount. We take every reasonable precaution
|
|
||||||
to avoid corrupting the user's files.
|
|
||||||
|
|
||||||
> This is the overriding goal, without which synchronizing files becomes
|
|
||||||
> pointless. This means that we do not make unsafe trade offs for the sake
|
|
||||||
> of performance or, in some cases, even usability.
|
|
||||||
|
|
||||||
### 2. Secure Against Attackers
|
|
||||||
|
|
||||||
Again, protecting the user's data is paramount. Regardless of our other
|
|
||||||
goals we must never allow the user's data to be susceptible to eavesdropping
|
|
||||||
or modification by unauthorized parties.
|
|
||||||
|
|
||||||
> This should be understood in context. It is not necessarily reasonable to
|
|
||||||
> expect Syncthing to be resistant against well equipped state level
|
|
||||||
> attackers. We will however do our best. Note also that this is different
|
|
||||||
> from anonymity which is not, currently, a goal.
|
|
||||||
|
|
||||||
### 3. Easy to Use
|
|
||||||
|
|
||||||
Syncthing should be approachable, understandable and inclusive.
|
|
||||||
|
|
||||||
> Complex concepts and maths form the base of Syncthing's functionality.
|
|
||||||
> This should nonetheless be abstracted or hidden to a degree where
|
|
||||||
> Syncthing is usable by the general public.
|
|
||||||
|
|
||||||
### 4. Automatic
|
|
||||||
|
|
||||||
User interaction should be required only when absolutely necessary.
|
|
||||||
|
|
||||||
> Specifically this means that changes to files are picked up without
|
|
||||||
> prompting, conflicts are resolved without prompting and connections are
|
|
||||||
> maintained without prompting. We only prompt the user when it is required
|
|
||||||
> to fulfill one of the (overriding) Secure, Safe or Easy goals.
|
|
||||||
|
|
||||||
### 5. Universally Available
|
|
||||||
|
|
||||||
Syncthing should run on every common computer. We are mindful that the
|
|
||||||
latest technology is not always available to any given individual.
|
|
||||||
|
|
||||||
> Computers include desktops, laptops, servers, virtual machines, small
|
|
||||||
> general purpose computers such as Raspberry Pis and, *where possible*,
|
|
||||||
> tablets and phones. NAS appliances, toasters, cars, firearms, thermostats
|
|
||||||
> and so on may include computing capabitilies but it is not our goal for
|
|
||||||
> Syncthing to run smoothly on these devices.
|
|
||||||
|
|
||||||
### 6. For Individuals
|
|
||||||
|
|
||||||
Syncthing is primarily about empowering the individual user with safe,
|
|
||||||
secure and easy to use file synchronization.
|
|
||||||
|
|
||||||
> We acknowledge that it's also useful in an enterprise setting and include
|
|
||||||
> functionality to support that. If this is in conflict with the
|
|
||||||
> requirements of the individual, those will however take priority.
|
|
||||||
|
|
||||||
### 7. Everything Else
|
|
||||||
|
|
||||||
There are many things we care about that don't make it on to the list. It is
|
|
||||||
fine to optimize for these values as well, as long as they are not in
|
|
||||||
conflict with the stated goals above.
|
|
||||||
|
|
||||||
> For example, performance is a thing we care about. We just don't care more
|
|
||||||
> about it than safety, security, etc. Maintainability of the code base and
|
|
||||||
> providing entertainment value for the maintainers are also things that
|
|
||||||
> matter. It is understood that there are aspects of Syncthing that are
|
|
||||||
> suboptimal or even in opposition with the goals above. However, we
|
|
||||||
> continuously strive to align Syncthing more and more with these goals.
|
|
||||||
Generated
+90
@@ -0,0 +1,90 @@
|
|||||||
|
{
|
||||||
|
"ImportPath": "github.com/syncthing/syncthing",
|
||||||
|
"GoVersion": "go1.5.2",
|
||||||
|
"Packages": [
|
||||||
|
"./cmd/..."
|
||||||
|
],
|
||||||
|
"Deps": [
|
||||||
|
{
|
||||||
|
"ImportPath": "github.com/bkaradzic/go-lz4",
|
||||||
|
"Rev": "74ddf82598bc4745b965729e9c6a463bedd33049"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ImportPath": "github.com/calmh/du",
|
||||||
|
"Rev": "3c0690cca16228b97741327b1b6781397afbdb24"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ImportPath": "github.com/calmh/luhn",
|
||||||
|
"Rev": "0c8388ff95fa92d4094011e5a04fc99dea3d1632"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ImportPath": "github.com/calmh/xdr",
|
||||||
|
"Rev": "9eb3e1a622d9364deb39c831f7e5f164393d7e37"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ImportPath": "github.com/golang/snappy",
|
||||||
|
"Rev": "723cc1e459b8eea2dea4583200fd60757d40097a"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ImportPath": "github.com/juju/ratelimit",
|
||||||
|
"Rev": "772f5c38e468398c4511514f4f6aa9a4185bc0a0"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ImportPath": "github.com/kardianos/osext",
|
||||||
|
"Rev": "29ae4ffbc9a6fe9fb2bc5029050ce6996ea1d3bc"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ImportPath": "github.com/rcrowley/go-metrics",
|
||||||
|
"Rev": "1ce93efbc8f9c568886b2ef85ce305b2217b3de3"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ImportPath": "github.com/syndtr/goleveldb/leveldb",
|
||||||
|
"Rev": "1a9d62f03ea92815b46fcaab357cfd4df264b1a0"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ImportPath": "github.com/thejerf/suture",
|
||||||
|
"Comment": "v1.0.1",
|
||||||
|
"Rev": "99c1f2d613756768fc4299acd9dc621e11ed3fd7"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ImportPath": "github.com/vitrun/qart/coding",
|
||||||
|
"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ImportPath": "github.com/vitrun/qart/gf256",
|
||||||
|
"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ImportPath": "github.com/vitrun/qart/qr",
|
||||||
|
"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ImportPath": "golang.org/x/crypto/bcrypt",
|
||||||
|
"Rev": "575fdbe86e5dd89229707ebec0575ce7d088a4a6"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ImportPath": "golang.org/x/crypto/blowfish",
|
||||||
|
"Rev": "575fdbe86e5dd89229707ebec0575ce7d088a4a6"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ImportPath": "golang.org/x/net/internal/iana",
|
||||||
|
"Rev": "042ba42fa6633b34205efc66ba5719cd3afd8d38"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ImportPath": "golang.org/x/net/ipv6",
|
||||||
|
"Rev": "042ba42fa6633b34205efc66ba5719cd3afd8d38"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ImportPath": "golang.org/x/net/proxy",
|
||||||
|
"Rev": "042ba42fa6633b34205efc66ba5719cd3afd8d38"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ImportPath": "golang.org/x/text/transform",
|
||||||
|
"Rev": "5eb8d4684c4796dd36c74f6452f2c0fa6c79597e"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"ImportPath": "golang.org/x/text/unicode/norm",
|
||||||
|
"Rev": "5eb8d4684c4796dd36c74f6452f2c0fa6c79597e"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
Generated
+5
@@ -0,0 +1,5 @@
|
|||||||
|
This directory tree is generated automatically by godep.
|
||||||
|
|
||||||
|
Please do not edit.
|
||||||
|
|
||||||
|
See https://github.com/tools/godep for more information.
|
||||||
Generated
+2
@@ -0,0 +1,2 @@
|
|||||||
|
/pkg
|
||||||
|
/bin
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
/lz4-example/lz4-example
|
||||||
+9
@@ -0,0 +1,9 @@
|
|||||||
|
language: go
|
||||||
|
|
||||||
|
go:
|
||||||
|
- 1.1
|
||||||
|
- 1.2
|
||||||
|
- 1.3
|
||||||
|
- 1.4
|
||||||
|
- 1.5
|
||||||
|
- tip
|
||||||
+24
@@ -0,0 +1,24 @@
|
|||||||
|
Copyright 2011-2012 Branimir Karadzic. All rights reserved.
|
||||||
|
Copyright 2013 Damian Gryski. All rights reserved.
|
||||||
|
|
||||||
|
Redistribution and use in source and binary forms, with or without modification,
|
||||||
|
are permitted provided that the following conditions are met:
|
||||||
|
|
||||||
|
1. Redistributions of source code must retain the above copyright notice, this
|
||||||
|
list of conditions and the following disclaimer.
|
||||||
|
|
||||||
|
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||||
|
this list of conditions and the following disclaimer in the documentation
|
||||||
|
and/or other materials provided with the distribution.
|
||||||
|
|
||||||
|
THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
|
||||||
|
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||||
|
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
|
||||||
|
SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||||
|
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||||
|
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
||||||
|
WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
||||||
|
OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
|
||||||
|
THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
|
||||||
+71
@@ -0,0 +1,71 @@
|
|||||||
|
go-lz4
|
||||||
|
======
|
||||||
|
|
||||||
|
go-lz4 is port of LZ4 lossless compression algorithm to Go. The original C code
|
||||||
|
is located at:
|
||||||
|
|
||||||
|
https://github.com/Cyan4973/lz4
|
||||||
|
|
||||||
|
Status
|
||||||
|
------
|
||||||
|
[](http://travis-ci.org/bkaradzic/go-lz4)
|
||||||
|
[](https://godoc.org/github.com/bkaradzic/go-lz4)
|
||||||
|
|
||||||
|
Usage
|
||||||
|
-----
|
||||||
|
|
||||||
|
go get github.com/bkaradzic/go-lz4
|
||||||
|
|
||||||
|
import "github.com/bkaradzic/go-lz4"
|
||||||
|
|
||||||
|
The package name is `lz4`
|
||||||
|
|
||||||
|
Notes
|
||||||
|
-----
|
||||||
|
|
||||||
|
* go-lz4 saves a uint32 with the original uncompressed length at the beginning
|
||||||
|
of the encoded buffer. They may get in the way of interoperability with
|
||||||
|
other implementations.
|
||||||
|
|
||||||
|
Contributors
|
||||||
|
------------
|
||||||
|
|
||||||
|
Damian Gryski ([@dgryski](https://github.com/dgryski))
|
||||||
|
Dustin Sallings ([@dustin](https://github.com/dustin))
|
||||||
|
|
||||||
|
Contact
|
||||||
|
-------
|
||||||
|
|
||||||
|
[@bkaradzic](https://twitter.com/bkaradzic)
|
||||||
|
http://www.stuckingeometry.com
|
||||||
|
|
||||||
|
Project page
|
||||||
|
https://github.com/bkaradzic/go-lz4
|
||||||
|
|
||||||
|
License
|
||||||
|
-------
|
||||||
|
|
||||||
|
Copyright 2011-2012 Branimir Karadzic. All rights reserved.
|
||||||
|
Copyright 2013 Damian Gryski. All rights reserved.
|
||||||
|
|
||||||
|
Redistribution and use in source and binary forms, with or without modification,
|
||||||
|
are permitted provided that the following conditions are met:
|
||||||
|
|
||||||
|
1. Redistributions of source code must retain the above copyright notice, this
|
||||||
|
list of conditions and the following disclaimer.
|
||||||
|
|
||||||
|
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||||
|
this list of conditions and the following disclaimer in the documentation
|
||||||
|
and/or other materials provided with the distribution.
|
||||||
|
|
||||||
|
THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
|
||||||
|
IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||||
|
MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
|
||||||
|
SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||||
|
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||||
|
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
||||||
|
WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
||||||
|
OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
|
||||||
|
THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
|
||||||
+23
@@ -0,0 +1,23 @@
|
|||||||
|
// +build gofuzz
|
||||||
|
|
||||||
|
package lz4
|
||||||
|
|
||||||
|
import "encoding/binary"
|
||||||
|
|
||||||
|
func Fuzz(data []byte) int {
|
||||||
|
|
||||||
|
if len(data) < 4 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
ln := binary.LittleEndian.Uint32(data)
|
||||||
|
if ln > (1 << 21) {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := Decode(nil, data); err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
return 1
|
||||||
|
}
|
||||||
+74
@@ -0,0 +1,74 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math/rand"
|
||||||
|
|
||||||
|
"github.com/bkaradzic/go-lz4"
|
||||||
|
|
||||||
|
// lz4's API matches snappy's, so we can easily see how it performs
|
||||||
|
// lz4 "code.google.com/p/snappy-go/snappy"
|
||||||
|
)
|
||||||
|
|
||||||
|
var input = `
|
||||||
|
ADVENTURE I. A SCANDAL IN BOHEMIA
|
||||||
|
|
||||||
|
I.
|
||||||
|
|
||||||
|
To Sherlock Holmes she is always THE woman. I have seldom heard
|
||||||
|
him mention her under any other name. In his eyes she eclipses
|
||||||
|
and predominates the whole of her sex. It was not that he felt
|
||||||
|
any emotion akin to love for Irene Adler. All emotions, and that
|
||||||
|
one particularly, were abhorrent to his cold, precise but
|
||||||
|
admirably balanced mind. He was, I take it, the most perfect
|
||||||
|
reasoning and observing machine that the world has seen, but as a
|
||||||
|
lover he would have placed himself in a false position. He never
|
||||||
|
spoke of the softer passions, save with a gibe and a sneer. They
|
||||||
|
were admirable things for the observer--excellent for drawing the
|
||||||
|
veil from men's motives and actions. But for the trained reasoner
|
||||||
|
to admit such intrusions into his own delicate and finely
|
||||||
|
adjusted temperament was to introduce a distracting factor which
|
||||||
|
might throw a doubt upon all his mental results. Grit in a
|
||||||
|
sensitive instrument, or a crack in one of his own high-power
|
||||||
|
lenses, would not be more disturbing than a strong emotion in a
|
||||||
|
nature such as his. And yet there was but one woman to him, and
|
||||||
|
that woman was the late Irene Adler, of dubious and questionable
|
||||||
|
memory.
|
||||||
|
|
||||||
|
I had seen little of Holmes lately. My marriage had drifted us
|
||||||
|
away from each other. My own complete happiness, and the
|
||||||
|
home-centred interests which rise up around the man who first
|
||||||
|
finds himself master of his own establishment, were sufficient to
|
||||||
|
absorb all my attention, while Holmes, who loathed every form of
|
||||||
|
society with his whole Bohemian soul, remained in our lodgings in
|
||||||
|
Baker Street, buried among his old books, and alternating from
|
||||||
|
week to week between cocaine and ambition, the drowsiness of the
|
||||||
|
drug, and the fierce energy of his own keen nature. He was still,
|
||||||
|
as ever, deeply attracted by the study of crime, and occupied his
|
||||||
|
immense faculties and extraordinary powers of observation in
|
||||||
|
following out those clues, and clearing up those mysteries which
|
||||||
|
had been abandoned as hopeless by the official police. From time
|
||||||
|
to time I heard some vague account of his doings: of his summons
|
||||||
|
to Odessa in the case of the Trepoff murder, of his clearing up
|
||||||
|
of the singular tragedy of the Atkinson brothers at Trincomalee,
|
||||||
|
and finally of the mission which he had accomplished so
|
||||||
|
delicately and successfully for the reigning family of Holland.
|
||||||
|
Beyond these signs of his activity, however, which I merely
|
||||||
|
shared with all the readers of the daily press, I knew little of
|
||||||
|
my former friend and companion.
|
||||||
|
`
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
|
||||||
|
compressed, _ := lz4.Encode(nil, []byte(input))
|
||||||
|
|
||||||
|
modified := make([]byte, len(compressed))
|
||||||
|
|
||||||
|
for {
|
||||||
|
copy(modified, compressed)
|
||||||
|
for i := 0; i < 100; i++ {
|
||||||
|
modified[rand.Intn(len(compressed)-4)+4] = byte(rand.Intn(256))
|
||||||
|
}
|
||||||
|
lz4.Decode(nil, modified)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
+94
@@ -0,0 +1,94 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2011 Branimir Karadzic. All rights reserved.
|
||||||
|
*
|
||||||
|
* Redistribution and use in source and binary forms, with or without modification,
|
||||||
|
* are permitted provided that the following conditions are met:
|
||||||
|
*
|
||||||
|
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||||
|
* list of conditions and the following disclaimer.
|
||||||
|
*
|
||||||
|
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||||
|
* this list of conditions and the following disclaimer in the documentation
|
||||||
|
* and/or other materials provided with the distribution.
|
||||||
|
*
|
||||||
|
* THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
|
||||||
|
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||||
|
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
|
||||||
|
* SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||||
|
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||||
|
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
||||||
|
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
||||||
|
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
|
||||||
|
* THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"runtime/pprof"
|
||||||
|
|
||||||
|
lz4 "github.com/bkaradzic/go-lz4"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
decompress = flag.Bool("d", false, "decompress")
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
|
||||||
|
var optCPUProfile = flag.String("cpuprofile", "", "profile")
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
if *optCPUProfile != "" {
|
||||||
|
f, err := os.Create(*optCPUProfile)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
pprof.StartCPUProfile(f)
|
||||||
|
defer pprof.StopCPUProfile()
|
||||||
|
}
|
||||||
|
|
||||||
|
args := flag.Args()
|
||||||
|
|
||||||
|
var data []byte
|
||||||
|
|
||||||
|
if len(args) < 2 {
|
||||||
|
fmt.Print("Usage: lz4 [-d] <input> <output>\n")
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
input, err := os.OpenFile(args[0], os.O_RDONLY, 0644)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Failed to open input file %s\n", args[0])
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
defer input.Close()
|
||||||
|
|
||||||
|
if *decompress {
|
||||||
|
data, _ = ioutil.ReadAll(input)
|
||||||
|
data, err = lz4.Decode(nil, data)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println("Failed to decode:", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
data, _ = ioutil.ReadAll(input)
|
||||||
|
data, err = lz4.Encode(nil, data)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Println("Failed to encode:", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
err = ioutil.WriteFile(args[1], data, 0644)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Failed to open output file %s\n", args[1])
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
+199
@@ -0,0 +1,199 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2011-2012 Branimir Karadzic. All rights reserved.
|
||||||
|
*
|
||||||
|
* Redistribution and use in source and binary forms, with or without modification,
|
||||||
|
* are permitted provided that the following conditions are met:
|
||||||
|
*
|
||||||
|
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||||
|
* list of conditions and the following disclaimer.
|
||||||
|
*
|
||||||
|
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||||
|
* this list of conditions and the following disclaimer in the documentation
|
||||||
|
* and/or other materials provided with the distribution.
|
||||||
|
*
|
||||||
|
* THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
|
||||||
|
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||||
|
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
|
||||||
|
* SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||||
|
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||||
|
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
||||||
|
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
||||||
|
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
|
||||||
|
* THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package lz4
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/binary"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
// ErrCorrupt indicates the input was corrupt
|
||||||
|
ErrCorrupt = errors.New("corrupt input")
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
mlBits = 4
|
||||||
|
mlMask = (1 << mlBits) - 1
|
||||||
|
runBits = 8 - mlBits
|
||||||
|
runMask = (1 << runBits) - 1
|
||||||
|
)
|
||||||
|
|
||||||
|
type decoder struct {
|
||||||
|
src []byte
|
||||||
|
dst []byte
|
||||||
|
spos uint32
|
||||||
|
dpos uint32
|
||||||
|
ref uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *decoder) readByte() (uint8, error) {
|
||||||
|
if int(d.spos) == len(d.src) {
|
||||||
|
return 0, io.EOF
|
||||||
|
}
|
||||||
|
b := d.src[d.spos]
|
||||||
|
d.spos++
|
||||||
|
return b, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *decoder) getLen() (uint32, error) {
|
||||||
|
|
||||||
|
length := uint32(0)
|
||||||
|
ln, err := d.readByte()
|
||||||
|
if err != nil {
|
||||||
|
return 0, ErrCorrupt
|
||||||
|
}
|
||||||
|
for ln == 255 {
|
||||||
|
length += 255
|
||||||
|
ln, err = d.readByte()
|
||||||
|
if err != nil {
|
||||||
|
return 0, ErrCorrupt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
length += uint32(ln)
|
||||||
|
|
||||||
|
return length, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *decoder) cp(length, decr uint32) {
|
||||||
|
|
||||||
|
if int(d.ref+length) < int(d.dpos) {
|
||||||
|
copy(d.dst[d.dpos:], d.dst[d.ref:d.ref+length])
|
||||||
|
} else {
|
||||||
|
for ii := uint32(0); ii < length; ii++ {
|
||||||
|
d.dst[d.dpos+ii] = d.dst[d.ref+ii]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
d.dpos += length
|
||||||
|
d.ref += length - decr
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *decoder) finish(err error) error {
|
||||||
|
if err == io.EOF {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decode returns the decoded form of src. The returned slice may be a
|
||||||
|
// subslice of dst if it was large enough to hold the entire decoded block.
|
||||||
|
func Decode(dst, src []byte) ([]byte, error) {
|
||||||
|
|
||||||
|
if len(src) < 4 {
|
||||||
|
return nil, ErrCorrupt
|
||||||
|
}
|
||||||
|
|
||||||
|
uncompressedLen := binary.LittleEndian.Uint32(src)
|
||||||
|
|
||||||
|
if uncompressedLen == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if uncompressedLen > MaxInputSize {
|
||||||
|
return nil, ErrTooLarge
|
||||||
|
}
|
||||||
|
|
||||||
|
if dst == nil || len(dst) < int(uncompressedLen) {
|
||||||
|
dst = make([]byte, uncompressedLen)
|
||||||
|
}
|
||||||
|
|
||||||
|
d := decoder{src: src, dst: dst[:uncompressedLen], spos: 4}
|
||||||
|
|
||||||
|
decr := []uint32{0, 3, 2, 3}
|
||||||
|
|
||||||
|
for {
|
||||||
|
code, err := d.readByte()
|
||||||
|
if err != nil {
|
||||||
|
return d.dst, d.finish(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
length := uint32(code >> mlBits)
|
||||||
|
if length == runMask {
|
||||||
|
ln, err := d.getLen()
|
||||||
|
if err != nil {
|
||||||
|
return nil, ErrCorrupt
|
||||||
|
}
|
||||||
|
length += ln
|
||||||
|
}
|
||||||
|
|
||||||
|
if int(d.spos+length) > len(d.src) || int(d.dpos+length) > len(d.dst) {
|
||||||
|
return nil, ErrCorrupt
|
||||||
|
}
|
||||||
|
|
||||||
|
for ii := uint32(0); ii < length; ii++ {
|
||||||
|
d.dst[d.dpos+ii] = d.src[d.spos+ii]
|
||||||
|
}
|
||||||
|
|
||||||
|
d.spos += length
|
||||||
|
d.dpos += length
|
||||||
|
|
||||||
|
if int(d.spos) == len(d.src) {
|
||||||
|
return d.dst, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if int(d.spos+2) >= len(d.src) {
|
||||||
|
return nil, ErrCorrupt
|
||||||
|
}
|
||||||
|
|
||||||
|
back := uint32(d.src[d.spos]) | uint32(d.src[d.spos+1])<<8
|
||||||
|
|
||||||
|
if back > d.dpos {
|
||||||
|
return nil, ErrCorrupt
|
||||||
|
}
|
||||||
|
|
||||||
|
d.spos += 2
|
||||||
|
d.ref = d.dpos - back
|
||||||
|
|
||||||
|
length = uint32(code & mlMask)
|
||||||
|
if length == mlMask {
|
||||||
|
ln, err := d.getLen()
|
||||||
|
if err != nil {
|
||||||
|
return nil, ErrCorrupt
|
||||||
|
}
|
||||||
|
length += ln
|
||||||
|
}
|
||||||
|
|
||||||
|
literal := d.dpos - d.ref
|
||||||
|
|
||||||
|
if literal < 4 {
|
||||||
|
if int(d.dpos+4) > len(d.dst) {
|
||||||
|
return nil, ErrCorrupt
|
||||||
|
}
|
||||||
|
|
||||||
|
d.cp(4, decr[literal])
|
||||||
|
} else {
|
||||||
|
length += 4
|
||||||
|
}
|
||||||
|
|
||||||
|
if d.dpos+length > uncompressedLen {
|
||||||
|
return nil, ErrCorrupt
|
||||||
|
}
|
||||||
|
|
||||||
|
d.cp(length, 0)
|
||||||
|
}
|
||||||
|
}
|
||||||
+190
@@ -0,0 +1,190 @@
|
|||||||
|
/*
|
||||||
|
* Copyright 2011-2012 Branimir Karadzic. All rights reserved.
|
||||||
|
*
|
||||||
|
* Redistribution and use in source and binary forms, with or without modification,
|
||||||
|
* are permitted provided that the following conditions are met:
|
||||||
|
*
|
||||||
|
* 1. Redistributions of source code must retain the above copyright notice, this
|
||||||
|
* list of conditions and the following disclaimer.
|
||||||
|
*
|
||||||
|
* 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||||
|
* this list of conditions and the following disclaimer in the documentation
|
||||||
|
* and/or other materials provided with the distribution.
|
||||||
|
*
|
||||||
|
* THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
|
||||||
|
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
||||||
|
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
|
||||||
|
* SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||||
|
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
||||||
|
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
|
||||||
|
* WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
||||||
|
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
|
||||||
|
* THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package lz4
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/binary"
|
||||||
|
"errors"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
minMatch = 4
|
||||||
|
hashLog = 17
|
||||||
|
hashTableSize = 1 << hashLog
|
||||||
|
hashShift = (minMatch * 8) - hashLog
|
||||||
|
incompressible uint32 = 128
|
||||||
|
uninitHash = 0x88888888
|
||||||
|
|
||||||
|
// MaxInputSize is the largest buffer than can be compressed in a single block
|
||||||
|
MaxInputSize = 0x7E000000
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
// ErrTooLarge indicates the input buffer was too large
|
||||||
|
ErrTooLarge = errors.New("input too large")
|
||||||
|
)
|
||||||
|
|
||||||
|
type encoder struct {
|
||||||
|
src []byte
|
||||||
|
dst []byte
|
||||||
|
hashTable []uint32
|
||||||
|
pos uint32
|
||||||
|
anchor uint32
|
||||||
|
dpos uint32
|
||||||
|
}
|
||||||
|
|
||||||
|
// CompressBound returns the maximum length of a lz4 block, given it's uncompressed length
|
||||||
|
func CompressBound(isize int) int {
|
||||||
|
if isize > MaxInputSize {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return isize + ((isize) / 255) + 16 + 4
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *encoder) writeLiterals(length, mlLen, pos uint32) {
|
||||||
|
|
||||||
|
ln := length
|
||||||
|
|
||||||
|
var code byte
|
||||||
|
if ln > runMask-1 {
|
||||||
|
code = runMask
|
||||||
|
} else {
|
||||||
|
code = byte(ln)
|
||||||
|
}
|
||||||
|
|
||||||
|
if mlLen > mlMask-1 {
|
||||||
|
e.dst[e.dpos] = (code << mlBits) + byte(mlMask)
|
||||||
|
} else {
|
||||||
|
e.dst[e.dpos] = (code << mlBits) + byte(mlLen)
|
||||||
|
}
|
||||||
|
e.dpos++
|
||||||
|
|
||||||
|
if code == runMask {
|
||||||
|
ln -= runMask
|
||||||
|
for ; ln > 254; ln -= 255 {
|
||||||
|
e.dst[e.dpos] = 255
|
||||||
|
e.dpos++
|
||||||
|
}
|
||||||
|
|
||||||
|
e.dst[e.dpos] = byte(ln)
|
||||||
|
e.dpos++
|
||||||
|
}
|
||||||
|
|
||||||
|
for ii := uint32(0); ii < length; ii++ {
|
||||||
|
e.dst[e.dpos+ii] = e.src[pos+ii]
|
||||||
|
}
|
||||||
|
|
||||||
|
e.dpos += length
|
||||||
|
}
|
||||||
|
|
||||||
|
// Encode returns the encoded form of src. The returned array may be a
|
||||||
|
// sub-slice of dst if it was large enough to hold the entire output.
|
||||||
|
func Encode(dst, src []byte) ([]byte, error) {
|
||||||
|
|
||||||
|
if len(src) >= MaxInputSize {
|
||||||
|
return nil, ErrTooLarge
|
||||||
|
}
|
||||||
|
|
||||||
|
if n := CompressBound(len(src)); len(dst) < n {
|
||||||
|
dst = make([]byte, n)
|
||||||
|
}
|
||||||
|
|
||||||
|
e := encoder{src: src, dst: dst, hashTable: make([]uint32, hashTableSize)}
|
||||||
|
|
||||||
|
binary.LittleEndian.PutUint32(dst, uint32(len(src)))
|
||||||
|
e.dpos = 4
|
||||||
|
|
||||||
|
var (
|
||||||
|
step uint32 = 1
|
||||||
|
limit = incompressible
|
||||||
|
)
|
||||||
|
|
||||||
|
for {
|
||||||
|
if int(e.pos)+12 >= len(e.src) {
|
||||||
|
e.writeLiterals(uint32(len(e.src))-e.anchor, 0, e.anchor)
|
||||||
|
return e.dst[:e.dpos], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
sequence := uint32(e.src[e.pos+3])<<24 | uint32(e.src[e.pos+2])<<16 | uint32(e.src[e.pos+1])<<8 | uint32(e.src[e.pos+0])
|
||||||
|
|
||||||
|
hash := (sequence * 2654435761) >> hashShift
|
||||||
|
ref := e.hashTable[hash] + uninitHash
|
||||||
|
e.hashTable[hash] = e.pos - uninitHash
|
||||||
|
|
||||||
|
if ((e.pos-ref)>>16) != 0 || uint32(e.src[ref+3])<<24|uint32(e.src[ref+2])<<16|uint32(e.src[ref+1])<<8|uint32(e.src[ref+0]) != sequence {
|
||||||
|
if e.pos-e.anchor > limit {
|
||||||
|
limit <<= 1
|
||||||
|
step += 1 + (step >> 2)
|
||||||
|
}
|
||||||
|
e.pos += step
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if step > 1 {
|
||||||
|
e.hashTable[hash] = ref - uninitHash
|
||||||
|
e.pos -= step - 1
|
||||||
|
step = 1
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
limit = incompressible
|
||||||
|
|
||||||
|
ln := e.pos - e.anchor
|
||||||
|
back := e.pos - ref
|
||||||
|
|
||||||
|
anchor := e.anchor
|
||||||
|
|
||||||
|
e.pos += minMatch
|
||||||
|
ref += minMatch
|
||||||
|
e.anchor = e.pos
|
||||||
|
|
||||||
|
for int(e.pos) < len(e.src)-5 && e.src[e.pos] == e.src[ref] {
|
||||||
|
e.pos++
|
||||||
|
ref++
|
||||||
|
}
|
||||||
|
|
||||||
|
mlLen := e.pos - e.anchor
|
||||||
|
|
||||||
|
e.writeLiterals(ln, mlLen, anchor)
|
||||||
|
e.dst[e.dpos] = uint8(back)
|
||||||
|
e.dst[e.dpos+1] = uint8(back >> 8)
|
||||||
|
e.dpos += 2
|
||||||
|
|
||||||
|
if mlLen > mlMask-1 {
|
||||||
|
mlLen -= mlMask
|
||||||
|
for mlLen > 254 {
|
||||||
|
mlLen -= 255
|
||||||
|
|
||||||
|
e.dst[e.dpos] = 255
|
||||||
|
e.dpos++
|
||||||
|
}
|
||||||
|
|
||||||
|
e.dst[e.dpos] = byte(mlLen)
|
||||||
|
e.dpos++
|
||||||
|
}
|
||||||
|
|
||||||
|
e.anchor = e.pos
|
||||||
|
}
|
||||||
|
}
|
||||||
+24
@@ -0,0 +1,24 @@
|
|||||||
|
This is free and unencumbered software released into the public domain.
|
||||||
|
|
||||||
|
Anyone is free to copy, modify, publish, use, compile, sell, or
|
||||||
|
distribute this software, either in source code form or as a compiled
|
||||||
|
binary, for any purpose, commercial or non-commercial, and by any
|
||||||
|
means.
|
||||||
|
|
||||||
|
In jurisdictions that recognize copyright laws, the author or authors
|
||||||
|
of this software dedicate any and all copyright interest in the
|
||||||
|
software to the public domain. We make this dedication for the benefit
|
||||||
|
of the public at large and to the detriment of our heirs and
|
||||||
|
successors. We intend this dedication to be an overt act of
|
||||||
|
relinquishment in perpetuity of all present and future rights to this
|
||||||
|
software under copyright law.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||||
|
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||||
|
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||||
|
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||||
|
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||||
|
OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
|
||||||
|
For more information, please refer to <http://unlicense.org>
|
||||||
+14
@@ -0,0 +1,14 @@
|
|||||||
|
du
|
||||||
|
==
|
||||||
|
|
||||||
|
Get total and available disk space on a given volume.
|
||||||
|
|
||||||
|
Documentation
|
||||||
|
-------------
|
||||||
|
|
||||||
|
http://godoc.org/github.com/calmh/du
|
||||||
|
|
||||||
|
License
|
||||||
|
-------
|
||||||
|
|
||||||
|
Public Domain
|
||||||
+21
@@ -0,0 +1,21 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/calmh/du"
|
||||||
|
)
|
||||||
|
|
||||||
|
var KB = int64(1024)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
usage, err := du.Get(os.Args[1])
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
fmt.Println("Free:", usage.FreeBytes/(KB*KB), "MiB")
|
||||||
|
fmt.Println("Available:", usage.AvailBytes/(KB*KB), "MiB")
|
||||||
|
fmt.Println("Size:", usage.TotalBytes/(KB*KB), "MiB")
|
||||||
|
}
|
||||||
+8
@@ -0,0 +1,8 @@
|
|||||||
|
package du
|
||||||
|
|
||||||
|
// Usage holds information about total and available storage on a volume.
|
||||||
|
type Usage struct {
|
||||||
|
TotalBytes int64 // Size of volume
|
||||||
|
FreeBytes int64 // Unused size
|
||||||
|
AvailBytes int64 // Available to a non-privileged user
|
||||||
|
}
|
||||||
+24
@@ -0,0 +1,24 @@
|
|||||||
|
// +build !windows,!netbsd,!openbsd,!solaris
|
||||||
|
|
||||||
|
package du
|
||||||
|
|
||||||
|
import (
|
||||||
|
"path/filepath"
|
||||||
|
"syscall"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Get returns the Usage of a given path, or an error if usage data is
|
||||||
|
// unavailable.
|
||||||
|
func Get(path string) (Usage, error) {
|
||||||
|
var stat syscall.Statfs_t
|
||||||
|
err := syscall.Statfs(filepath.Clean(path), &stat)
|
||||||
|
if err != nil {
|
||||||
|
return Usage{}, err
|
||||||
|
}
|
||||||
|
u := Usage{
|
||||||
|
FreeBytes: int64(stat.Bfree) * int64(stat.Bsize),
|
||||||
|
TotalBytes: int64(stat.Blocks) * int64(stat.Bsize),
|
||||||
|
AvailBytes: int64(stat.Bavail) * int64(stat.Bsize),
|
||||||
|
}
|
||||||
|
return u, nil
|
||||||
|
}
|
||||||
+13
@@ -0,0 +1,13 @@
|
|||||||
|
// +build netbsd openbsd solaris
|
||||||
|
|
||||||
|
package du
|
||||||
|
|
||||||
|
import "errors"
|
||||||
|
|
||||||
|
var ErrUnsupported = errors.New("unsupported platform")
|
||||||
|
|
||||||
|
// Get returns the Usage of a given path, or an error if usage data is
|
||||||
|
// unavailable.
|
||||||
|
func Get(path string) (Usage, error) {
|
||||||
|
return Usage{}, ErrUnsupported
|
||||||
|
}
|
||||||
+27
@@ -0,0 +1,27 @@
|
|||||||
|
package du
|
||||||
|
|
||||||
|
import (
|
||||||
|
"syscall"
|
||||||
|
"unsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Get returns the Usage of a given path, or an error if usage data is
|
||||||
|
// unavailable.
|
||||||
|
func Get(path string) (Usage, error) {
|
||||||
|
h := syscall.MustLoadDLL("kernel32.dll")
|
||||||
|
c := h.MustFindProc("GetDiskFreeSpaceExW")
|
||||||
|
|
||||||
|
var u Usage
|
||||||
|
|
||||||
|
ret, _, err := c.Call(
|
||||||
|
uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
|
||||||
|
uintptr(unsafe.Pointer(&u.FreeBytes)),
|
||||||
|
uintptr(unsafe.Pointer(&u.TotalBytes)),
|
||||||
|
uintptr(unsafe.Pointer(&u.AvailBytes)))
|
||||||
|
|
||||||
|
if ret == 0 {
|
||||||
|
return u, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return u, nil
|
||||||
|
}
|
||||||
+19
@@ -0,0 +1,19 @@
|
|||||||
|
Copyright (C) 2014 Jakob Borg
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||||
|
this software and associated documentation files (the "Software"), to deal in
|
||||||
|
the Software without restriction, including without limitation the rights to
|
||||||
|
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
|
||||||
|
of the Software, and to permit persons to whom the Software is furnished to do
|
||||||
|
so, subject to the following conditions:
|
||||||
|
|
||||||
|
- The above copyright notice and this permission notice shall be included in
|
||||||
|
all copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
+70
@@ -0,0 +1,70 @@
|
|||||||
|
// Copyright (C) 2014 Jakob Borg
|
||||||
|
|
||||||
|
// Package luhn generates and validates Luhn mod N check digits.
|
||||||
|
package luhn
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// An alphabet is a string of N characters, representing the digits of a given
|
||||||
|
// base N.
|
||||||
|
type Alphabet string
|
||||||
|
|
||||||
|
var (
|
||||||
|
Base32 Alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Generate returns a check digit for the string s, which should be composed
|
||||||
|
// of characters from the Alphabet a.
|
||||||
|
func (a Alphabet) Generate(s string) (rune, error) {
|
||||||
|
if err := a.check(); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
factor := 1
|
||||||
|
sum := 0
|
||||||
|
n := len(a)
|
||||||
|
|
||||||
|
for i := range s {
|
||||||
|
codepoint := strings.IndexByte(string(a), s[i])
|
||||||
|
if codepoint == -1 {
|
||||||
|
return 0, fmt.Errorf("Digit %q not valid in alphabet %q", s[i], a)
|
||||||
|
}
|
||||||
|
addend := factor * codepoint
|
||||||
|
if factor == 2 {
|
||||||
|
factor = 1
|
||||||
|
} else {
|
||||||
|
factor = 2
|
||||||
|
}
|
||||||
|
addend = (addend / n) + (addend % n)
|
||||||
|
sum += addend
|
||||||
|
}
|
||||||
|
remainder := sum % n
|
||||||
|
checkCodepoint := (n - remainder) % n
|
||||||
|
return rune(a[checkCodepoint]), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate returns true if the last character of the string s is correct, for
|
||||||
|
// a string s composed of characters in the alphabet a.
|
||||||
|
func (a Alphabet) Validate(s string) bool {
|
||||||
|
t := s[:len(s)-1]
|
||||||
|
c, err := a.Generate(t)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return rune(s[len(s)-1]) == c
|
||||||
|
}
|
||||||
|
|
||||||
|
// check returns an error if the given alphabet does not consist of unique characters
|
||||||
|
func (a Alphabet) check() error {
|
||||||
|
cm := make(map[byte]bool, len(a))
|
||||||
|
for i := range a {
|
||||||
|
if cm[a[i]] {
|
||||||
|
return fmt.Errorf("Digit %q non-unique in alphabet %q", a[i], a)
|
||||||
|
}
|
||||||
|
cm[a[i]] = true
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
coverage.out
|
||||||
+19
@@ -0,0 +1,19 @@
|
|||||||
|
language: go
|
||||||
|
go:
|
||||||
|
- tip
|
||||||
|
|
||||||
|
install:
|
||||||
|
- export PATH=$PATH:$HOME/gopath/bin
|
||||||
|
- go get golang.org/x/tools/cover
|
||||||
|
- go get github.com/mattn/goveralls
|
||||||
|
|
||||||
|
script:
|
||||||
|
- ./generate.sh
|
||||||
|
- go test -coverprofile=coverage.out
|
||||||
|
|
||||||
|
after_success:
|
||||||
|
- goveralls -coverprofile=coverage.out -service=travis-ci -package=calmh/xdr -repotoken="$COVERALLS_TOKEN"
|
||||||
|
|
||||||
|
env:
|
||||||
|
global:
|
||||||
|
secure: SmgnrGfp2zLrA44ChRMpjPeujubt9veZ8Fx/OseMWECmacyV5N/TuDhzIbwo6QwV4xB0sBacoPzvxQbJRVjNKsPiSu72UbcQmQ7flN4Tf7nW09tSh1iW8NgrpBCq/3UYLoBu2iPBEBKm93IK0aGNAKs6oEkB0fU27iTVBwiTXOY=
|
||||||
Generated
Vendored
+10
-10
@@ -1,19 +1,19 @@
|
|||||||
(c) 2010-2014 Google, Inc. http://angularjs.org
|
Copyright (C) 2014 Jakob Borg.
|
||||||
|
|
||||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
of this software and associated documentation files (the "Software"), to deal
|
of this software and associated documentation files (the "Software"), to
|
||||||
in the Software without restriction, including without limitation the rights
|
deal in the Software without restriction, including without limitation the
|
||||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
|
||||||
copies of the Software, and to permit persons to whom the Software is
|
sell copies of the Software, and to permit persons to whom the Software is
|
||||||
furnished to do so, subject to the following conditions:
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
The above copyright notice and this permission notice shall be included in
|
- The above copyright notice and this permission notice shall be included in
|
||||||
all copies or substantial portions of the Software.
|
all copies or substantial portions of the Software.
|
||||||
|
|
||||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
||||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
||||||
THE SOFTWARE.
|
IN THE SOFTWARE.
|
||||||
+12
@@ -0,0 +1,12 @@
|
|||||||
|
xdr
|
||||||
|
===
|
||||||
|
|
||||||
|
[](https://circleci.com/gh/calmh/xdr)
|
||||||
|
[](https://coveralls.io/r/calmh/xdr?branch=master)
|
||||||
|
[](http://godoc.org/github.com/calmh/xdr)
|
||||||
|
[](http://opensource.org/licenses/MIT)
|
||||||
|
|
||||||
|
This is an XDR encoding/decoding library. It uses code generation and
|
||||||
|
not reflection. It supports the IPDR bastardized XDR format when built
|
||||||
|
with `-tags ipdr`.
|
||||||
|
|
||||||
+3
@@ -0,0 +1,3 @@
|
|||||||
|
dependencies:
|
||||||
|
post:
|
||||||
|
- ./generate.sh
|
||||||
+530
@@ -0,0 +1,530 @@
|
|||||||
|
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
|
||||||
|
// is governed by an MIT-style license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"go/ast"
|
||||||
|
"go/format"
|
||||||
|
"go/parser"
|
||||||
|
"go/token"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"text/template"
|
||||||
|
)
|
||||||
|
|
||||||
|
type fieldInfo struct {
|
||||||
|
Name string
|
||||||
|
IsBasic bool // handled by one the native Read/WriteUint64 etc functions
|
||||||
|
IsSlice bool // field is a slice of FieldType
|
||||||
|
FieldType string // original type of field, i.e. "int"
|
||||||
|
Encoder string // the encoder name, i.e. "Uint64" for Read/WriteUint64
|
||||||
|
Convert string // what to convert to when encoding, i.e. "uint64"
|
||||||
|
Max int // max size for slices and strings
|
||||||
|
Submax int // max size for strings inside slices
|
||||||
|
}
|
||||||
|
|
||||||
|
type structInfo struct {
|
||||||
|
Name string
|
||||||
|
Fields []fieldInfo
|
||||||
|
}
|
||||||
|
|
||||||
|
var headerTpl = template.Must(template.New("header").Parse(`// ************************************************************
|
||||||
|
// This file is automatically generated by genxdr. Do not edit.
|
||||||
|
// ************************************************************
|
||||||
|
|
||||||
|
package {{.Package}}
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"io"
|
||||||
|
|
||||||
|
"github.com/calmh/xdr"
|
||||||
|
)
|
||||||
|
`))
|
||||||
|
|
||||||
|
var encodeTpl = template.Must(template.New("encoder").Parse(`
|
||||||
|
func (o {{.TypeName}}) EncodeXDR(w io.Writer) (int, error) {
|
||||||
|
var xw = xdr.NewWriter(w)
|
||||||
|
return o.EncodeXDRInto(xw)
|
||||||
|
}//+n
|
||||||
|
|
||||||
|
func (o {{.TypeName}}) MarshalXDR() ([]byte, error) {
|
||||||
|
return o.AppendXDR(make([]byte, 0, 128))
|
||||||
|
}//+n
|
||||||
|
|
||||||
|
func (o {{.TypeName}}) MustMarshalXDR() []byte {
|
||||||
|
bs, err := o.MarshalXDR()
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
return bs
|
||||||
|
}//+n
|
||||||
|
|
||||||
|
func (o {{.TypeName}}) AppendXDR(bs []byte) ([]byte, error) {
|
||||||
|
var aw = xdr.AppendWriter(bs)
|
||||||
|
var xw = xdr.NewWriter(&aw)
|
||||||
|
_, err := o.EncodeXDRInto(xw)
|
||||||
|
return []byte(aw), err
|
||||||
|
}//+n
|
||||||
|
|
||||||
|
func (o {{.TypeName}}) EncodeXDRInto(xw *xdr.Writer) (int, error) {
|
||||||
|
{{range $fieldInfo := .Fields}}
|
||||||
|
{{if not $fieldInfo.IsSlice}}
|
||||||
|
{{if ne $fieldInfo.Convert ""}}
|
||||||
|
xw.Write{{$fieldInfo.Encoder}}({{$fieldInfo.Convert}}(o.{{$fieldInfo.Name}}))
|
||||||
|
{{else if $fieldInfo.IsBasic}}
|
||||||
|
{{if ge $fieldInfo.Max 1}}
|
||||||
|
if l := len(o.{{$fieldInfo.Name}}); l > {{$fieldInfo.Max}} {
|
||||||
|
return xw.Tot(), xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", l, {{$fieldInfo.Max}})
|
||||||
|
}
|
||||||
|
{{end}}
|
||||||
|
xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}})
|
||||||
|
{{else}}
|
||||||
|
_, err := o.{{$fieldInfo.Name}}.EncodeXDRInto(xw)
|
||||||
|
if err != nil {
|
||||||
|
return xw.Tot(), err
|
||||||
|
}
|
||||||
|
{{end}}
|
||||||
|
{{else}}
|
||||||
|
{{if ge $fieldInfo.Max 1}}
|
||||||
|
if l := len(o.{{$fieldInfo.Name}}); l > {{$fieldInfo.Max}} {
|
||||||
|
return xw.Tot(), xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", l, {{$fieldInfo.Max}})
|
||||||
|
}
|
||||||
|
{{end}}
|
||||||
|
xw.WriteUint32(uint32(len(o.{{$fieldInfo.Name}})))
|
||||||
|
for i := range o.{{$fieldInfo.Name}} {
|
||||||
|
{{if ne $fieldInfo.Convert ""}}
|
||||||
|
xw.Write{{$fieldInfo.Encoder}}({{$fieldInfo.Convert}}(o.{{$fieldInfo.Name}}[i]))
|
||||||
|
{{else if $fieldInfo.IsBasic}}
|
||||||
|
xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}}[i])
|
||||||
|
{{else}}
|
||||||
|
_, err := o.{{$fieldInfo.Name}}[i].EncodeXDRInto(xw)
|
||||||
|
if err != nil {
|
||||||
|
return xw.Tot(), err
|
||||||
|
}
|
||||||
|
{{end}}
|
||||||
|
}
|
||||||
|
{{end}}
|
||||||
|
{{end}}
|
||||||
|
return xw.Tot(), xw.Error()
|
||||||
|
}//+n
|
||||||
|
|
||||||
|
func (o *{{.TypeName}}) DecodeXDR(r io.Reader) error {
|
||||||
|
xr := xdr.NewReader(r)
|
||||||
|
return o.DecodeXDRFrom(xr)
|
||||||
|
}//+n
|
||||||
|
|
||||||
|
func (o *{{.TypeName}}) UnmarshalXDR(bs []byte) error {
|
||||||
|
var br = bytes.NewReader(bs)
|
||||||
|
var xr = xdr.NewReader(br)
|
||||||
|
return o.DecodeXDRFrom(xr)
|
||||||
|
}//+n
|
||||||
|
|
||||||
|
func (o *{{.TypeName}}) DecodeXDRFrom(xr *xdr.Reader) error {
|
||||||
|
{{range $fieldInfo := .Fields}}
|
||||||
|
{{if not $fieldInfo.IsSlice}}
|
||||||
|
{{if ne $fieldInfo.Convert ""}}
|
||||||
|
o.{{$fieldInfo.Name}} = {{$fieldInfo.FieldType}}(xr.Read{{$fieldInfo.Encoder}}())
|
||||||
|
{{else if $fieldInfo.IsBasic}}
|
||||||
|
{{if ge $fieldInfo.Max 1}}
|
||||||
|
o.{{$fieldInfo.Name}} = xr.Read{{$fieldInfo.Encoder}}Max({{$fieldInfo.Max}})
|
||||||
|
{{else}}
|
||||||
|
o.{{$fieldInfo.Name}} = xr.Read{{$fieldInfo.Encoder}}()
|
||||||
|
{{end}}
|
||||||
|
{{else}}
|
||||||
|
(&o.{{$fieldInfo.Name}}).DecodeXDRFrom(xr)
|
||||||
|
{{end}}
|
||||||
|
{{else}}
|
||||||
|
_{{$fieldInfo.Name}}Size := int(xr.ReadUint32())
|
||||||
|
if _{{$fieldInfo.Name}}Size < 0 {
|
||||||
|
return xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", _{{$fieldInfo.Name}}Size, {{$fieldInfo.Max}})
|
||||||
|
}
|
||||||
|
{{if ge $fieldInfo.Max 1}}
|
||||||
|
if _{{$fieldInfo.Name}}Size > {{$fieldInfo.Max}} {
|
||||||
|
return xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", _{{$fieldInfo.Name}}Size, {{$fieldInfo.Max}})
|
||||||
|
}
|
||||||
|
{{end}}
|
||||||
|
o.{{$fieldInfo.Name}} = make([]{{$fieldInfo.FieldType}}, _{{$fieldInfo.Name}}Size)
|
||||||
|
for i := range o.{{$fieldInfo.Name}} {
|
||||||
|
{{if ne $fieldInfo.Convert ""}}
|
||||||
|
o.{{$fieldInfo.Name}}[i] = {{$fieldInfo.FieldType}}(xr.Read{{$fieldInfo.Encoder}}())
|
||||||
|
{{else if $fieldInfo.IsBasic}}
|
||||||
|
{{if ge $fieldInfo.Submax 1}}
|
||||||
|
o.{{$fieldInfo.Name}}[i] = xr.Read{{$fieldInfo.Encoder}}Max({{$fieldInfo.Submax}})
|
||||||
|
{{else}}
|
||||||
|
o.{{$fieldInfo.Name}}[i] = xr.Read{{$fieldInfo.Encoder}}()
|
||||||
|
{{end}}
|
||||||
|
{{else}}
|
||||||
|
(&o.{{$fieldInfo.Name}}[i]).DecodeXDRFrom(xr)
|
||||||
|
{{end}}
|
||||||
|
}
|
||||||
|
{{end}}
|
||||||
|
{{end}}
|
||||||
|
return xr.Error()
|
||||||
|
}`))
|
||||||
|
|
||||||
|
var emptyTypeTpl = template.Must(template.New("encoder").Parse(`
|
||||||
|
func (o {{.TypeName}}) EncodeXDR(w io.Writer) (int, error) {
|
||||||
|
return 0, nil
|
||||||
|
}//+n
|
||||||
|
|
||||||
|
func (o {{.TypeName}}) MarshalXDR() ([]byte, error) {
|
||||||
|
return nil, nil
|
||||||
|
}//+n
|
||||||
|
|
||||||
|
func (o {{.TypeName}}) MustMarshalXDR() []byte {
|
||||||
|
return nil
|
||||||
|
}//+n
|
||||||
|
|
||||||
|
func (o {{.TypeName}}) AppendXDR(bs []byte) ([]byte, error) {
|
||||||
|
return bs, nil
|
||||||
|
}//+n
|
||||||
|
|
||||||
|
func (o {{.TypeName}}) EncodeXDRInto(xw *xdr.Writer) (int, error) {
|
||||||
|
return xw.Tot(), xw.Error()
|
||||||
|
}//+n
|
||||||
|
|
||||||
|
func (o *{{.TypeName}}) DecodeXDR(r io.Reader) error {
|
||||||
|
return nil
|
||||||
|
}//+n
|
||||||
|
|
||||||
|
func (o *{{.TypeName}}) UnmarshalXDR(bs []byte) error {
|
||||||
|
return nil
|
||||||
|
}//+n
|
||||||
|
|
||||||
|
func (o *{{.TypeName}}) DecodeXDRFrom(xr *xdr.Reader) error {
|
||||||
|
return xr.Error()
|
||||||
|
}`))
|
||||||
|
|
||||||
|
var maxRe = regexp.MustCompile(`(?:\Wmax:)(\d+)(?:\s*,\s*(\d+))?`)
|
||||||
|
|
||||||
|
type typeSet struct {
|
||||||
|
Type string
|
||||||
|
Encoder string
|
||||||
|
}
|
||||||
|
|
||||||
|
var xdrEncoders = map[string]typeSet{
|
||||||
|
"int8": typeSet{"uint8", "Uint8"},
|
||||||
|
"uint8": typeSet{"", "Uint8"},
|
||||||
|
"int16": typeSet{"uint16", "Uint16"},
|
||||||
|
"uint16": typeSet{"", "Uint16"},
|
||||||
|
"int32": typeSet{"uint32", "Uint32"},
|
||||||
|
"uint32": typeSet{"", "Uint32"},
|
||||||
|
"int64": typeSet{"uint64", "Uint64"},
|
||||||
|
"uint64": typeSet{"", "Uint64"},
|
||||||
|
"int": typeSet{"uint64", "Uint64"},
|
||||||
|
"string": typeSet{"", "String"},
|
||||||
|
"[]byte": typeSet{"", "Bytes"},
|
||||||
|
"bool": typeSet{"", "Bool"},
|
||||||
|
}
|
||||||
|
|
||||||
|
func handleStruct(t *ast.StructType) []fieldInfo {
|
||||||
|
var fs []fieldInfo
|
||||||
|
|
||||||
|
for _, sf := range t.Fields.List {
|
||||||
|
if len(sf.Names) == 0 {
|
||||||
|
// We don't handle anonymous fields
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
fn := sf.Names[0].Name
|
||||||
|
var max1, max2 int
|
||||||
|
if sf.Comment != nil {
|
||||||
|
c := sf.Comment.List[0].Text
|
||||||
|
m := maxRe.FindStringSubmatch(c)
|
||||||
|
if len(m) >= 2 {
|
||||||
|
max1, _ = strconv.Atoi(m[1])
|
||||||
|
}
|
||||||
|
if len(m) >= 3 {
|
||||||
|
max2, _ = strconv.Atoi(m[2])
|
||||||
|
}
|
||||||
|
if strings.Contains(c, "noencode") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var f fieldInfo
|
||||||
|
switch ft := sf.Type.(type) {
|
||||||
|
case *ast.Ident:
|
||||||
|
tn := ft.Name
|
||||||
|
if enc, ok := xdrEncoders[tn]; ok {
|
||||||
|
f = fieldInfo{
|
||||||
|
Name: fn,
|
||||||
|
IsBasic: true,
|
||||||
|
FieldType: tn,
|
||||||
|
Encoder: enc.Encoder,
|
||||||
|
Convert: enc.Type,
|
||||||
|
Max: max1,
|
||||||
|
Submax: max2,
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
f = fieldInfo{
|
||||||
|
Name: fn,
|
||||||
|
IsBasic: false,
|
||||||
|
FieldType: tn,
|
||||||
|
Max: max1,
|
||||||
|
Submax: max2,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
case *ast.ArrayType:
|
||||||
|
if ft.Len != nil {
|
||||||
|
// We don't handle arrays
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
tn := ft.Elt.(*ast.Ident).Name
|
||||||
|
if enc, ok := xdrEncoders["[]"+tn]; ok {
|
||||||
|
f = fieldInfo{
|
||||||
|
Name: fn,
|
||||||
|
IsBasic: true,
|
||||||
|
FieldType: tn,
|
||||||
|
Encoder: enc.Encoder,
|
||||||
|
Convert: enc.Type,
|
||||||
|
Max: max1,
|
||||||
|
Submax: max2,
|
||||||
|
}
|
||||||
|
} else if enc, ok := xdrEncoders[tn]; ok {
|
||||||
|
f = fieldInfo{
|
||||||
|
Name: fn,
|
||||||
|
IsBasic: true,
|
||||||
|
IsSlice: true,
|
||||||
|
FieldType: tn,
|
||||||
|
Encoder: enc.Encoder,
|
||||||
|
Convert: enc.Type,
|
||||||
|
Max: max1,
|
||||||
|
Submax: max2,
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
f = fieldInfo{
|
||||||
|
Name: fn,
|
||||||
|
IsSlice: true,
|
||||||
|
FieldType: tn,
|
||||||
|
Max: max1,
|
||||||
|
Submax: max2,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
case *ast.SelectorExpr:
|
||||||
|
f = fieldInfo{
|
||||||
|
Name: fn,
|
||||||
|
FieldType: ft.Sel.Name,
|
||||||
|
Max: max1,
|
||||||
|
Submax: max2,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fs = append(fs, f)
|
||||||
|
}
|
||||||
|
|
||||||
|
return fs
|
||||||
|
}
|
||||||
|
|
||||||
|
func generateCode(output io.Writer, s structInfo) {
|
||||||
|
name := s.Name
|
||||||
|
fs := s.Fields
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
var err error
|
||||||
|
if len(fs) == 0 {
|
||||||
|
// This is an empty type. We can create a quite simple codec for it.
|
||||||
|
err = emptyTypeTpl.Execute(&buf, map[string]interface{}{"TypeName": name})
|
||||||
|
} else {
|
||||||
|
// Generate with the default template.
|
||||||
|
err = encodeTpl.Execute(&buf, map[string]interface{}{"TypeName": name, "Fields": fs})
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
bs := regexp.MustCompile(`(\s*\n)+`).ReplaceAll(buf.Bytes(), []byte("\n"))
|
||||||
|
bs = bytes.Replace(bs, []byte("//+n"), []byte("\n"), -1)
|
||||||
|
|
||||||
|
bs, err = format.Source(bs)
|
||||||
|
if err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
fmt.Fprintln(output, string(bs))
|
||||||
|
}
|
||||||
|
|
||||||
|
func uncamelize(s string) string {
|
||||||
|
return regexp.MustCompile("[a-z][A-Z]").ReplaceAllStringFunc(s, func(camel string) string {
|
||||||
|
return camel[:1] + " " + camel[1:]
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func generateDiagram(output io.Writer, s structInfo) {
|
||||||
|
sn := s.Name
|
||||||
|
fs := s.Fields
|
||||||
|
|
||||||
|
fmt.Fprintln(output, sn+" Structure:")
|
||||||
|
|
||||||
|
if len(fs) == 0 {
|
||||||
|
fmt.Fprintln(output, "(contains no fields)")
|
||||||
|
fmt.Fprintln(output)
|
||||||
|
fmt.Fprintln(output)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Fprintln(output)
|
||||||
|
fmt.Fprintln(output, " 0 1 2 3")
|
||||||
|
fmt.Fprintln(output, " 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1")
|
||||||
|
line := "+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+"
|
||||||
|
fmt.Fprintln(output, line)
|
||||||
|
|
||||||
|
for _, f := range fs {
|
||||||
|
tn := f.FieldType
|
||||||
|
name := uncamelize(f.Name)
|
||||||
|
|
||||||
|
if f.IsSlice {
|
||||||
|
fmt.Fprintf(output, "| %s |\n", center("Number of "+name, 61))
|
||||||
|
fmt.Fprintln(output, line)
|
||||||
|
}
|
||||||
|
switch tn {
|
||||||
|
case "bool":
|
||||||
|
fmt.Fprintf(output, "| %s |V|\n", center(name+" (V=0 or 1)", 59))
|
||||||
|
fmt.Fprintln(output, line)
|
||||||
|
case "int16", "uint16":
|
||||||
|
fmt.Fprintf(output, "| %s | %s |\n", center("0x0000", 29), center(name, 29))
|
||||||
|
fmt.Fprintln(output, line)
|
||||||
|
case "int32", "uint32":
|
||||||
|
fmt.Fprintf(output, "| %s |\n", center(name, 61))
|
||||||
|
fmt.Fprintln(output, line)
|
||||||
|
case "int64", "uint64":
|
||||||
|
fmt.Fprintf(output, "| %-61s |\n", "")
|
||||||
|
fmt.Fprintf(output, "+ %s +\n", center(name+" (64 bits)", 61))
|
||||||
|
fmt.Fprintf(output, "| %-61s |\n", "")
|
||||||
|
fmt.Fprintln(output, line)
|
||||||
|
case "string", "byte": // XXX We assume slice of byte!
|
||||||
|
fmt.Fprintf(output, "| %s |\n", center("Length of "+name, 61))
|
||||||
|
fmt.Fprintln(output, line)
|
||||||
|
fmt.Fprintf(output, "/ %61s /\n", "")
|
||||||
|
fmt.Fprintf(output, "\\ %s \\\n", center(name+" (variable length)", 61))
|
||||||
|
fmt.Fprintf(output, "/ %61s /\n", "")
|
||||||
|
fmt.Fprintln(output, line)
|
||||||
|
default:
|
||||||
|
if f.IsSlice {
|
||||||
|
tn = "Zero or more " + tn + " Structures"
|
||||||
|
fmt.Fprintf(output, "/ %s /\n", center("", 61))
|
||||||
|
fmt.Fprintf(output, "\\ %s \\\n", center(tn, 61))
|
||||||
|
fmt.Fprintf(output, "/ %s /\n", center("", 61))
|
||||||
|
} else {
|
||||||
|
tn = tn + " Structure"
|
||||||
|
fmt.Fprintf(output, "/ %s /\n", center("", 61))
|
||||||
|
fmt.Fprintf(output, "\\ %s \\\n", center(tn, 61))
|
||||||
|
fmt.Fprintf(output, "/ %s /\n", center("", 61))
|
||||||
|
}
|
||||||
|
fmt.Fprintln(output, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Fprintln(output)
|
||||||
|
fmt.Fprintln(output)
|
||||||
|
}
|
||||||
|
|
||||||
|
func generateXdr(output io.Writer, s structInfo) {
|
||||||
|
sn := s.Name
|
||||||
|
fs := s.Fields
|
||||||
|
|
||||||
|
fmt.Fprintf(output, "struct %s {\n", sn)
|
||||||
|
|
||||||
|
for _, f := range fs {
|
||||||
|
tn := f.FieldType
|
||||||
|
fn := f.Name
|
||||||
|
suf := ""
|
||||||
|
l := ""
|
||||||
|
if f.Max > 0 {
|
||||||
|
l = strconv.Itoa(f.Max)
|
||||||
|
}
|
||||||
|
if f.IsSlice {
|
||||||
|
suf = "<" + l + ">"
|
||||||
|
}
|
||||||
|
|
||||||
|
switch tn {
|
||||||
|
case "int16", "int32":
|
||||||
|
fmt.Fprintf(output, "\tint %s%s;\n", fn, suf)
|
||||||
|
case "uint16", "uint32":
|
||||||
|
fmt.Fprintf(output, "\tunsigned int %s%s;\n", fn, suf)
|
||||||
|
case "int64":
|
||||||
|
fmt.Fprintf(output, "\thyper %s%s;\n", fn, suf)
|
||||||
|
case "uint64":
|
||||||
|
fmt.Fprintf(output, "\tunsigned hyper %s%s;\n", fn, suf)
|
||||||
|
case "string":
|
||||||
|
fmt.Fprintf(output, "\tstring %s<%s>;\n", fn, l)
|
||||||
|
case "byte":
|
||||||
|
fmt.Fprintf(output, "\topaque %s<%s>;\n", fn, l)
|
||||||
|
default:
|
||||||
|
fmt.Fprintf(output, "\t%s %s%s;\n", tn, fn, suf)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Fprintln(output, "}")
|
||||||
|
fmt.Fprintln(output)
|
||||||
|
}
|
||||||
|
|
||||||
|
func center(s string, w int) string {
|
||||||
|
w -= len(s)
|
||||||
|
l := w / 2
|
||||||
|
r := l
|
||||||
|
if l+r < w {
|
||||||
|
r++
|
||||||
|
}
|
||||||
|
return strings.Repeat(" ", l) + s + strings.Repeat(" ", r)
|
||||||
|
}
|
||||||
|
|
||||||
|
func inspector(structs *[]structInfo) func(ast.Node) bool {
|
||||||
|
return func(n ast.Node) bool {
|
||||||
|
switch n := n.(type) {
|
||||||
|
case *ast.TypeSpec:
|
||||||
|
switch t := n.Type.(type) {
|
||||||
|
case *ast.StructType:
|
||||||
|
name := n.Name.Name
|
||||||
|
fs := handleStruct(t)
|
||||||
|
*structs = append(*structs, structInfo{name, fs})
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
default:
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
outputFile := flag.String("o", "", "Output file, blank for stdout")
|
||||||
|
flag.Parse()
|
||||||
|
fname := flag.Arg(0)
|
||||||
|
|
||||||
|
fset := token.NewFileSet()
|
||||||
|
f, err := parser.ParseFile(fset, fname, nil, parser.ParseComments)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var structs []structInfo
|
||||||
|
i := inspector(&structs)
|
||||||
|
ast.Inspect(f, i)
|
||||||
|
|
||||||
|
var output io.Writer = os.Stdout
|
||||||
|
if *outputFile != "" {
|
||||||
|
fd, err := os.Create(*outputFile)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
output = fd
|
||||||
|
}
|
||||||
|
|
||||||
|
headerTpl.Execute(output, map[string]string{"Package": f.Name.Name})
|
||||||
|
for _, s := range structs {
|
||||||
|
fmt.Fprintf(output, "\n/*\n\n")
|
||||||
|
generateDiagram(output, s)
|
||||||
|
generateXdr(output, s)
|
||||||
|
fmt.Fprintf(output, "*/\n")
|
||||||
|
generateCode(output, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
+16
@@ -0,0 +1,16 @@
|
|||||||
|
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
|
||||||
|
// is governed by an MIT-style license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package xdr
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
debug = len(os.Getenv("XDRTRACE")) > 0
|
||||||
|
dl = log.New(os.Stdout, "xdr: ", log.Lshortfile|log.Ltime|log.Lmicroseconds)
|
||||||
|
)
|
||||||
|
|
||||||
|
const maxDebugBytes = 32
|
||||||
+5
@@ -0,0 +1,5 @@
|
|||||||
|
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
|
||||||
|
// is governed by an MIT-style license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// Package xdr implements an XDR (RFC 4506) encoder/decoder.
|
||||||
|
package xdr
|
||||||
+4
@@ -0,0 +1,4 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
go run cmd/genxdr/main.go -- bench_test.go > bench_xdr_test.go
|
||||||
|
go run cmd/genxdr/main.go -- encdec_test.go > encdec_xdr_test.go
|
||||||
+10
@@ -0,0 +1,10 @@
|
|||||||
|
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
|
||||||
|
// is governed by an MIT-style license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// +build ipdr
|
||||||
|
|
||||||
|
package xdr
|
||||||
|
|
||||||
|
func pad(l int) int {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
+14
@@ -0,0 +1,14 @@
|
|||||||
|
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
|
||||||
|
// is governed by an MIT-style license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// +build !ipdr
|
||||||
|
|
||||||
|
package xdr
|
||||||
|
|
||||||
|
func pad(l int) int {
|
||||||
|
d := l % 4
|
||||||
|
if d == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return 4 - d
|
||||||
|
}
|
||||||
+171
@@ -0,0 +1,171 @@
|
|||||||
|
// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
|
||||||
|
// All rights reserved. Use of this source code is governed by an MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package xdr
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"reflect"
|
||||||
|
"unsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Reader struct {
|
||||||
|
r io.Reader
|
||||||
|
err error
|
||||||
|
b [8]byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewReader(r io.Reader) *Reader {
|
||||||
|
return &Reader{
|
||||||
|
r: r,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reader) ReadRaw(bs []byte) (int, error) {
|
||||||
|
if r.err != nil {
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
|
||||||
|
var n int
|
||||||
|
n, r.err = io.ReadFull(r.r, bs)
|
||||||
|
return n, r.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reader) ReadString() string {
|
||||||
|
return r.ReadStringMax(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reader) ReadStringMax(max int) string {
|
||||||
|
buf := r.ReadBytesMaxInto(max, nil)
|
||||||
|
bh := (*reflect.SliceHeader)(unsafe.Pointer(&buf))
|
||||||
|
sh := reflect.StringHeader{
|
||||||
|
Data: bh.Data,
|
||||||
|
Len: bh.Len,
|
||||||
|
}
|
||||||
|
return *((*string)(unsafe.Pointer(&sh)))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reader) ReadBytes() []byte {
|
||||||
|
return r.ReadBytesInto(nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reader) ReadBytesMax(max int) []byte {
|
||||||
|
return r.ReadBytesMaxInto(max, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reader) ReadBytesInto(dst []byte) []byte {
|
||||||
|
return r.ReadBytesMaxInto(0, dst)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reader) ReadBytesMaxInto(max int, dst []byte) []byte {
|
||||||
|
if r.err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
l := int(r.ReadUint32())
|
||||||
|
if r.err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if l < 0 || max > 0 && l > max {
|
||||||
|
// l may be negative on 32 bit builds
|
||||||
|
r.err = ElementSizeExceeded("bytes field", l, max)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if fullLen := l + pad(l); fullLen > len(dst) {
|
||||||
|
dst = make([]byte, fullLen)
|
||||||
|
} else {
|
||||||
|
dst = dst[:fullLen]
|
||||||
|
}
|
||||||
|
|
||||||
|
var n int
|
||||||
|
n, r.err = io.ReadFull(r.r, dst)
|
||||||
|
if r.err != nil {
|
||||||
|
if debug {
|
||||||
|
dl.Printf("rd bytes (%d): %v", len(dst), r.err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if debug {
|
||||||
|
if n > maxDebugBytes {
|
||||||
|
dl.Printf("rd bytes (%d): %x...", len(dst), dst[:maxDebugBytes])
|
||||||
|
} else {
|
||||||
|
dl.Printf("rd bytes (%d): %x", len(dst), dst)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return dst[:l]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reader) ReadBool() bool {
|
||||||
|
return r.ReadUint8() != 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reader) ReadUint32() uint32 {
|
||||||
|
if r.err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
_, r.err = io.ReadFull(r.r, r.b[:4])
|
||||||
|
if r.err != nil {
|
||||||
|
if debug {
|
||||||
|
dl.Printf("rd uint32: %v", r.err)
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
v := uint32(r.b[3]) | uint32(r.b[2])<<8 | uint32(r.b[1])<<16 | uint32(r.b[0])<<24
|
||||||
|
|
||||||
|
if debug {
|
||||||
|
dl.Printf("rd uint32=%d (0x%08x)", v, v)
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reader) ReadUint64() uint64 {
|
||||||
|
if r.err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
_, r.err = io.ReadFull(r.r, r.b[:8])
|
||||||
|
if r.err != nil {
|
||||||
|
if debug {
|
||||||
|
dl.Printf("rd uint64: %v", r.err)
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
v := uint64(r.b[7]) | uint64(r.b[6])<<8 | uint64(r.b[5])<<16 | uint64(r.b[4])<<24 |
|
||||||
|
uint64(r.b[3])<<32 | uint64(r.b[2])<<40 | uint64(r.b[1])<<48 | uint64(r.b[0])<<56
|
||||||
|
|
||||||
|
if debug {
|
||||||
|
dl.Printf("rd uint64=%d (0x%016x)", v, v)
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
type XDRError struct {
|
||||||
|
op string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e XDRError) Error() string {
|
||||||
|
return "xdr " + e.op + ": " + e.err.Error()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e XDRError) IsEOF() bool {
|
||||||
|
return e.err == io.EOF
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reader) Error() error {
|
||||||
|
if r.err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return XDRError{"read", r.err}
|
||||||
|
}
|
||||||
|
|
||||||
|
func ElementSizeExceeded(field string, size, limit int) error {
|
||||||
|
return fmt.Errorf("%s exceeds size limit; %d > %d", field, size, limit)
|
||||||
|
}
|
||||||
+49
@@ -0,0 +1,49 @@
|
|||||||
|
// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
|
||||||
|
// All rights reserved. Use of this source code is governed by an MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// +build ipdr
|
||||||
|
|
||||||
|
package xdr
|
||||||
|
|
||||||
|
import "io"
|
||||||
|
|
||||||
|
func (r *Reader) ReadUint8() uint8 {
|
||||||
|
if r.err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
_, r.err = io.ReadFull(r.r, r.b[:1])
|
||||||
|
if r.err != nil {
|
||||||
|
if debug {
|
||||||
|
dl.Printf("rd uint8: %v", r.err)
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
if debug {
|
||||||
|
dl.Printf("rd uint8=%d (0x%02x)", r.b[0], r.b[0])
|
||||||
|
}
|
||||||
|
return r.b[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reader) ReadUint16() uint16 {
|
||||||
|
if r.err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
_, r.err = io.ReadFull(r.r, r.b[:2])
|
||||||
|
if r.err != nil {
|
||||||
|
if debug {
|
||||||
|
dl.Printf("rd uint16: %v", r.err)
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
v := uint16(r.b[1]) | uint16(r.b[0])<<8
|
||||||
|
|
||||||
|
if debug {
|
||||||
|
dl.Printf("rd uint16=%d (0x%04x)", v, v)
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
+15
@@ -0,0 +1,15 @@
|
|||||||
|
// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
|
||||||
|
// All rights reserved. Use of this source code is governed by an MIT-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// +build !ipdr
|
||||||
|
|
||||||
|
package xdr
|
||||||
|
|
||||||
|
func (r *Reader) ReadUint8() uint8 {
|
||||||
|
return uint8(r.ReadUint32())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reader) ReadUint16() uint16 {
|
||||||
|
return uint16(r.ReadUint32())
|
||||||
|
}
|
||||||
+146
@@ -0,0 +1,146 @@
|
|||||||
|
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
|
||||||
|
// is governed by an MIT-style license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package xdr
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"reflect"
|
||||||
|
"unsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
var padBytes = []byte{0, 0, 0}
|
||||||
|
|
||||||
|
type Writer struct {
|
||||||
|
w io.Writer
|
||||||
|
tot int
|
||||||
|
err error
|
||||||
|
b [8]byte
|
||||||
|
}
|
||||||
|
|
||||||
|
type AppendWriter []byte
|
||||||
|
|
||||||
|
func (w *AppendWriter) Write(bs []byte) (int, error) {
|
||||||
|
*w = append(*w, bs...)
|
||||||
|
return len(bs), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewWriter(w io.Writer) *Writer {
|
||||||
|
return &Writer{
|
||||||
|
w: w,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) WriteRaw(bs []byte) (int, error) {
|
||||||
|
if w.err != nil {
|
||||||
|
return 0, w.err
|
||||||
|
}
|
||||||
|
|
||||||
|
var n int
|
||||||
|
n, w.err = w.w.Write(bs)
|
||||||
|
return n, w.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) WriteString(s string) (int, error) {
|
||||||
|
sh := *((*reflect.StringHeader)(unsafe.Pointer(&s)))
|
||||||
|
bh := reflect.SliceHeader{
|
||||||
|
Data: sh.Data,
|
||||||
|
Len: sh.Len,
|
||||||
|
Cap: sh.Len,
|
||||||
|
}
|
||||||
|
return w.WriteBytes(*(*[]byte)(unsafe.Pointer(&bh)))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) WriteBytes(bs []byte) (int, error) {
|
||||||
|
if w.err != nil {
|
||||||
|
return 0, w.err
|
||||||
|
}
|
||||||
|
|
||||||
|
w.WriteUint32(uint32(len(bs)))
|
||||||
|
if w.err != nil {
|
||||||
|
return 0, w.err
|
||||||
|
}
|
||||||
|
|
||||||
|
if debug {
|
||||||
|
if len(bs) > maxDebugBytes {
|
||||||
|
dl.Printf("wr bytes (%d): %x...", len(bs), bs[:maxDebugBytes])
|
||||||
|
} else {
|
||||||
|
dl.Printf("wr bytes (%d): %x", len(bs), bs)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var l, n int
|
||||||
|
n, w.err = w.w.Write(bs)
|
||||||
|
l += n
|
||||||
|
|
||||||
|
if p := pad(len(bs)); w.err == nil && p > 0 {
|
||||||
|
n, w.err = w.w.Write(padBytes[:p])
|
||||||
|
l += n
|
||||||
|
}
|
||||||
|
|
||||||
|
w.tot += l
|
||||||
|
return l, w.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) WriteBool(v bool) (int, error) {
|
||||||
|
if v {
|
||||||
|
return w.WriteUint8(1)
|
||||||
|
} else {
|
||||||
|
return w.WriteUint8(0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) WriteUint32(v uint32) (int, error) {
|
||||||
|
if w.err != nil {
|
||||||
|
return 0, w.err
|
||||||
|
}
|
||||||
|
|
||||||
|
if debug {
|
||||||
|
dl.Printf("wr uint32=%d", v)
|
||||||
|
}
|
||||||
|
|
||||||
|
w.b[0] = byte(v >> 24)
|
||||||
|
w.b[1] = byte(v >> 16)
|
||||||
|
w.b[2] = byte(v >> 8)
|
||||||
|
w.b[3] = byte(v)
|
||||||
|
|
||||||
|
var l int
|
||||||
|
l, w.err = w.w.Write(w.b[:4])
|
||||||
|
w.tot += l
|
||||||
|
return l, w.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) WriteUint64(v uint64) (int, error) {
|
||||||
|
if w.err != nil {
|
||||||
|
return 0, w.err
|
||||||
|
}
|
||||||
|
|
||||||
|
if debug {
|
||||||
|
dl.Printf("wr uint64=%d", v)
|
||||||
|
}
|
||||||
|
|
||||||
|
w.b[0] = byte(v >> 56)
|
||||||
|
w.b[1] = byte(v >> 48)
|
||||||
|
w.b[2] = byte(v >> 40)
|
||||||
|
w.b[3] = byte(v >> 32)
|
||||||
|
w.b[4] = byte(v >> 24)
|
||||||
|
w.b[5] = byte(v >> 16)
|
||||||
|
w.b[6] = byte(v >> 8)
|
||||||
|
w.b[7] = byte(v)
|
||||||
|
|
||||||
|
var l int
|
||||||
|
l, w.err = w.w.Write(w.b[:8])
|
||||||
|
w.tot += l
|
||||||
|
return l, w.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) Tot() int {
|
||||||
|
return w.tot
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) Error() error {
|
||||||
|
if w.err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return XDRError{"write", w.err}
|
||||||
|
}
|
||||||
+41
@@ -0,0 +1,41 @@
|
|||||||
|
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
|
||||||
|
// is governed by an MIT-style license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// +build ipdr
|
||||||
|
|
||||||
|
package xdr
|
||||||
|
|
||||||
|
func (w *Writer) WriteUint8(v uint8) (int, error) {
|
||||||
|
if w.err != nil {
|
||||||
|
return 0, w.err
|
||||||
|
}
|
||||||
|
|
||||||
|
if debug {
|
||||||
|
dl.Printf("wr uint8=%d", v)
|
||||||
|
}
|
||||||
|
|
||||||
|
w.b[0] = byte(v)
|
||||||
|
|
||||||
|
var l int
|
||||||
|
l, w.err = w.w.Write(w.b[:1])
|
||||||
|
w.tot += l
|
||||||
|
return l, w.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) WriteUint16(v uint16) (int, error) {
|
||||||
|
if w.err != nil {
|
||||||
|
return 0, w.err
|
||||||
|
}
|
||||||
|
|
||||||
|
if debug {
|
||||||
|
dl.Printf("wr uint8=%d", v)
|
||||||
|
}
|
||||||
|
|
||||||
|
w.b[0] = byte(v >> 8)
|
||||||
|
w.b[1] = byte(v)
|
||||||
|
|
||||||
|
var l int
|
||||||
|
l, w.err = w.w.Write(w.b[:2])
|
||||||
|
w.tot += l
|
||||||
|
return l, w.err
|
||||||
|
}
|
||||||
+14
@@ -0,0 +1,14 @@
|
|||||||
|
// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
|
||||||
|
// is governed by an MIT-style license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// +build !ipdr
|
||||||
|
|
||||||
|
package xdr
|
||||||
|
|
||||||
|
func (w *Writer) WriteUint8(v uint8) (int, error) {
|
||||||
|
return w.WriteUint32(uint32(v))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Writer) WriteUint16(v uint16) (int, error) {
|
||||||
|
return w.WriteUint32(uint32(v))
|
||||||
|
}
|
||||||
+14
@@ -0,0 +1,14 @@
|
|||||||
|
# This is the official list of Snappy-Go authors for copyright purposes.
|
||||||
|
# This file is distinct from the CONTRIBUTORS files.
|
||||||
|
# See the latter for an explanation.
|
||||||
|
|
||||||
|
# Names should be added to this file as
|
||||||
|
# Name or Organization <email address>
|
||||||
|
# The email address is not required for organizations.
|
||||||
|
|
||||||
|
# Please keep the list sorted.
|
||||||
|
|
||||||
|
Damian Gryski <dgryski@gmail.com>
|
||||||
|
Google Inc.
|
||||||
|
Jan Mercl <0xjnml@gmail.com>
|
||||||
|
Sebastien Binet <seb.binet@gmail.com>
|
||||||
+36
@@ -0,0 +1,36 @@
|
|||||||
|
# This is the official list of people who can contribute
|
||||||
|
# (and typically have contributed) code to the Snappy-Go repository.
|
||||||
|
# The AUTHORS file lists the copyright holders; this file
|
||||||
|
# lists people. For example, Google employees are listed here
|
||||||
|
# but not in AUTHORS, because Google holds the copyright.
|
||||||
|
#
|
||||||
|
# The submission process automatically checks to make sure
|
||||||
|
# that people submitting code are listed in this file (by email address).
|
||||||
|
#
|
||||||
|
# Names should be added to this file only after verifying that
|
||||||
|
# the individual or the individual's organization has agreed to
|
||||||
|
# the appropriate Contributor License Agreement, found here:
|
||||||
|
#
|
||||||
|
# http://code.google.com/legal/individual-cla-v1.0.html
|
||||||
|
# http://code.google.com/legal/corporate-cla-v1.0.html
|
||||||
|
#
|
||||||
|
# The agreement for individuals can be filled out on the web.
|
||||||
|
#
|
||||||
|
# When adding J Random Contributor's name to this file,
|
||||||
|
# either J's name or J's organization's name should be
|
||||||
|
# added to the AUTHORS file, depending on whether the
|
||||||
|
# individual or corporate CLA was used.
|
||||||
|
|
||||||
|
# Names should be added to this file like so:
|
||||||
|
# Name <email address>
|
||||||
|
|
||||||
|
# Please keep the list sorted.
|
||||||
|
|
||||||
|
Damian Gryski <dgryski@gmail.com>
|
||||||
|
Jan Mercl <0xjnml@gmail.com>
|
||||||
|
Kai Backman <kaib@golang.org>
|
||||||
|
Marc-Antoine Ruel <maruel@chromium.org>
|
||||||
|
Nigel Tao <nigeltao@golang.org>
|
||||||
|
Rob Pike <r@golang.org>
|
||||||
|
Russ Cox <rsc@golang.org>
|
||||||
|
Sebastien Binet <seb.binet@gmail.com>
|
||||||
+27
@@ -0,0 +1,27 @@
|
|||||||
|
Copyright (c) 2011 The Snappy-Go Authors. All rights reserved.
|
||||||
|
|
||||||
|
Redistribution and use in source and binary forms, with or without
|
||||||
|
modification, are permitted provided that the following conditions are
|
||||||
|
met:
|
||||||
|
|
||||||
|
* Redistributions of source code must retain the above copyright
|
||||||
|
notice, this list of conditions and the following disclaimer.
|
||||||
|
* Redistributions in binary form must reproduce the above
|
||||||
|
copyright notice, this list of conditions and the following disclaimer
|
||||||
|
in the documentation and/or other materials provided with the
|
||||||
|
distribution.
|
||||||
|
* Neither the name of Google Inc. nor the names of its
|
||||||
|
contributors may be used to endorse or promote products derived from
|
||||||
|
this software without specific prior written permission.
|
||||||
|
|
||||||
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||||
|
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||||
|
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||||
|
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||||
|
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||||
|
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||||
|
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
+7
@@ -0,0 +1,7 @@
|
|||||||
|
The Snappy compression format in the Go programming language.
|
||||||
|
|
||||||
|
To download and install from source:
|
||||||
|
$ go get github.com/golang/snappy
|
||||||
|
|
||||||
|
Unless otherwise noted, the Snappy-Go source files are distributed
|
||||||
|
under the BSD-style license found in the LICENSE file.
|
||||||
+294
@@ -0,0 +1,294 @@
|
|||||||
|
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package snappy
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/binary"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
// ErrCorrupt reports that the input is invalid.
|
||||||
|
ErrCorrupt = errors.New("snappy: corrupt input")
|
||||||
|
// ErrTooLarge reports that the uncompressed length is too large.
|
||||||
|
ErrTooLarge = errors.New("snappy: decoded block is too large")
|
||||||
|
// ErrUnsupported reports that the input isn't supported.
|
||||||
|
ErrUnsupported = errors.New("snappy: unsupported input")
|
||||||
|
)
|
||||||
|
|
||||||
|
// DecodedLen returns the length of the decoded block.
|
||||||
|
func DecodedLen(src []byte) (int, error) {
|
||||||
|
v, _, err := decodedLen(src)
|
||||||
|
return v, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// decodedLen returns the length of the decoded block and the number of bytes
|
||||||
|
// that the length header occupied.
|
||||||
|
func decodedLen(src []byte) (blockLen, headerLen int, err error) {
|
||||||
|
v, n := binary.Uvarint(src)
|
||||||
|
if n <= 0 || v > 0xffffffff {
|
||||||
|
return 0, 0, ErrCorrupt
|
||||||
|
}
|
||||||
|
|
||||||
|
const wordSize = 32 << (^uint(0) >> 32 & 1)
|
||||||
|
if wordSize == 32 && v > 0x7fffffff {
|
||||||
|
return 0, 0, ErrTooLarge
|
||||||
|
}
|
||||||
|
return int(v), n, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decode returns the decoded form of src. The returned slice may be a sub-
|
||||||
|
// slice of dst if dst was large enough to hold the entire decoded block.
|
||||||
|
// Otherwise, a newly allocated slice will be returned.
|
||||||
|
// It is valid to pass a nil dst.
|
||||||
|
func Decode(dst, src []byte) ([]byte, error) {
|
||||||
|
dLen, s, err := decodedLen(src)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(dst) < dLen {
|
||||||
|
dst = make([]byte, dLen)
|
||||||
|
}
|
||||||
|
|
||||||
|
var d, offset, length int
|
||||||
|
for s < len(src) {
|
||||||
|
switch src[s] & 0x03 {
|
||||||
|
case tagLiteral:
|
||||||
|
x := uint(src[s] >> 2)
|
||||||
|
switch {
|
||||||
|
case x < 60:
|
||||||
|
s++
|
||||||
|
case x == 60:
|
||||||
|
s += 2
|
||||||
|
if s > len(src) {
|
||||||
|
return nil, ErrCorrupt
|
||||||
|
}
|
||||||
|
x = uint(src[s-1])
|
||||||
|
case x == 61:
|
||||||
|
s += 3
|
||||||
|
if s > len(src) {
|
||||||
|
return nil, ErrCorrupt
|
||||||
|
}
|
||||||
|
x = uint(src[s-2]) | uint(src[s-1])<<8
|
||||||
|
case x == 62:
|
||||||
|
s += 4
|
||||||
|
if s > len(src) {
|
||||||
|
return nil, ErrCorrupt
|
||||||
|
}
|
||||||
|
x = uint(src[s-3]) | uint(src[s-2])<<8 | uint(src[s-1])<<16
|
||||||
|
case x == 63:
|
||||||
|
s += 5
|
||||||
|
if s > len(src) {
|
||||||
|
return nil, ErrCorrupt
|
||||||
|
}
|
||||||
|
x = uint(src[s-4]) | uint(src[s-3])<<8 | uint(src[s-2])<<16 | uint(src[s-1])<<24
|
||||||
|
}
|
||||||
|
length = int(x + 1)
|
||||||
|
if length <= 0 {
|
||||||
|
return nil, errors.New("snappy: unsupported literal length")
|
||||||
|
}
|
||||||
|
if length > len(dst)-d || length > len(src)-s {
|
||||||
|
return nil, ErrCorrupt
|
||||||
|
}
|
||||||
|
copy(dst[d:], src[s:s+length])
|
||||||
|
d += length
|
||||||
|
s += length
|
||||||
|
continue
|
||||||
|
|
||||||
|
case tagCopy1:
|
||||||
|
s += 2
|
||||||
|
if s > len(src) {
|
||||||
|
return nil, ErrCorrupt
|
||||||
|
}
|
||||||
|
length = 4 + int(src[s-2])>>2&0x7
|
||||||
|
offset = int(src[s-2])&0xe0<<3 | int(src[s-1])
|
||||||
|
|
||||||
|
case tagCopy2:
|
||||||
|
s += 3
|
||||||
|
if s > len(src) {
|
||||||
|
return nil, ErrCorrupt
|
||||||
|
}
|
||||||
|
length = 1 + int(src[s-3])>>2
|
||||||
|
offset = int(src[s-2]) | int(src[s-1])<<8
|
||||||
|
|
||||||
|
case tagCopy4:
|
||||||
|
return nil, errors.New("snappy: unsupported COPY_4 tag")
|
||||||
|
}
|
||||||
|
|
||||||
|
end := d + length
|
||||||
|
if offset > d || end > len(dst) {
|
||||||
|
return nil, ErrCorrupt
|
||||||
|
}
|
||||||
|
for ; d < end; d++ {
|
||||||
|
dst[d] = dst[d-offset]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if d != dLen {
|
||||||
|
return nil, ErrCorrupt
|
||||||
|
}
|
||||||
|
return dst[:d], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewReader returns a new Reader that decompresses from r, using the framing
|
||||||
|
// format described at
|
||||||
|
// https://github.com/google/snappy/blob/master/framing_format.txt
|
||||||
|
func NewReader(r io.Reader) *Reader {
|
||||||
|
return &Reader{
|
||||||
|
r: r,
|
||||||
|
decoded: make([]byte, maxUncompressedChunkLen),
|
||||||
|
buf: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)+checksumSize),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reader is an io.Reader than can read Snappy-compressed bytes.
|
||||||
|
type Reader struct {
|
||||||
|
r io.Reader
|
||||||
|
err error
|
||||||
|
decoded []byte
|
||||||
|
buf []byte
|
||||||
|
// decoded[i:j] contains decoded bytes that have not yet been passed on.
|
||||||
|
i, j int
|
||||||
|
readHeader bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reset discards any buffered data, resets all state, and switches the Snappy
|
||||||
|
// reader to read from r. This permits reusing a Reader rather than allocating
|
||||||
|
// a new one.
|
||||||
|
func (r *Reader) Reset(reader io.Reader) {
|
||||||
|
r.r = reader
|
||||||
|
r.err = nil
|
||||||
|
r.i = 0
|
||||||
|
r.j = 0
|
||||||
|
r.readHeader = false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Reader) readFull(p []byte) (ok bool) {
|
||||||
|
if _, r.err = io.ReadFull(r.r, p); r.err != nil {
|
||||||
|
if r.err == io.ErrUnexpectedEOF {
|
||||||
|
r.err = ErrCorrupt
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read satisfies the io.Reader interface.
|
||||||
|
func (r *Reader) Read(p []byte) (int, error) {
|
||||||
|
if r.err != nil {
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
if r.i < r.j {
|
||||||
|
n := copy(p, r.decoded[r.i:r.j])
|
||||||
|
r.i += n
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
if !r.readFull(r.buf[:4]) {
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
chunkType := r.buf[0]
|
||||||
|
if !r.readHeader {
|
||||||
|
if chunkType != chunkTypeStreamIdentifier {
|
||||||
|
r.err = ErrCorrupt
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
r.readHeader = true
|
||||||
|
}
|
||||||
|
chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16
|
||||||
|
if chunkLen > len(r.buf) {
|
||||||
|
r.err = ErrUnsupported
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
|
||||||
|
// The chunk types are specified at
|
||||||
|
// https://github.com/google/snappy/blob/master/framing_format.txt
|
||||||
|
switch chunkType {
|
||||||
|
case chunkTypeCompressedData:
|
||||||
|
// Section 4.2. Compressed data (chunk type 0x00).
|
||||||
|
if chunkLen < checksumSize {
|
||||||
|
r.err = ErrCorrupt
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
buf := r.buf[:chunkLen]
|
||||||
|
if !r.readFull(buf) {
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
|
||||||
|
buf = buf[checksumSize:]
|
||||||
|
|
||||||
|
n, err := DecodedLen(buf)
|
||||||
|
if err != nil {
|
||||||
|
r.err = err
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
if n > len(r.decoded) {
|
||||||
|
r.err = ErrCorrupt
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
if _, err := Decode(r.decoded, buf); err != nil {
|
||||||
|
r.err = err
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
if crc(r.decoded[:n]) != checksum {
|
||||||
|
r.err = ErrCorrupt
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
r.i, r.j = 0, n
|
||||||
|
continue
|
||||||
|
|
||||||
|
case chunkTypeUncompressedData:
|
||||||
|
// Section 4.3. Uncompressed data (chunk type 0x01).
|
||||||
|
if chunkLen < checksumSize {
|
||||||
|
r.err = ErrCorrupt
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
buf := r.buf[:checksumSize]
|
||||||
|
if !r.readFull(buf) {
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
|
||||||
|
// Read directly into r.decoded instead of via r.buf.
|
||||||
|
n := chunkLen - checksumSize
|
||||||
|
if !r.readFull(r.decoded[:n]) {
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
if crc(r.decoded[:n]) != checksum {
|
||||||
|
r.err = ErrCorrupt
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
r.i, r.j = 0, n
|
||||||
|
continue
|
||||||
|
|
||||||
|
case chunkTypeStreamIdentifier:
|
||||||
|
// Section 4.1. Stream identifier (chunk type 0xff).
|
||||||
|
if chunkLen != len(magicBody) {
|
||||||
|
r.err = ErrCorrupt
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
if !r.readFull(r.buf[:len(magicBody)]) {
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
for i := 0; i < len(magicBody); i++ {
|
||||||
|
if r.buf[i] != magicBody[i] {
|
||||||
|
r.err = ErrCorrupt
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if chunkType <= 0x7f {
|
||||||
|
// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
|
||||||
|
r.err = ErrUnsupported
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
// Section 4.4 Padding (chunk type 0xfe).
|
||||||
|
// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
|
||||||
|
if !r.readFull(r.buf[:chunkLen]) {
|
||||||
|
return 0, r.err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+254
@@ -0,0 +1,254 @@
|
|||||||
|
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package snappy
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/binary"
|
||||||
|
"io"
|
||||||
|
)
|
||||||
|
|
||||||
|
// We limit how far copy back-references can go, the same as the C++ code.
|
||||||
|
const maxOffset = 1 << 15
|
||||||
|
|
||||||
|
// emitLiteral writes a literal chunk and returns the number of bytes written.
|
||||||
|
func emitLiteral(dst, lit []byte) int {
|
||||||
|
i, n := 0, uint(len(lit)-1)
|
||||||
|
switch {
|
||||||
|
case n < 60:
|
||||||
|
dst[0] = uint8(n)<<2 | tagLiteral
|
||||||
|
i = 1
|
||||||
|
case n < 1<<8:
|
||||||
|
dst[0] = 60<<2 | tagLiteral
|
||||||
|
dst[1] = uint8(n)
|
||||||
|
i = 2
|
||||||
|
case n < 1<<16:
|
||||||
|
dst[0] = 61<<2 | tagLiteral
|
||||||
|
dst[1] = uint8(n)
|
||||||
|
dst[2] = uint8(n >> 8)
|
||||||
|
i = 3
|
||||||
|
case n < 1<<24:
|
||||||
|
dst[0] = 62<<2 | tagLiteral
|
||||||
|
dst[1] = uint8(n)
|
||||||
|
dst[2] = uint8(n >> 8)
|
||||||
|
dst[3] = uint8(n >> 16)
|
||||||
|
i = 4
|
||||||
|
case int64(n) < 1<<32:
|
||||||
|
dst[0] = 63<<2 | tagLiteral
|
||||||
|
dst[1] = uint8(n)
|
||||||
|
dst[2] = uint8(n >> 8)
|
||||||
|
dst[3] = uint8(n >> 16)
|
||||||
|
dst[4] = uint8(n >> 24)
|
||||||
|
i = 5
|
||||||
|
default:
|
||||||
|
panic("snappy: source buffer is too long")
|
||||||
|
}
|
||||||
|
if copy(dst[i:], lit) != len(lit) {
|
||||||
|
panic("snappy: destination buffer is too short")
|
||||||
|
}
|
||||||
|
return i + len(lit)
|
||||||
|
}
|
||||||
|
|
||||||
|
// emitCopy writes a copy chunk and returns the number of bytes written.
|
||||||
|
func emitCopy(dst []byte, offset, length int) int {
|
||||||
|
i := 0
|
||||||
|
for length > 0 {
|
||||||
|
x := length - 4
|
||||||
|
if 0 <= x && x < 1<<3 && offset < 1<<11 {
|
||||||
|
dst[i+0] = uint8(offset>>8)&0x07<<5 | uint8(x)<<2 | tagCopy1
|
||||||
|
dst[i+1] = uint8(offset)
|
||||||
|
i += 2
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
x = length
|
||||||
|
if x > 1<<6 {
|
||||||
|
x = 1 << 6
|
||||||
|
}
|
||||||
|
dst[i+0] = uint8(x-1)<<2 | tagCopy2
|
||||||
|
dst[i+1] = uint8(offset)
|
||||||
|
dst[i+2] = uint8(offset >> 8)
|
||||||
|
i += 3
|
||||||
|
length -= x
|
||||||
|
}
|
||||||
|
return i
|
||||||
|
}
|
||||||
|
|
||||||
|
// Encode returns the encoded form of src. The returned slice may be a sub-
|
||||||
|
// slice of dst if dst was large enough to hold the entire encoded block.
|
||||||
|
// Otherwise, a newly allocated slice will be returned.
|
||||||
|
// It is valid to pass a nil dst.
|
||||||
|
func Encode(dst, src []byte) []byte {
|
||||||
|
if n := MaxEncodedLen(len(src)); len(dst) < n {
|
||||||
|
dst = make([]byte, n)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||||
|
d := binary.PutUvarint(dst, uint64(len(src)))
|
||||||
|
|
||||||
|
// Return early if src is short.
|
||||||
|
if len(src) <= 4 {
|
||||||
|
if len(src) != 0 {
|
||||||
|
d += emitLiteral(dst[d:], src)
|
||||||
|
}
|
||||||
|
return dst[:d]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive.
|
||||||
|
const maxTableSize = 1 << 14
|
||||||
|
shift, tableSize := uint(32-8), 1<<8
|
||||||
|
for tableSize < maxTableSize && tableSize < len(src) {
|
||||||
|
shift--
|
||||||
|
tableSize *= 2
|
||||||
|
}
|
||||||
|
var table [maxTableSize]int
|
||||||
|
|
||||||
|
// Iterate over the source bytes.
|
||||||
|
var (
|
||||||
|
s int // The iterator position.
|
||||||
|
t int // The last position with the same hash as s.
|
||||||
|
lit int // The start position of any pending literal bytes.
|
||||||
|
)
|
||||||
|
for s+3 < len(src) {
|
||||||
|
// Update the hash table.
|
||||||
|
b0, b1, b2, b3 := src[s], src[s+1], src[s+2], src[s+3]
|
||||||
|
h := uint32(b0) | uint32(b1)<<8 | uint32(b2)<<16 | uint32(b3)<<24
|
||||||
|
p := &table[(h*0x1e35a7bd)>>shift]
|
||||||
|
// We need to to store values in [-1, inf) in table. To save
|
||||||
|
// some initialization time, (re)use the table's zero value
|
||||||
|
// and shift the values against this zero: add 1 on writes,
|
||||||
|
// subtract 1 on reads.
|
||||||
|
t, *p = *p-1, s+1
|
||||||
|
// If t is invalid or src[s:s+4] differs from src[t:t+4], accumulate a literal byte.
|
||||||
|
if t < 0 || s-t >= maxOffset || b0 != src[t] || b1 != src[t+1] || b2 != src[t+2] || b3 != src[t+3] {
|
||||||
|
s++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Otherwise, we have a match. First, emit any pending literal bytes.
|
||||||
|
if lit != s {
|
||||||
|
d += emitLiteral(dst[d:], src[lit:s])
|
||||||
|
}
|
||||||
|
// Extend the match to be as long as possible.
|
||||||
|
s0 := s
|
||||||
|
s, t = s+4, t+4
|
||||||
|
for s < len(src) && src[s] == src[t] {
|
||||||
|
s++
|
||||||
|
t++
|
||||||
|
}
|
||||||
|
// Emit the copied bytes.
|
||||||
|
d += emitCopy(dst[d:], s-t, s-s0)
|
||||||
|
lit = s
|
||||||
|
}
|
||||||
|
|
||||||
|
// Emit any final pending literal bytes and return.
|
||||||
|
if lit != len(src) {
|
||||||
|
d += emitLiteral(dst[d:], src[lit:])
|
||||||
|
}
|
||||||
|
return dst[:d]
|
||||||
|
}
|
||||||
|
|
||||||
|
// MaxEncodedLen returns the maximum length of a snappy block, given its
|
||||||
|
// uncompressed length.
|
||||||
|
func MaxEncodedLen(srcLen int) int {
|
||||||
|
// Compressed data can be defined as:
|
||||||
|
// compressed := item* literal*
|
||||||
|
// item := literal* copy
|
||||||
|
//
|
||||||
|
// The trailing literal sequence has a space blowup of at most 62/60
|
||||||
|
// since a literal of length 60 needs one tag byte + one extra byte
|
||||||
|
// for length information.
|
||||||
|
//
|
||||||
|
// Item blowup is trickier to measure. Suppose the "copy" op copies
|
||||||
|
// 4 bytes of data. Because of a special check in the encoding code,
|
||||||
|
// we produce a 4-byte copy only if the offset is < 65536. Therefore
|
||||||
|
// the copy op takes 3 bytes to encode, and this type of item leads
|
||||||
|
// to at most the 62/60 blowup for representing literals.
|
||||||
|
//
|
||||||
|
// Suppose the "copy" op copies 5 bytes of data. If the offset is big
|
||||||
|
// enough, it will take 5 bytes to encode the copy op. Therefore the
|
||||||
|
// worst case here is a one-byte literal followed by a five-byte copy.
|
||||||
|
// That is, 6 bytes of input turn into 7 bytes of "compressed" data.
|
||||||
|
//
|
||||||
|
// This last factor dominates the blowup, so the final estimate is:
|
||||||
|
return 32 + srcLen + srcLen/6
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewWriter returns a new Writer that compresses to w, using the framing
|
||||||
|
// format described at
|
||||||
|
// https://github.com/google/snappy/blob/master/framing_format.txt
|
||||||
|
func NewWriter(w io.Writer) *Writer {
|
||||||
|
return &Writer{
|
||||||
|
w: w,
|
||||||
|
enc: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Writer is an io.Writer than can write Snappy-compressed bytes.
|
||||||
|
type Writer struct {
|
||||||
|
w io.Writer
|
||||||
|
err error
|
||||||
|
enc []byte
|
||||||
|
buf [checksumSize + chunkHeaderSize]byte
|
||||||
|
wroteHeader bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reset discards the writer's state and switches the Snappy writer to write to
|
||||||
|
// w. This permits reusing a Writer rather than allocating a new one.
|
||||||
|
func (w *Writer) Reset(writer io.Writer) {
|
||||||
|
w.w = writer
|
||||||
|
w.err = nil
|
||||||
|
w.wroteHeader = false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Write satisfies the io.Writer interface.
|
||||||
|
func (w *Writer) Write(p []byte) (n int, errRet error) {
|
||||||
|
if w.err != nil {
|
||||||
|
return 0, w.err
|
||||||
|
}
|
||||||
|
if !w.wroteHeader {
|
||||||
|
copy(w.enc, magicChunk)
|
||||||
|
if _, err := w.w.Write(w.enc[:len(magicChunk)]); err != nil {
|
||||||
|
w.err = err
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
w.wroteHeader = true
|
||||||
|
}
|
||||||
|
for len(p) > 0 {
|
||||||
|
var uncompressed []byte
|
||||||
|
if len(p) > maxUncompressedChunkLen {
|
||||||
|
uncompressed, p = p[:maxUncompressedChunkLen], p[maxUncompressedChunkLen:]
|
||||||
|
} else {
|
||||||
|
uncompressed, p = p, nil
|
||||||
|
}
|
||||||
|
checksum := crc(uncompressed)
|
||||||
|
|
||||||
|
// Compress the buffer, discarding the result if the improvement
|
||||||
|
// isn't at least 12.5%.
|
||||||
|
chunkType := uint8(chunkTypeCompressedData)
|
||||||
|
chunkBody := Encode(w.enc, uncompressed)
|
||||||
|
if len(chunkBody) >= len(uncompressed)-len(uncompressed)/8 {
|
||||||
|
chunkType, chunkBody = chunkTypeUncompressedData, uncompressed
|
||||||
|
}
|
||||||
|
|
||||||
|
chunkLen := 4 + len(chunkBody)
|
||||||
|
w.buf[0] = chunkType
|
||||||
|
w.buf[1] = uint8(chunkLen >> 0)
|
||||||
|
w.buf[2] = uint8(chunkLen >> 8)
|
||||||
|
w.buf[3] = uint8(chunkLen >> 16)
|
||||||
|
w.buf[4] = uint8(checksum >> 0)
|
||||||
|
w.buf[5] = uint8(checksum >> 8)
|
||||||
|
w.buf[6] = uint8(checksum >> 16)
|
||||||
|
w.buf[7] = uint8(checksum >> 24)
|
||||||
|
if _, err := w.w.Write(w.buf[:]); err != nil {
|
||||||
|
w.err = err
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
if _, err := w.w.Write(chunkBody); err != nil {
|
||||||
|
w.err = err
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
n += len(uncompressed)
|
||||||
|
}
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
+68
@@ -0,0 +1,68 @@
|
|||||||
|
// Copyright 2011 The Snappy-Go Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// Package snappy implements the snappy block-based compression format.
|
||||||
|
// It aims for very high speeds and reasonable compression.
|
||||||
|
//
|
||||||
|
// The C++ snappy implementation is at https://github.com/google/snappy
|
||||||
|
package snappy
|
||||||
|
|
||||||
|
import (
|
||||||
|
"hash/crc32"
|
||||||
|
)
|
||||||
|
|
||||||
|
/*
|
||||||
|
Each encoded block begins with the varint-encoded length of the decoded data,
|
||||||
|
followed by a sequence of chunks. Chunks begin and end on byte boundaries. The
|
||||||
|
first byte of each chunk is broken into its 2 least and 6 most significant bits
|
||||||
|
called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag.
|
||||||
|
Zero means a literal tag. All other values mean a copy tag.
|
||||||
|
|
||||||
|
For literal tags:
|
||||||
|
- If m < 60, the next 1 + m bytes are literal bytes.
|
||||||
|
- Otherwise, let n be the little-endian unsigned integer denoted by the next
|
||||||
|
m - 59 bytes. The next 1 + n bytes after that are literal bytes.
|
||||||
|
|
||||||
|
For copy tags, length bytes are copied from offset bytes ago, in the style of
|
||||||
|
Lempel-Ziv compression algorithms. In particular:
|
||||||
|
- For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12).
|
||||||
|
The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10
|
||||||
|
of the offset. The next byte is bits 0-7 of the offset.
|
||||||
|
- For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65).
|
||||||
|
The length is 1 + m. The offset is the little-endian unsigned integer
|
||||||
|
denoted by the next 2 bytes.
|
||||||
|
- For l == 3, this tag is a legacy format that is no longer supported.
|
||||||
|
*/
|
||||||
|
const (
|
||||||
|
tagLiteral = 0x00
|
||||||
|
tagCopy1 = 0x01
|
||||||
|
tagCopy2 = 0x02
|
||||||
|
tagCopy4 = 0x03
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
checksumSize = 4
|
||||||
|
chunkHeaderSize = 4
|
||||||
|
magicChunk = "\xff\x06\x00\x00" + magicBody
|
||||||
|
magicBody = "sNaPpY"
|
||||||
|
// https://github.com/google/snappy/blob/master/framing_format.txt says
|
||||||
|
// that "the uncompressed data in a chunk must be no longer than 65536 bytes".
|
||||||
|
maxUncompressedChunkLen = 65536
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
chunkTypeCompressedData = 0x00
|
||||||
|
chunkTypeUncompressedData = 0x01
|
||||||
|
chunkTypePadding = 0xfe
|
||||||
|
chunkTypeStreamIdentifier = 0xff
|
||||||
|
)
|
||||||
|
|
||||||
|
var crcTable = crc32.MakeTable(crc32.Castagnoli)
|
||||||
|
|
||||||
|
// crc implements the checksum specified in section 3 of
|
||||||
|
// https://github.com/google/snappy/blob/master/framing_format.txt
|
||||||
|
func crc(b []byte) uint32 {
|
||||||
|
c := crc32.Update(0, crcTable, b)
|
||||||
|
return uint32(c>>15|c<<17) + 0xa282ead8
|
||||||
|
}
|
||||||
+191
@@ -0,0 +1,191 @@
|
|||||||
|
All files in this repository are licensed as follows. If you contribute
|
||||||
|
to this repository, it is assumed that you license your contribution
|
||||||
|
under the same license unless you state otherwise.
|
||||||
|
|
||||||
|
All files Copyright (C) 2015 Canonical Ltd. unless otherwise specified in the file.
|
||||||
|
|
||||||
|
This software is licensed under the LGPLv3, included below.
|
||||||
|
|
||||||
|
As a special exception to the GNU Lesser General Public License version 3
|
||||||
|
("LGPL3"), the copyright holders of this Library give you permission to
|
||||||
|
convey to a third party a Combined Work that links statically or dynamically
|
||||||
|
to this Library without providing any Minimal Corresponding Source or
|
||||||
|
Minimal Application Code as set out in 4d or providing the installation
|
||||||
|
information set out in section 4e, provided that you comply with the other
|
||||||
|
provisions of LGPL3 and provided that you meet, for the Application the
|
||||||
|
terms and conditions of the license(s) which apply to the Application.
|
||||||
|
|
||||||
|
Except as stated in this special exception, the provisions of LGPL3 will
|
||||||
|
continue to comply in full to this Library. If you modify this Library, you
|
||||||
|
may apply this exception to your version of this Library, but you are not
|
||||||
|
obliged to do so. If you do not wish to do so, delete this exception
|
||||||
|
statement from your version. This exception does not (and cannot) modify any
|
||||||
|
license terms which apply to the Application, with which you must still
|
||||||
|
comply.
|
||||||
|
|
||||||
|
|
||||||
|
GNU LESSER GENERAL PUBLIC LICENSE
|
||||||
|
Version 3, 29 June 2007
|
||||||
|
|
||||||
|
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
|
||||||
|
Everyone is permitted to copy and distribute verbatim copies
|
||||||
|
of this license document, but changing it is not allowed.
|
||||||
|
|
||||||
|
|
||||||
|
This version of the GNU Lesser General Public License incorporates
|
||||||
|
the terms and conditions of version 3 of the GNU General Public
|
||||||
|
License, supplemented by the additional permissions listed below.
|
||||||
|
|
||||||
|
0. Additional Definitions.
|
||||||
|
|
||||||
|
As used herein, "this License" refers to version 3 of the GNU Lesser
|
||||||
|
General Public License, and the "GNU GPL" refers to version 3 of the GNU
|
||||||
|
General Public License.
|
||||||
|
|
||||||
|
"The Library" refers to a covered work governed by this License,
|
||||||
|
other than an Application or a Combined Work as defined below.
|
||||||
|
|
||||||
|
An "Application" is any work that makes use of an interface provided
|
||||||
|
by the Library, but which is not otherwise based on the Library.
|
||||||
|
Defining a subclass of a class defined by the Library is deemed a mode
|
||||||
|
of using an interface provided by the Library.
|
||||||
|
|
||||||
|
A "Combined Work" is a work produced by combining or linking an
|
||||||
|
Application with the Library. The particular version of the Library
|
||||||
|
with which the Combined Work was made is also called the "Linked
|
||||||
|
Version".
|
||||||
|
|
||||||
|
The "Minimal Corresponding Source" for a Combined Work means the
|
||||||
|
Corresponding Source for the Combined Work, excluding any source code
|
||||||
|
for portions of the Combined Work that, considered in isolation, are
|
||||||
|
based on the Application, and not on the Linked Version.
|
||||||
|
|
||||||
|
The "Corresponding Application Code" for a Combined Work means the
|
||||||
|
object code and/or source code for the Application, including any data
|
||||||
|
and utility programs needed for reproducing the Combined Work from the
|
||||||
|
Application, but excluding the System Libraries of the Combined Work.
|
||||||
|
|
||||||
|
1. Exception to Section 3 of the GNU GPL.
|
||||||
|
|
||||||
|
You may convey a covered work under sections 3 and 4 of this License
|
||||||
|
without being bound by section 3 of the GNU GPL.
|
||||||
|
|
||||||
|
2. Conveying Modified Versions.
|
||||||
|
|
||||||
|
If you modify a copy of the Library, and, in your modifications, a
|
||||||
|
facility refers to a function or data to be supplied by an Application
|
||||||
|
that uses the facility (other than as an argument passed when the
|
||||||
|
facility is invoked), then you may convey a copy of the modified
|
||||||
|
version:
|
||||||
|
|
||||||
|
a) under this License, provided that you make a good faith effort to
|
||||||
|
ensure that, in the event an Application does not supply the
|
||||||
|
function or data, the facility still operates, and performs
|
||||||
|
whatever part of its purpose remains meaningful, or
|
||||||
|
|
||||||
|
b) under the GNU GPL, with none of the additional permissions of
|
||||||
|
this License applicable to that copy.
|
||||||
|
|
||||||
|
3. Object Code Incorporating Material from Library Header Files.
|
||||||
|
|
||||||
|
The object code form of an Application may incorporate material from
|
||||||
|
a header file that is part of the Library. You may convey such object
|
||||||
|
code under terms of your choice, provided that, if the incorporated
|
||||||
|
material is not limited to numerical parameters, data structure
|
||||||
|
layouts and accessors, or small macros, inline functions and templates
|
||||||
|
(ten or fewer lines in length), you do both of the following:
|
||||||
|
|
||||||
|
a) Give prominent notice with each copy of the object code that the
|
||||||
|
Library is used in it and that the Library and its use are
|
||||||
|
covered by this License.
|
||||||
|
|
||||||
|
b) Accompany the object code with a copy of the GNU GPL and this license
|
||||||
|
document.
|
||||||
|
|
||||||
|
4. Combined Works.
|
||||||
|
|
||||||
|
You may convey a Combined Work under terms of your choice that,
|
||||||
|
taken together, effectively do not restrict modification of the
|
||||||
|
portions of the Library contained in the Combined Work and reverse
|
||||||
|
engineering for debugging such modifications, if you also do each of
|
||||||
|
the following:
|
||||||
|
|
||||||
|
a) Give prominent notice with each copy of the Combined Work that
|
||||||
|
the Library is used in it and that the Library and its use are
|
||||||
|
covered by this License.
|
||||||
|
|
||||||
|
b) Accompany the Combined Work with a copy of the GNU GPL and this license
|
||||||
|
document.
|
||||||
|
|
||||||
|
c) For a Combined Work that displays copyright notices during
|
||||||
|
execution, include the copyright notice for the Library among
|
||||||
|
these notices, as well as a reference directing the user to the
|
||||||
|
copies of the GNU GPL and this license document.
|
||||||
|
|
||||||
|
d) Do one of the following:
|
||||||
|
|
||||||
|
0) Convey the Minimal Corresponding Source under the terms of this
|
||||||
|
License, and the Corresponding Application Code in a form
|
||||||
|
suitable for, and under terms that permit, the user to
|
||||||
|
recombine or relink the Application with a modified version of
|
||||||
|
the Linked Version to produce a modified Combined Work, in the
|
||||||
|
manner specified by section 6 of the GNU GPL for conveying
|
||||||
|
Corresponding Source.
|
||||||
|
|
||||||
|
1) Use a suitable shared library mechanism for linking with the
|
||||||
|
Library. A suitable mechanism is one that (a) uses at run time
|
||||||
|
a copy of the Library already present on the user's computer
|
||||||
|
system, and (b) will operate properly with a modified version
|
||||||
|
of the Library that is interface-compatible with the Linked
|
||||||
|
Version.
|
||||||
|
|
||||||
|
e) Provide Installation Information, but only if you would otherwise
|
||||||
|
be required to provide such information under section 6 of the
|
||||||
|
GNU GPL, and only to the extent that such information is
|
||||||
|
necessary to install and execute a modified version of the
|
||||||
|
Combined Work produced by recombining or relinking the
|
||||||
|
Application with a modified version of the Linked Version. (If
|
||||||
|
you use option 4d0, the Installation Information must accompany
|
||||||
|
the Minimal Corresponding Source and Corresponding Application
|
||||||
|
Code. If you use option 4d1, you must provide the Installation
|
||||||
|
Information in the manner specified by section 6 of the GNU GPL
|
||||||
|
for conveying Corresponding Source.)
|
||||||
|
|
||||||
|
5. Combined Libraries.
|
||||||
|
|
||||||
|
You may place library facilities that are a work based on the
|
||||||
|
Library side by side in a single library together with other library
|
||||||
|
facilities that are not Applications and are not covered by this
|
||||||
|
License, and convey such a combined library under terms of your
|
||||||
|
choice, if you do both of the following:
|
||||||
|
|
||||||
|
a) Accompany the combined library with a copy of the same work based
|
||||||
|
on the Library, uncombined with any other library facilities,
|
||||||
|
conveyed under the terms of this License.
|
||||||
|
|
||||||
|
b) Give prominent notice with the combined library that part of it
|
||||||
|
is a work based on the Library, and explaining where to find the
|
||||||
|
accompanying uncombined form of the same work.
|
||||||
|
|
||||||
|
6. Revised Versions of the GNU Lesser General Public License.
|
||||||
|
|
||||||
|
The Free Software Foundation may publish revised and/or new versions
|
||||||
|
of the GNU Lesser General Public License from time to time. Such new
|
||||||
|
versions will be similar in spirit to the present version, but may
|
||||||
|
differ in detail to address new problems or concerns.
|
||||||
|
|
||||||
|
Each version is given a distinguishing version number. If the
|
||||||
|
Library as you received it specifies that a certain numbered version
|
||||||
|
of the GNU Lesser General Public License "or any later version"
|
||||||
|
applies to it, you have the option of following the terms and
|
||||||
|
conditions either of that published version or of any later version
|
||||||
|
published by the Free Software Foundation. If the Library as you
|
||||||
|
received it does not specify a version number of the GNU Lesser
|
||||||
|
General Public License, you may choose any version of the GNU Lesser
|
||||||
|
General Public License ever published by the Free Software Foundation.
|
||||||
|
|
||||||
|
If the Library as you received it specifies that a proxy can decide
|
||||||
|
whether future versions of the GNU Lesser General Public License shall
|
||||||
|
apply, that proxy's public statement of acceptance of any version is
|
||||||
|
permanent authorization for you to choose that version for the
|
||||||
|
Library.
|
||||||
+117
@@ -0,0 +1,117 @@
|
|||||||
|
# ratelimit
|
||||||
|
--
|
||||||
|
import "github.com/juju/ratelimit"
|
||||||
|
|
||||||
|
The ratelimit package provides an efficient token bucket implementation. See
|
||||||
|
http://en.wikipedia.org/wiki/Token_bucket.
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
#### func Reader
|
||||||
|
|
||||||
|
```go
|
||||||
|
func Reader(r io.Reader, bucket *Bucket) io.Reader
|
||||||
|
```
|
||||||
|
Reader returns a reader that is rate limited by the given token bucket. Each
|
||||||
|
token in the bucket represents one byte.
|
||||||
|
|
||||||
|
#### func Writer
|
||||||
|
|
||||||
|
```go
|
||||||
|
func Writer(w io.Writer, bucket *Bucket) io.Writer
|
||||||
|
```
|
||||||
|
Writer returns a writer that is rate limited by the given token bucket. Each
|
||||||
|
token in the bucket represents one byte.
|
||||||
|
|
||||||
|
#### type Bucket
|
||||||
|
|
||||||
|
```go
|
||||||
|
type Bucket struct {
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Bucket represents a token bucket that fills at a predetermined rate. Methods on
|
||||||
|
Bucket may be called concurrently.
|
||||||
|
|
||||||
|
#### func NewBucket
|
||||||
|
|
||||||
|
```go
|
||||||
|
func NewBucket(fillInterval time.Duration, capacity int64) *Bucket
|
||||||
|
```
|
||||||
|
NewBucket returns a new token bucket that fills at the rate of one token every
|
||||||
|
fillInterval, up to the given maximum capacity. Both arguments must be positive.
|
||||||
|
The bucket is initially full.
|
||||||
|
|
||||||
|
#### func NewBucketWithQuantum
|
||||||
|
|
||||||
|
```go
|
||||||
|
func NewBucketWithQuantum(fillInterval time.Duration, capacity, quantum int64) *Bucket
|
||||||
|
```
|
||||||
|
NewBucketWithQuantum is similar to NewBucket, but allows the specification of
|
||||||
|
the quantum size - quantum tokens are added every fillInterval.
|
||||||
|
|
||||||
|
#### func NewBucketWithRate
|
||||||
|
|
||||||
|
```go
|
||||||
|
func NewBucketWithRate(rate float64, capacity int64) *Bucket
|
||||||
|
```
|
||||||
|
NewBucketWithRate returns a token bucket that fills the bucket at the rate of
|
||||||
|
rate tokens per second up to the given maximum capacity. Because of limited
|
||||||
|
clock resolution, at high rates, the actual rate may be up to 1% different from
|
||||||
|
the specified rate.
|
||||||
|
|
||||||
|
#### func (*Bucket) Rate
|
||||||
|
|
||||||
|
```go
|
||||||
|
func (tb *Bucket) Rate() float64
|
||||||
|
```
|
||||||
|
Rate returns the fill rate of the bucket, in tokens per second.
|
||||||
|
|
||||||
|
#### func (*Bucket) Take
|
||||||
|
|
||||||
|
```go
|
||||||
|
func (tb *Bucket) Take(count int64) time.Duration
|
||||||
|
```
|
||||||
|
Take takes count tokens from the bucket without blocking. It returns the time
|
||||||
|
that the caller should wait until the tokens are actually available.
|
||||||
|
|
||||||
|
Note that if the request is irrevocable - there is no way to return tokens to
|
||||||
|
the bucket once this method commits us to taking them.
|
||||||
|
|
||||||
|
#### func (*Bucket) TakeAvailable
|
||||||
|
|
||||||
|
```go
|
||||||
|
func (tb *Bucket) TakeAvailable(count int64) int64
|
||||||
|
```
|
||||||
|
TakeAvailable takes up to count immediately available tokens from the bucket. It
|
||||||
|
returns the number of tokens removed, or zero if there are no available tokens.
|
||||||
|
It does not block.
|
||||||
|
|
||||||
|
#### func (*Bucket) TakeMaxDuration
|
||||||
|
|
||||||
|
```go
|
||||||
|
func (tb *Bucket) TakeMaxDuration(count int64, maxWait time.Duration) (time.Duration, bool)
|
||||||
|
```
|
||||||
|
TakeMaxDuration is like Take, except that it will only take tokens from the
|
||||||
|
bucket if the wait time for the tokens is no greater than maxWait.
|
||||||
|
|
||||||
|
If it would take longer than maxWait for the tokens to become available, it does
|
||||||
|
nothing and reports false, otherwise it returns the time that the caller should
|
||||||
|
wait until the tokens are actually available, and reports true.
|
||||||
|
|
||||||
|
#### func (*Bucket) Wait
|
||||||
|
|
||||||
|
```go
|
||||||
|
func (tb *Bucket) Wait(count int64)
|
||||||
|
```
|
||||||
|
Wait takes count tokens from the bucket, waiting until they are available.
|
||||||
|
|
||||||
|
#### func (*Bucket) WaitMaxDuration
|
||||||
|
|
||||||
|
```go
|
||||||
|
func (tb *Bucket) WaitMaxDuration(count int64, maxWait time.Duration) bool
|
||||||
|
```
|
||||||
|
WaitMaxDuration is like Wait except that it will only take tokens from the
|
||||||
|
bucket if it needs to wait for no greater than maxWait. It reports whether any
|
||||||
|
tokens have been removed from the bucket If no tokens have been removed, it
|
||||||
|
returns immediately.
|
||||||
+221
@@ -0,0 +1,221 @@
|
|||||||
|
// Copyright 2014 Canonical Ltd.
|
||||||
|
// Licensed under the LGPLv3 with static-linking exception.
|
||||||
|
// See LICENCE file for details.
|
||||||
|
|
||||||
|
// The ratelimit package provides an efficient token bucket implementation
|
||||||
|
// that can be used to limit the rate of arbitrary things.
|
||||||
|
// See http://en.wikipedia.org/wiki/Token_bucket.
|
||||||
|
package ratelimit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strconv"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
"math"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Bucket represents a token bucket that fills at a predetermined rate.
|
||||||
|
// Methods on Bucket may be called concurrently.
|
||||||
|
type Bucket struct {
|
||||||
|
startTime time.Time
|
||||||
|
capacity int64
|
||||||
|
quantum int64
|
||||||
|
fillInterval time.Duration
|
||||||
|
|
||||||
|
// The mutex guards the fields following it.
|
||||||
|
mu sync.Mutex
|
||||||
|
|
||||||
|
// avail holds the number of available tokens
|
||||||
|
// in the bucket, as of availTick ticks from startTime.
|
||||||
|
// It will be negative when there are consumers
|
||||||
|
// waiting for tokens.
|
||||||
|
avail int64
|
||||||
|
availTick int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewBucket returns a new token bucket that fills at the
|
||||||
|
// rate of one token every fillInterval, up to the given
|
||||||
|
// maximum capacity. Both arguments must be
|
||||||
|
// positive. The bucket is initially full.
|
||||||
|
func NewBucket(fillInterval time.Duration, capacity int64) *Bucket {
|
||||||
|
return NewBucketWithQuantum(fillInterval, capacity, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
// rateMargin specifes the allowed variance of actual
|
||||||
|
// rate from specified rate. 1% seems reasonable.
|
||||||
|
const rateMargin = 0.01
|
||||||
|
|
||||||
|
// NewBucketWithRate returns a token bucket that fills the bucket
|
||||||
|
// at the rate of rate tokens per second up to the given
|
||||||
|
// maximum capacity. Because of limited clock resolution,
|
||||||
|
// at high rates, the actual rate may be up to 1% different from the
|
||||||
|
// specified rate.
|
||||||
|
func NewBucketWithRate(rate float64, capacity int64) *Bucket {
|
||||||
|
for quantum := int64(1); quantum < 1<<50; quantum = nextQuantum(quantum) {
|
||||||
|
fillInterval := time.Duration(1e9 * float64(quantum) / rate)
|
||||||
|
if fillInterval <= 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
tb := NewBucketWithQuantum(fillInterval, capacity, quantum)
|
||||||
|
if diff := math.Abs(tb.Rate() - rate); diff/rate <= rateMargin {
|
||||||
|
return tb
|
||||||
|
}
|
||||||
|
}
|
||||||
|
panic("cannot find suitable quantum for " + strconv.FormatFloat(rate, 'g', -1, 64))
|
||||||
|
}
|
||||||
|
|
||||||
|
// nextQuantum returns the next quantum to try after q.
|
||||||
|
// We grow the quantum exponentially, but slowly, so we
|
||||||
|
// get a good fit in the lower numbers.
|
||||||
|
func nextQuantum(q int64) int64 {
|
||||||
|
q1 := q * 11 / 10
|
||||||
|
if q1 == q {
|
||||||
|
q1++
|
||||||
|
}
|
||||||
|
return q1
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewBucketWithQuantum is similar to NewBucket, but allows
|
||||||
|
// the specification of the quantum size - quantum tokens
|
||||||
|
// are added every fillInterval.
|
||||||
|
func NewBucketWithQuantum(fillInterval time.Duration, capacity, quantum int64) *Bucket {
|
||||||
|
if fillInterval <= 0 {
|
||||||
|
panic("token bucket fill interval is not > 0")
|
||||||
|
}
|
||||||
|
if capacity <= 0 {
|
||||||
|
panic("token bucket capacity is not > 0")
|
||||||
|
}
|
||||||
|
if quantum <= 0 {
|
||||||
|
panic("token bucket quantum is not > 0")
|
||||||
|
}
|
||||||
|
return &Bucket{
|
||||||
|
startTime: time.Now(),
|
||||||
|
capacity: capacity,
|
||||||
|
quantum: quantum,
|
||||||
|
avail: capacity,
|
||||||
|
fillInterval: fillInterval,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait takes count tokens from the bucket, waiting until they are
|
||||||
|
// available.
|
||||||
|
func (tb *Bucket) Wait(count int64) {
|
||||||
|
if d := tb.Take(count); d > 0 {
|
||||||
|
time.Sleep(d)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WaitMaxDuration is like Wait except that it will
|
||||||
|
// only take tokens from the bucket if it needs to wait
|
||||||
|
// for no greater than maxWait. It reports whether
|
||||||
|
// any tokens have been removed from the bucket
|
||||||
|
// If no tokens have been removed, it returns immediately.
|
||||||
|
func (tb *Bucket) WaitMaxDuration(count int64, maxWait time.Duration) bool {
|
||||||
|
d, ok := tb.TakeMaxDuration(count, maxWait)
|
||||||
|
if d > 0 {
|
||||||
|
time.Sleep(d)
|
||||||
|
}
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
|
||||||
|
const infinityDuration time.Duration = 0x7fffffffffffffff
|
||||||
|
|
||||||
|
// Take takes count tokens from the bucket without blocking. It returns
|
||||||
|
// the time that the caller should wait until the tokens are actually
|
||||||
|
// available.
|
||||||
|
//
|
||||||
|
// Note that if the request is irrevocable - there is no way to return
|
||||||
|
// tokens to the bucket once this method commits us to taking them.
|
||||||
|
func (tb *Bucket) Take(count int64) time.Duration {
|
||||||
|
d, _ := tb.take(time.Now(), count, infinityDuration)
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
|
// TakeMaxDuration is like Take, except that
|
||||||
|
// it will only take tokens from the bucket if the wait
|
||||||
|
// time for the tokens is no greater than maxWait.
|
||||||
|
//
|
||||||
|
// If it would take longer than maxWait for the tokens
|
||||||
|
// to become available, it does nothing and reports false,
|
||||||
|
// otherwise it returns the time that the caller should
|
||||||
|
// wait until the tokens are actually available, and reports
|
||||||
|
// true.
|
||||||
|
func (tb *Bucket) TakeMaxDuration(count int64, maxWait time.Duration) (time.Duration, bool) {
|
||||||
|
return tb.take(time.Now(), count, maxWait)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TakeAvailable takes up to count immediately available tokens from the
|
||||||
|
// bucket. It returns the number of tokens removed, or zero if there are
|
||||||
|
// no available tokens. It does not block.
|
||||||
|
func (tb *Bucket) TakeAvailable(count int64) int64 {
|
||||||
|
return tb.takeAvailable(time.Now(), count)
|
||||||
|
}
|
||||||
|
|
||||||
|
// takeAvailable is the internal version of TakeAvailable - it takes the
|
||||||
|
// current time as an argument to enable easy testing.
|
||||||
|
func (tb *Bucket) takeAvailable(now time.Time, count int64) int64 {
|
||||||
|
if count <= 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
tb.mu.Lock()
|
||||||
|
defer tb.mu.Unlock()
|
||||||
|
|
||||||
|
tb.adjust(now)
|
||||||
|
if tb.avail <= 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
if count > tb.avail {
|
||||||
|
count = tb.avail
|
||||||
|
}
|
||||||
|
tb.avail -= count
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rate returns the fill rate of the bucket, in tokens per second.
|
||||||
|
func (tb *Bucket) Rate() float64 {
|
||||||
|
return 1e9 * float64(tb.quantum) / float64(tb.fillInterval)
|
||||||
|
}
|
||||||
|
|
||||||
|
// take is the internal version of Take - it takes the current time as
|
||||||
|
// an argument to enable easy testing.
|
||||||
|
func (tb *Bucket) take(now time.Time, count int64, maxWait time.Duration) (time.Duration, bool) {
|
||||||
|
if count <= 0 {
|
||||||
|
return 0, true
|
||||||
|
}
|
||||||
|
tb.mu.Lock()
|
||||||
|
defer tb.mu.Unlock()
|
||||||
|
|
||||||
|
currentTick := tb.adjust(now)
|
||||||
|
avail := tb.avail - count
|
||||||
|
if avail >= 0 {
|
||||||
|
tb.avail = avail
|
||||||
|
return 0, true
|
||||||
|
}
|
||||||
|
// Round up the missing tokens to the nearest multiple
|
||||||
|
// of quantum - the tokens won't be available until
|
||||||
|
// that tick.
|
||||||
|
endTick := currentTick + (-avail+tb.quantum-1)/tb.quantum
|
||||||
|
endTime := tb.startTime.Add(time.Duration(endTick) * tb.fillInterval)
|
||||||
|
waitTime := endTime.Sub(now)
|
||||||
|
if waitTime > maxWait {
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
tb.avail = avail
|
||||||
|
return waitTime, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// adjust adjusts the current bucket capacity based on the current time.
|
||||||
|
// It returns the current tick.
|
||||||
|
func (tb *Bucket) adjust(now time.Time) (currentTick int64) {
|
||||||
|
currentTick = int64(now.Sub(tb.startTime) / tb.fillInterval)
|
||||||
|
|
||||||
|
if tb.avail >= tb.capacity {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
tb.avail += (currentTick - tb.availTick) * tb.quantum
|
||||||
|
if tb.avail > tb.capacity {
|
||||||
|
tb.avail = tb.capacity
|
||||||
|
}
|
||||||
|
tb.availTick = currentTick
|
||||||
|
return
|
||||||
|
}
|
||||||
+51
@@ -0,0 +1,51 @@
|
|||||||
|
// Copyright 2014 Canonical Ltd.
|
||||||
|
// Licensed under the LGPLv3 with static-linking exception.
|
||||||
|
// See LICENCE file for details.
|
||||||
|
|
||||||
|
package ratelimit
|
||||||
|
|
||||||
|
import "io"
|
||||||
|
|
||||||
|
type reader struct {
|
||||||
|
r io.Reader
|
||||||
|
bucket *Bucket
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reader returns a reader that is rate limited by
|
||||||
|
// the given token bucket. Each token in the bucket
|
||||||
|
// represents one byte.
|
||||||
|
func Reader(r io.Reader, bucket *Bucket) io.Reader {
|
||||||
|
return &reader{
|
||||||
|
r: r,
|
||||||
|
bucket: bucket,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *reader) Read(buf []byte) (int, error) {
|
||||||
|
n, err := r.r.Read(buf)
|
||||||
|
if n <= 0 {
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
r.bucket.Wait(int64(n))
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
type writer struct {
|
||||||
|
w io.Writer
|
||||||
|
bucket *Bucket
|
||||||
|
}
|
||||||
|
|
||||||
|
// Writer returns a reader that is rate limited by
|
||||||
|
// the given token bucket. Each token in the bucket
|
||||||
|
// represents one byte.
|
||||||
|
func Writer(w io.Writer, bucket *Bucket) io.Writer {
|
||||||
|
return &writer{
|
||||||
|
w: w,
|
||||||
|
bucket: bucket,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *writer) Write(buf []byte) (int, error) {
|
||||||
|
w.bucket.Wait(int64(len(buf)))
|
||||||
|
return w.w.Write(buf)
|
||||||
|
}
|
||||||
+27
@@ -0,0 +1,27 @@
|
|||||||
|
Copyright (c) 2012 The Go Authors. All rights reserved.
|
||||||
|
|
||||||
|
Redistribution and use in source and binary forms, with or without
|
||||||
|
modification, are permitted provided that the following conditions are
|
||||||
|
met:
|
||||||
|
|
||||||
|
* Redistributions of source code must retain the above copyright
|
||||||
|
notice, this list of conditions and the following disclaimer.
|
||||||
|
* Redistributions in binary form must reproduce the above
|
||||||
|
copyright notice, this list of conditions and the following disclaimer
|
||||||
|
in the documentation and/or other materials provided with the
|
||||||
|
distribution.
|
||||||
|
* Neither the name of Google Inc. nor the names of its
|
||||||
|
contributors may be used to endorse or promote products derived from
|
||||||
|
this software without specific prior written permission.
|
||||||
|
|
||||||
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||||
|
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||||
|
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||||
|
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||||
|
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||||
|
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||||
|
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||||
|
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||||
|
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
|
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
+16
@@ -0,0 +1,16 @@
|
|||||||
|
### Extensions to the "os" package.
|
||||||
|
|
||||||
|
## Find the current Executable and ExecutableFolder.
|
||||||
|
|
||||||
|
There is sometimes utility in finding the current executable file
|
||||||
|
that is running. This can be used for upgrading the current executable
|
||||||
|
or finding resources located relative to the executable file. Both
|
||||||
|
working directory and the os.Args[0] value are arbitrary and cannot
|
||||||
|
be relied on; os.Args[0] can be "faked".
|
||||||
|
|
||||||
|
Multi-platform and supports:
|
||||||
|
* Linux
|
||||||
|
* OS X
|
||||||
|
* Windows
|
||||||
|
* Plan 9
|
||||||
|
* BSDs.
|
||||||
+33
@@ -0,0 +1,33 @@
|
|||||||
|
// Copyright 2012 The Go Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// Extensions to the standard "os" package.
|
||||||
|
package osext
|
||||||
|
|
||||||
|
import "path/filepath"
|
||||||
|
|
||||||
|
var cx, ce = executableClean()
|
||||||
|
|
||||||
|
func executableClean() (string, error) {
|
||||||
|
p, err := executable()
|
||||||
|
return filepath.Clean(p), err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Executable returns an absolute path that can be used to
|
||||||
|
// re-invoke the current program.
|
||||||
|
// It may not be valid after the current program exits.
|
||||||
|
func Executable() (string, error) {
|
||||||
|
return cx, ce
|
||||||
|
}
|
||||||
|
|
||||||
|
// Returns same path as Executable, returns just the folder
|
||||||
|
// path. Excludes the executable name and any trailing slash.
|
||||||
|
func ExecutableFolder() (string, error) {
|
||||||
|
p, err := Executable()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
|
||||||
|
return filepath.Dir(p), nil
|
||||||
|
}
|
||||||
+20
@@ -0,0 +1,20 @@
|
|||||||
|
// Copyright 2012 The Go Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package osext
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"syscall"
|
||||||
|
)
|
||||||
|
|
||||||
|
func executable() (string, error) {
|
||||||
|
f, err := os.Open("/proc/" + strconv.Itoa(os.Getpid()) + "/text")
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
return syscall.Fd2path(int(f.Fd()))
|
||||||
|
}
|
||||||
+36
@@ -0,0 +1,36 @@
|
|||||||
|
// Copyright 2012 The Go Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// +build linux netbsd solaris dragonfly
|
||||||
|
|
||||||
|
package osext
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
func executable() (string, error) {
|
||||||
|
switch runtime.GOOS {
|
||||||
|
case "linux":
|
||||||
|
const deletedTag = " (deleted)"
|
||||||
|
execpath, err := os.Readlink("/proc/self/exe")
|
||||||
|
if err != nil {
|
||||||
|
return execpath, err
|
||||||
|
}
|
||||||
|
execpath = strings.TrimSuffix(execpath, deletedTag)
|
||||||
|
execpath = strings.TrimPrefix(execpath, deletedTag)
|
||||||
|
return execpath, nil
|
||||||
|
case "netbsd":
|
||||||
|
return os.Readlink("/proc/curproc/exe")
|
||||||
|
case "dragonfly":
|
||||||
|
return os.Readlink("/proc/curproc/file")
|
||||||
|
case "solaris":
|
||||||
|
return os.Readlink(fmt.Sprintf("/proc/%d/path/a.out", os.Getpid()))
|
||||||
|
}
|
||||||
|
return "", errors.New("ExecPath not implemented for " + runtime.GOOS)
|
||||||
|
}
|
||||||
+126
@@ -0,0 +1,126 @@
|
|||||||
|
// Copyright 2012 The Go Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
// +build darwin freebsd openbsd
|
||||||
|
|
||||||
|
package osext
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"syscall"
|
||||||
|
"unsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
var initCwd, initCwdErr = os.Getwd()
|
||||||
|
|
||||||
|
func executable() (string, error) {
|
||||||
|
var mib [4]int32
|
||||||
|
switch runtime.GOOS {
|
||||||
|
case "freebsd":
|
||||||
|
mib = [4]int32{1 /* CTL_KERN */, 14 /* KERN_PROC */, 12 /* KERN_PROC_PATHNAME */, -1}
|
||||||
|
case "darwin":
|
||||||
|
mib = [4]int32{1 /* CTL_KERN */, 38 /* KERN_PROCARGS */, int32(os.Getpid()), -1}
|
||||||
|
case "openbsd":
|
||||||
|
mib = [4]int32{1 /* CTL_KERN */, 55 /* KERN_PROC_ARGS */, int32(os.Getpid()), 1 /* KERN_PROC_ARGV */}
|
||||||
|
}
|
||||||
|
|
||||||
|
n := uintptr(0)
|
||||||
|
// Get length.
|
||||||
|
_, _, errNum := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, 0, uintptr(unsafe.Pointer(&n)), 0, 0)
|
||||||
|
if errNum != 0 {
|
||||||
|
return "", errNum
|
||||||
|
}
|
||||||
|
if n == 0 { // This shouldn't happen.
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
buf := make([]byte, n)
|
||||||
|
_, _, errNum = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, uintptr(unsafe.Pointer(&buf[0])), uintptr(unsafe.Pointer(&n)), 0, 0)
|
||||||
|
if errNum != 0 {
|
||||||
|
return "", errNum
|
||||||
|
}
|
||||||
|
if n == 0 { // This shouldn't happen.
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var execPath string
|
||||||
|
switch runtime.GOOS {
|
||||||
|
case "openbsd":
|
||||||
|
// buf now contains **argv, with pointers to each of the C-style
|
||||||
|
// NULL terminated arguments.
|
||||||
|
var args []string
|
||||||
|
argv := uintptr(unsafe.Pointer(&buf[0]))
|
||||||
|
Loop:
|
||||||
|
for {
|
||||||
|
argp := *(**[1 << 20]byte)(unsafe.Pointer(argv))
|
||||||
|
if argp == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
for i := 0; uintptr(i) < n; i++ {
|
||||||
|
// we don't want the full arguments list
|
||||||
|
if string(argp[i]) == " " {
|
||||||
|
break Loop
|
||||||
|
}
|
||||||
|
if argp[i] != 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
args = append(args, string(argp[:i]))
|
||||||
|
n -= uintptr(i)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if n < unsafe.Sizeof(argv) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
argv += unsafe.Sizeof(argv)
|
||||||
|
n -= unsafe.Sizeof(argv)
|
||||||
|
}
|
||||||
|
execPath = args[0]
|
||||||
|
// There is no canonical way to get an executable path on
|
||||||
|
// OpenBSD, so check PATH in case we are called directly
|
||||||
|
if execPath[0] != '/' && execPath[0] != '.' {
|
||||||
|
execIsInPath, err := exec.LookPath(execPath)
|
||||||
|
if err == nil {
|
||||||
|
execPath = execIsInPath
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
for i, v := range buf {
|
||||||
|
if v == 0 {
|
||||||
|
buf = buf[:i]
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
execPath = string(buf)
|
||||||
|
}
|
||||||
|
|
||||||
|
var err error
|
||||||
|
// execPath will not be empty due to above checks.
|
||||||
|
// Try to get the absolute path if the execPath is not rooted.
|
||||||
|
if execPath[0] != '/' {
|
||||||
|
execPath, err = getAbs(execPath)
|
||||||
|
if err != nil {
|
||||||
|
return execPath, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// For darwin KERN_PROCARGS may return the path to a symlink rather than the
|
||||||
|
// actual executable.
|
||||||
|
if runtime.GOOS == "darwin" {
|
||||||
|
if execPath, err = filepath.EvalSymlinks(execPath); err != nil {
|
||||||
|
return execPath, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return execPath, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func getAbs(execPath string) (string, error) {
|
||||||
|
if initCwdErr != nil {
|
||||||
|
return execPath, initCwdErr
|
||||||
|
}
|
||||||
|
// The execPath may begin with a "../" or a "./" so clean it first.
|
||||||
|
// Join the two paths, trailing and starting slashes undetermined, so use
|
||||||
|
// the generic Join function.
|
||||||
|
return filepath.Join(initCwd, filepath.Clean(execPath)), nil
|
||||||
|
}
|
||||||
+34
@@ -0,0 +1,34 @@
|
|||||||
|
// Copyright 2012 The Go Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style
|
||||||
|
// license that can be found in the LICENSE file.
|
||||||
|
|
||||||
|
package osext
|
||||||
|
|
||||||
|
import (
|
||||||
|
"syscall"
|
||||||
|
"unicode/utf16"
|
||||||
|
"unsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
kernel = syscall.MustLoadDLL("kernel32.dll")
|
||||||
|
getModuleFileNameProc = kernel.MustFindProc("GetModuleFileNameW")
|
||||||
|
)
|
||||||
|
|
||||||
|
// GetModuleFileName() with hModule = NULL
|
||||||
|
func executable() (exePath string, err error) {
|
||||||
|
return getModuleFileName()
|
||||||
|
}
|
||||||
|
|
||||||
|
func getModuleFileName() (string, error) {
|
||||||
|
var n uint32
|
||||||
|
b := make([]uint16, syscall.MAX_PATH)
|
||||||
|
size := uint32(len(b))
|
||||||
|
|
||||||
|
r0, _, e1 := getModuleFileNameProc.Call(0, uintptr(unsafe.Pointer(&b[0])), uintptr(size))
|
||||||
|
n = uint32(r0)
|
||||||
|
if n == 0 {
|
||||||
|
return "", e1
|
||||||
|
}
|
||||||
|
return string(utf16.Decode(b[0:n])), nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
*.[68]
|
||||||
|
*.a
|
||||||
|
*.out
|
||||||
|
*.swp
|
||||||
|
_obj
|
||||||
|
_testmain.go
|
||||||
|
cmd/metrics-bench/metrics-bench
|
||||||
|
cmd/metrics-example/metrics-example
|
||||||
|
cmd/never-read/never-read
|
||||||
+13
@@ -0,0 +1,13 @@
|
|||||||
|
language: go
|
||||||
|
|
||||||
|
go:
|
||||||
|
- 1.2
|
||||||
|
- 1.3
|
||||||
|
- 1.4
|
||||||
|
|
||||||
|
script:
|
||||||
|
- ./validate.sh
|
||||||
|
|
||||||
|
# this should give us faster builds according to
|
||||||
|
# http://docs.travis-ci.com/user/migrating-from-legacy/
|
||||||
|
sudo: false
|
||||||
+29
@@ -0,0 +1,29 @@
|
|||||||
|
Copyright 2012 Richard Crowley. All rights reserved.
|
||||||
|
|
||||||
|
Redistribution and use in source and binary forms, with or without
|
||||||
|
modification, are permitted provided that the following conditions are
|
||||||
|
met:
|
||||||
|
|
||||||
|
1. Redistributions of source code must retain the above copyright
|
||||||
|
notice, this list of conditions and the following disclaimer.
|
||||||
|
|
||||||
|
2. Redistributions in binary form must reproduce the above
|
||||||
|
copyright notice, this list of conditions and the following
|
||||||
|
disclaimer in the documentation and/or other materials provided
|
||||||
|
with the distribution.
|
||||||
|
|
||||||
|
THIS SOFTWARE IS PROVIDED BY RICHARD CROWLEY ``AS IS'' AND ANY EXPRESS
|
||||||
|
OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
||||||
|
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||||
|
DISCLAIMED. IN NO EVENT SHALL RICHARD CROWLEY OR CONTRIBUTORS BE LIABLE
|
||||||
|
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
||||||
|
CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||||
|
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||||
|
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||||
|
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||||
|
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
|
||||||
|
THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
|
||||||
|
The views and conclusions contained in the software and documentation
|
||||||
|
are those of the authors and should not be interpreted as representing
|
||||||
|
official policies, either expressed or implied, of Richard Crowley.
|
||||||
+126
@@ -0,0 +1,126 @@
|
|||||||
|
go-metrics
|
||||||
|
==========
|
||||||
|
|
||||||
|

|
||||||
|
|
||||||
|
Go port of Coda Hale's Metrics library: <https://github.com/dropwizard/metrics>.
|
||||||
|
|
||||||
|
Documentation: <http://godoc.org/github.com/rcrowley/go-metrics>.
|
||||||
|
|
||||||
|
Usage
|
||||||
|
-----
|
||||||
|
|
||||||
|
Create and update metrics:
|
||||||
|
|
||||||
|
```go
|
||||||
|
c := metrics.NewCounter()
|
||||||
|
metrics.Register("foo", c)
|
||||||
|
c.Inc(47)
|
||||||
|
|
||||||
|
g := metrics.NewGauge()
|
||||||
|
metrics.Register("bar", g)
|
||||||
|
g.Update(47)
|
||||||
|
|
||||||
|
s := metrics.NewExpDecaySample(1028, 0.015) // or metrics.NewUniformSample(1028)
|
||||||
|
h := metrics.NewHistogram(s)
|
||||||
|
metrics.Register("baz", h)
|
||||||
|
h.Update(47)
|
||||||
|
|
||||||
|
m := metrics.NewMeter()
|
||||||
|
metrics.Register("quux", m)
|
||||||
|
m.Mark(47)
|
||||||
|
|
||||||
|
t := metrics.NewTimer()
|
||||||
|
metrics.Register("bang", t)
|
||||||
|
t.Time(func() {})
|
||||||
|
t.Update(47)
|
||||||
|
```
|
||||||
|
|
||||||
|
Periodically log every metric in human-readable form to standard error:
|
||||||
|
|
||||||
|
```go
|
||||||
|
go metrics.Log(metrics.DefaultRegistry, 60e9, log.New(os.Stderr, "metrics: ", log.Lmicroseconds))
|
||||||
|
```
|
||||||
|
|
||||||
|
Periodically log every metric in slightly-more-parseable form to syslog:
|
||||||
|
|
||||||
|
```go
|
||||||
|
w, _ := syslog.Dial("unixgram", "/dev/log", syslog.LOG_INFO, "metrics")
|
||||||
|
go metrics.Syslog(metrics.DefaultRegistry, 60e9, w)
|
||||||
|
```
|
||||||
|
|
||||||
|
Periodically emit every metric to Graphite using the [Graphite client](https://github.com/cyberdelia/go-metrics-graphite):
|
||||||
|
|
||||||
|
```go
|
||||||
|
|
||||||
|
import "github.com/cyberdelia/go-metrics-graphite"
|
||||||
|
|
||||||
|
addr, _ := net.ResolveTCPAddr("tcp", "127.0.0.1:2003")
|
||||||
|
go graphite.Graphite(metrics.DefaultRegistry, 10e9, "metrics", addr)
|
||||||
|
```
|
||||||
|
|
||||||
|
Periodically emit every metric into InfluxDB:
|
||||||
|
|
||||||
|
**NOTE:** this has been pulled out of the library due to constant fluctuations
|
||||||
|
in the InfluxDB API. In fact, all client libraries are on their way out. see
|
||||||
|
issues [#121](https://github.com/rcrowley/go-metrics/issues/121) and
|
||||||
|
[#124](https://github.com/rcrowley/go-metrics/issues/124) for progress and details.
|
||||||
|
|
||||||
|
```go
|
||||||
|
import "github.com/rcrowley/go-metrics/influxdb"
|
||||||
|
|
||||||
|
go influxdb.Influxdb(metrics.DefaultRegistry, 10e9, &influxdb.Config{
|
||||||
|
Host: "127.0.0.1:8086",
|
||||||
|
Database: "metrics",
|
||||||
|
Username: "test",
|
||||||
|
Password: "test",
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
Periodically upload every metric to Librato using the [Librato client](https://github.com/mihasya/go-metrics-librato):
|
||||||
|
|
||||||
|
**Note**: the client included with this repository under the `librato` package
|
||||||
|
has been deprecated and moved to the repository linked above.
|
||||||
|
|
||||||
|
```go
|
||||||
|
import "github.com/mihasya/go-metrics-librato"
|
||||||
|
|
||||||
|
go librato.Librato(metrics.DefaultRegistry,
|
||||||
|
10e9, // interval
|
||||||
|
"example@example.com", // account owner email address
|
||||||
|
"token", // Librato API token
|
||||||
|
"hostname", // source
|
||||||
|
[]float64{0.95}, // percentiles to send
|
||||||
|
time.Millisecond, // time unit
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Periodically emit every metric to StatHat:
|
||||||
|
|
||||||
|
```go
|
||||||
|
import "github.com/rcrowley/go-metrics/stathat"
|
||||||
|
|
||||||
|
go stathat.Stathat(metrics.DefaultRegistry, 10e9, "example@example.com")
|
||||||
|
```
|
||||||
|
|
||||||
|
Installation
|
||||||
|
------------
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get github.com/rcrowley/go-metrics
|
||||||
|
```
|
||||||
|
|
||||||
|
StatHat support additionally requires their Go client:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get github.com/stathat/go
|
||||||
|
```
|
||||||
|
|
||||||
|
Publishing Metrics
|
||||||
|
------------------
|
||||||
|
|
||||||
|
Clients are available for the following destinations:
|
||||||
|
|
||||||
|
* Librato - [https://github.com/mihasya/go-metrics-librato](https://github.com/mihasya/go-metrics-librato)
|
||||||
|
* Graphite - [https://github.com/cyberdelia/go-metrics-graphite](https://github.com/cyberdelia/go-metrics-graphite)
|
||||||
|
* InfluxDB - [https://github.com/vrischmann/go-metrics-influxdb](https://github.com/vrischmann/go-metrics-influxdb)
|
||||||
Generated
Vendored
+20
@@ -0,0 +1,20 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"github.com/rcrowley/go-metrics"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
r := metrics.NewRegistry()
|
||||||
|
for i := 0; i < 10000; i++ {
|
||||||
|
r.Register(fmt.Sprintf("counter-%d", i), metrics.NewCounter())
|
||||||
|
r.Register(fmt.Sprintf("gauge-%d", i), metrics.NewGauge())
|
||||||
|
r.Register(fmt.Sprintf("gaugefloat64-%d", i), metrics.NewGaugeFloat64())
|
||||||
|
r.Register(fmt.Sprintf("histogram-uniform-%d", i), metrics.NewHistogram(metrics.NewUniformSample(1028)))
|
||||||
|
r.Register(fmt.Sprintf("histogram-exp-%d", i), metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015)))
|
||||||
|
r.Register(fmt.Sprintf("meter-%d", i), metrics.NewMeter())
|
||||||
|
}
|
||||||
|
time.Sleep(600e9)
|
||||||
|
}
|
||||||
Generated
Vendored
+154
@@ -0,0 +1,154 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"github.com/rcrowley/go-metrics"
|
||||||
|
// "github.com/rcrowley/go-metrics/stathat"
|
||||||
|
"log"
|
||||||
|
"math/rand"
|
||||||
|
"os"
|
||||||
|
// "syslog"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const fanout = 10
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
|
||||||
|
r := metrics.NewRegistry()
|
||||||
|
|
||||||
|
c := metrics.NewCounter()
|
||||||
|
r.Register("foo", c)
|
||||||
|
for i := 0; i < fanout; i++ {
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
c.Dec(19)
|
||||||
|
time.Sleep(300e6)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
c.Inc(47)
|
||||||
|
time.Sleep(400e6)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
g := metrics.NewGauge()
|
||||||
|
r.Register("bar", g)
|
||||||
|
for i := 0; i < fanout; i++ {
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
g.Update(19)
|
||||||
|
time.Sleep(300e6)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
g.Update(47)
|
||||||
|
time.Sleep(400e6)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
gf := metrics.NewGaugeFloat64()
|
||||||
|
r.Register("barfloat64", gf)
|
||||||
|
for i := 0; i < fanout; i++ {
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
g.Update(19.0)
|
||||||
|
time.Sleep(300e6)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
g.Update(47.0)
|
||||||
|
time.Sleep(400e6)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
hc := metrics.NewHealthcheck(func(h metrics.Healthcheck) {
|
||||||
|
if 0 < rand.Intn(2) {
|
||||||
|
h.Healthy()
|
||||||
|
} else {
|
||||||
|
h.Unhealthy(errors.New("baz"))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
r.Register("baz", hc)
|
||||||
|
|
||||||
|
s := metrics.NewExpDecaySample(1028, 0.015)
|
||||||
|
//s := metrics.NewUniformSample(1028)
|
||||||
|
h := metrics.NewHistogram(s)
|
||||||
|
r.Register("bang", h)
|
||||||
|
for i := 0; i < fanout; i++ {
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
h.Update(19)
|
||||||
|
time.Sleep(300e6)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
h.Update(47)
|
||||||
|
time.Sleep(400e6)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
m := metrics.NewMeter()
|
||||||
|
r.Register("quux", m)
|
||||||
|
for i := 0; i < fanout; i++ {
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
m.Mark(19)
|
||||||
|
time.Sleep(300e6)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
m.Mark(47)
|
||||||
|
time.Sleep(400e6)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
t := metrics.NewTimer()
|
||||||
|
r.Register("hooah", t)
|
||||||
|
for i := 0; i < fanout; i++ {
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
t.Time(func() { time.Sleep(300e6) })
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
t.Time(func() { time.Sleep(400e6) })
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
metrics.RegisterDebugGCStats(r)
|
||||||
|
go metrics.CaptureDebugGCStats(r, 5e9)
|
||||||
|
|
||||||
|
metrics.RegisterRuntimeMemStats(r)
|
||||||
|
go metrics.CaptureRuntimeMemStats(r, 5e9)
|
||||||
|
|
||||||
|
metrics.Log(r, 60e9, log.New(os.Stderr, "metrics: ", log.Lmicroseconds))
|
||||||
|
|
||||||
|
/*
|
||||||
|
w, err := syslog.Dial("unixgram", "/dev/log", syslog.LOG_INFO, "metrics")
|
||||||
|
if nil != err { log.Fatalln(err) }
|
||||||
|
metrics.Syslog(r, 60e9, w)
|
||||||
|
*/
|
||||||
|
|
||||||
|
/*
|
||||||
|
addr, _ := net.ResolveTCPAddr("tcp", "127.0.0.1:2003")
|
||||||
|
metrics.Graphite(r, 10e9, "metrics", addr)
|
||||||
|
*/
|
||||||
|
|
||||||
|
/*
|
||||||
|
stathat.Stathat(r, 10e9, "example@example.com")
|
||||||
|
*/
|
||||||
|
|
||||||
|
}
|
||||||
Generated
Vendored
+22
@@ -0,0 +1,22 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
"net"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
addr, _ := net.ResolveTCPAddr("tcp", "127.0.0.1:2003")
|
||||||
|
l, err := net.ListenTCP("tcp", addr)
|
||||||
|
if nil != err {
|
||||||
|
log.Fatalln(err)
|
||||||
|
}
|
||||||
|
log.Println("listening", l.Addr())
|
||||||
|
for {
|
||||||
|
c, err := l.AcceptTCP()
|
||||||
|
if nil != err {
|
||||||
|
log.Fatalln(err)
|
||||||
|
}
|
||||||
|
log.Println("accepted", c.RemoteAddr())
|
||||||
|
}
|
||||||
|
}
|
||||||
+112
@@ -0,0 +1,112 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import "sync/atomic"
|
||||||
|
|
||||||
|
// Counters hold an int64 value that can be incremented and decremented.
|
||||||
|
type Counter interface {
|
||||||
|
Clear()
|
||||||
|
Count() int64
|
||||||
|
Dec(int64)
|
||||||
|
Inc(int64)
|
||||||
|
Snapshot() Counter
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetOrRegisterCounter returns an existing Counter or constructs and registers
|
||||||
|
// a new StandardCounter.
|
||||||
|
func GetOrRegisterCounter(name string, r Registry) Counter {
|
||||||
|
if nil == r {
|
||||||
|
r = DefaultRegistry
|
||||||
|
}
|
||||||
|
return r.GetOrRegister(name, NewCounter).(Counter)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewCounter constructs a new StandardCounter.
|
||||||
|
func NewCounter() Counter {
|
||||||
|
if UseNilMetrics {
|
||||||
|
return NilCounter{}
|
||||||
|
}
|
||||||
|
return &StandardCounter{0}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRegisteredCounter constructs and registers a new StandardCounter.
|
||||||
|
func NewRegisteredCounter(name string, r Registry) Counter {
|
||||||
|
c := NewCounter()
|
||||||
|
if nil == r {
|
||||||
|
r = DefaultRegistry
|
||||||
|
}
|
||||||
|
r.Register(name, c)
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
// CounterSnapshot is a read-only copy of another Counter.
|
||||||
|
type CounterSnapshot int64
|
||||||
|
|
||||||
|
// Clear panics.
|
||||||
|
func (CounterSnapshot) Clear() {
|
||||||
|
panic("Clear called on a CounterSnapshot")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count returns the count at the time the snapshot was taken.
|
||||||
|
func (c CounterSnapshot) Count() int64 { return int64(c) }
|
||||||
|
|
||||||
|
// Dec panics.
|
||||||
|
func (CounterSnapshot) Dec(int64) {
|
||||||
|
panic("Dec called on a CounterSnapshot")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Inc panics.
|
||||||
|
func (CounterSnapshot) Inc(int64) {
|
||||||
|
panic("Inc called on a CounterSnapshot")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot returns the snapshot.
|
||||||
|
func (c CounterSnapshot) Snapshot() Counter { return c }
|
||||||
|
|
||||||
|
// NilCounter is a no-op Counter.
|
||||||
|
type NilCounter struct{}
|
||||||
|
|
||||||
|
// Clear is a no-op.
|
||||||
|
func (NilCounter) Clear() {}
|
||||||
|
|
||||||
|
// Count is a no-op.
|
||||||
|
func (NilCounter) Count() int64 { return 0 }
|
||||||
|
|
||||||
|
// Dec is a no-op.
|
||||||
|
func (NilCounter) Dec(i int64) {}
|
||||||
|
|
||||||
|
// Inc is a no-op.
|
||||||
|
func (NilCounter) Inc(i int64) {}
|
||||||
|
|
||||||
|
// Snapshot is a no-op.
|
||||||
|
func (NilCounter) Snapshot() Counter { return NilCounter{} }
|
||||||
|
|
||||||
|
// StandardCounter is the standard implementation of a Counter and uses the
|
||||||
|
// sync/atomic package to manage a single int64 value.
|
||||||
|
type StandardCounter struct {
|
||||||
|
count int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clear sets the counter to zero.
|
||||||
|
func (c *StandardCounter) Clear() {
|
||||||
|
atomic.StoreInt64(&c.count, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count returns the current count.
|
||||||
|
func (c *StandardCounter) Count() int64 {
|
||||||
|
return atomic.LoadInt64(&c.count)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dec decrements the counter by the given amount.
|
||||||
|
func (c *StandardCounter) Dec(i int64) {
|
||||||
|
atomic.AddInt64(&c.count, -i)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Inc increments the counter by the given amount.
|
||||||
|
func (c *StandardCounter) Inc(i int64) {
|
||||||
|
atomic.AddInt64(&c.count, i)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot returns a read-only copy of the counter.
|
||||||
|
func (c *StandardCounter) Snapshot() Counter {
|
||||||
|
return CounterSnapshot(c.Count())
|
||||||
|
}
|
||||||
+76
@@ -0,0 +1,76 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"runtime/debug"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
debugMetrics struct {
|
||||||
|
GCStats struct {
|
||||||
|
LastGC Gauge
|
||||||
|
NumGC Gauge
|
||||||
|
Pause Histogram
|
||||||
|
//PauseQuantiles Histogram
|
||||||
|
PauseTotal Gauge
|
||||||
|
}
|
||||||
|
ReadGCStats Timer
|
||||||
|
}
|
||||||
|
gcStats debug.GCStats
|
||||||
|
)
|
||||||
|
|
||||||
|
// Capture new values for the Go garbage collector statistics exported in
|
||||||
|
// debug.GCStats. This is designed to be called as a goroutine.
|
||||||
|
func CaptureDebugGCStats(r Registry, d time.Duration) {
|
||||||
|
for _ = range time.Tick(d) {
|
||||||
|
CaptureDebugGCStatsOnce(r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Capture new values for the Go garbage collector statistics exported in
|
||||||
|
// debug.GCStats. This is designed to be called in a background goroutine.
|
||||||
|
// Giving a registry which has not been given to RegisterDebugGCStats will
|
||||||
|
// panic.
|
||||||
|
//
|
||||||
|
// Be careful (but much less so) with this because debug.ReadGCStats calls
|
||||||
|
// the C function runtime·lock(runtime·mheap) which, while not a stop-the-world
|
||||||
|
// operation, isn't something you want to be doing all the time.
|
||||||
|
func CaptureDebugGCStatsOnce(r Registry) {
|
||||||
|
lastGC := gcStats.LastGC
|
||||||
|
t := time.Now()
|
||||||
|
debug.ReadGCStats(&gcStats)
|
||||||
|
debugMetrics.ReadGCStats.UpdateSince(t)
|
||||||
|
|
||||||
|
debugMetrics.GCStats.LastGC.Update(int64(gcStats.LastGC.UnixNano()))
|
||||||
|
debugMetrics.GCStats.NumGC.Update(int64(gcStats.NumGC))
|
||||||
|
if lastGC != gcStats.LastGC && 0 < len(gcStats.Pause) {
|
||||||
|
debugMetrics.GCStats.Pause.Update(int64(gcStats.Pause[0]))
|
||||||
|
}
|
||||||
|
//debugMetrics.GCStats.PauseQuantiles.Update(gcStats.PauseQuantiles)
|
||||||
|
debugMetrics.GCStats.PauseTotal.Update(int64(gcStats.PauseTotal))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register metrics for the Go garbage collector statistics exported in
|
||||||
|
// debug.GCStats. The metrics are named by their fully-qualified Go symbols,
|
||||||
|
// i.e. debug.GCStats.PauseTotal.
|
||||||
|
func RegisterDebugGCStats(r Registry) {
|
||||||
|
debugMetrics.GCStats.LastGC = NewGauge()
|
||||||
|
debugMetrics.GCStats.NumGC = NewGauge()
|
||||||
|
debugMetrics.GCStats.Pause = NewHistogram(NewExpDecaySample(1028, 0.015))
|
||||||
|
//debugMetrics.GCStats.PauseQuantiles = NewHistogram(NewExpDecaySample(1028, 0.015))
|
||||||
|
debugMetrics.GCStats.PauseTotal = NewGauge()
|
||||||
|
debugMetrics.ReadGCStats = NewTimer()
|
||||||
|
|
||||||
|
r.Register("debug.GCStats.LastGC", debugMetrics.GCStats.LastGC)
|
||||||
|
r.Register("debug.GCStats.NumGC", debugMetrics.GCStats.NumGC)
|
||||||
|
r.Register("debug.GCStats.Pause", debugMetrics.GCStats.Pause)
|
||||||
|
//r.Register("debug.GCStats.PauseQuantiles", debugMetrics.GCStats.PauseQuantiles)
|
||||||
|
r.Register("debug.GCStats.PauseTotal", debugMetrics.GCStats.PauseTotal)
|
||||||
|
r.Register("debug.ReadGCStats", debugMetrics.ReadGCStats)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Allocate an initial slice for gcStats.Pause to avoid allocations during
|
||||||
|
// normal operation.
|
||||||
|
func init() {
|
||||||
|
gcStats.Pause = make([]time.Duration, 11)
|
||||||
|
}
|
||||||
+118
@@ -0,0 +1,118 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
)
|
||||||
|
|
||||||
|
// EWMAs continuously calculate an exponentially-weighted moving average
|
||||||
|
// based on an outside source of clock ticks.
|
||||||
|
type EWMA interface {
|
||||||
|
Rate() float64
|
||||||
|
Snapshot() EWMA
|
||||||
|
Tick()
|
||||||
|
Update(int64)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewEWMA constructs a new EWMA with the given alpha.
|
||||||
|
func NewEWMA(alpha float64) EWMA {
|
||||||
|
if UseNilMetrics {
|
||||||
|
return NilEWMA{}
|
||||||
|
}
|
||||||
|
return &StandardEWMA{alpha: alpha}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewEWMA1 constructs a new EWMA for a one-minute moving average.
|
||||||
|
func NewEWMA1() EWMA {
|
||||||
|
return NewEWMA(1 - math.Exp(-5.0/60.0/1))
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewEWMA5 constructs a new EWMA for a five-minute moving average.
|
||||||
|
func NewEWMA5() EWMA {
|
||||||
|
return NewEWMA(1 - math.Exp(-5.0/60.0/5))
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewEWMA15 constructs a new EWMA for a fifteen-minute moving average.
|
||||||
|
func NewEWMA15() EWMA {
|
||||||
|
return NewEWMA(1 - math.Exp(-5.0/60.0/15))
|
||||||
|
}
|
||||||
|
|
||||||
|
// EWMASnapshot is a read-only copy of another EWMA.
|
||||||
|
type EWMASnapshot float64
|
||||||
|
|
||||||
|
// Rate returns the rate of events per second at the time the snapshot was
|
||||||
|
// taken.
|
||||||
|
func (a EWMASnapshot) Rate() float64 { return float64(a) }
|
||||||
|
|
||||||
|
// Snapshot returns the snapshot.
|
||||||
|
func (a EWMASnapshot) Snapshot() EWMA { return a }
|
||||||
|
|
||||||
|
// Tick panics.
|
||||||
|
func (EWMASnapshot) Tick() {
|
||||||
|
panic("Tick called on an EWMASnapshot")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update panics.
|
||||||
|
func (EWMASnapshot) Update(int64) {
|
||||||
|
panic("Update called on an EWMASnapshot")
|
||||||
|
}
|
||||||
|
|
||||||
|
// NilEWMA is a no-op EWMA.
|
||||||
|
type NilEWMA struct{}
|
||||||
|
|
||||||
|
// Rate is a no-op.
|
||||||
|
func (NilEWMA) Rate() float64 { return 0.0 }
|
||||||
|
|
||||||
|
// Snapshot is a no-op.
|
||||||
|
func (NilEWMA) Snapshot() EWMA { return NilEWMA{} }
|
||||||
|
|
||||||
|
// Tick is a no-op.
|
||||||
|
func (NilEWMA) Tick() {}
|
||||||
|
|
||||||
|
// Update is a no-op.
|
||||||
|
func (NilEWMA) Update(n int64) {}
|
||||||
|
|
||||||
|
// StandardEWMA is the standard implementation of an EWMA and tracks the number
|
||||||
|
// of uncounted events and processes them on each tick. It uses the
|
||||||
|
// sync/atomic package to manage uncounted events.
|
||||||
|
type StandardEWMA struct {
|
||||||
|
uncounted int64 // /!\ this should be the first member to ensure 64-bit alignment
|
||||||
|
alpha float64
|
||||||
|
rate float64
|
||||||
|
init bool
|
||||||
|
mutex sync.Mutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rate returns the moving average rate of events per second.
|
||||||
|
func (a *StandardEWMA) Rate() float64 {
|
||||||
|
a.mutex.Lock()
|
||||||
|
defer a.mutex.Unlock()
|
||||||
|
return a.rate * float64(1e9)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot returns a read-only copy of the EWMA.
|
||||||
|
func (a *StandardEWMA) Snapshot() EWMA {
|
||||||
|
return EWMASnapshot(a.Rate())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Tick ticks the clock to update the moving average. It assumes it is called
|
||||||
|
// every five seconds.
|
||||||
|
func (a *StandardEWMA) Tick() {
|
||||||
|
count := atomic.LoadInt64(&a.uncounted)
|
||||||
|
atomic.AddInt64(&a.uncounted, -count)
|
||||||
|
instantRate := float64(count) / float64(5e9)
|
||||||
|
a.mutex.Lock()
|
||||||
|
defer a.mutex.Unlock()
|
||||||
|
if a.init {
|
||||||
|
a.rate += a.alpha * (instantRate - a.rate)
|
||||||
|
} else {
|
||||||
|
a.init = true
|
||||||
|
a.rate = instantRate
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update adds n uncounted events.
|
||||||
|
func (a *StandardEWMA) Update(n int64) {
|
||||||
|
atomic.AddInt64(&a.uncounted, n)
|
||||||
|
}
|
||||||
+84
@@ -0,0 +1,84 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import "sync/atomic"
|
||||||
|
|
||||||
|
// Gauges hold an int64 value that can be set arbitrarily.
|
||||||
|
type Gauge interface {
|
||||||
|
Snapshot() Gauge
|
||||||
|
Update(int64)
|
||||||
|
Value() int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetOrRegisterGauge returns an existing Gauge or constructs and registers a
|
||||||
|
// new StandardGauge.
|
||||||
|
func GetOrRegisterGauge(name string, r Registry) Gauge {
|
||||||
|
if nil == r {
|
||||||
|
r = DefaultRegistry
|
||||||
|
}
|
||||||
|
return r.GetOrRegister(name, NewGauge).(Gauge)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewGauge constructs a new StandardGauge.
|
||||||
|
func NewGauge() Gauge {
|
||||||
|
if UseNilMetrics {
|
||||||
|
return NilGauge{}
|
||||||
|
}
|
||||||
|
return &StandardGauge{0}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRegisteredGauge constructs and registers a new StandardGauge.
|
||||||
|
func NewRegisteredGauge(name string, r Registry) Gauge {
|
||||||
|
c := NewGauge()
|
||||||
|
if nil == r {
|
||||||
|
r = DefaultRegistry
|
||||||
|
}
|
||||||
|
r.Register(name, c)
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
// GaugeSnapshot is a read-only copy of another Gauge.
|
||||||
|
type GaugeSnapshot int64
|
||||||
|
|
||||||
|
// Snapshot returns the snapshot.
|
||||||
|
func (g GaugeSnapshot) Snapshot() Gauge { return g }
|
||||||
|
|
||||||
|
// Update panics.
|
||||||
|
func (GaugeSnapshot) Update(int64) {
|
||||||
|
panic("Update called on a GaugeSnapshot")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Value returns the value at the time the snapshot was taken.
|
||||||
|
func (g GaugeSnapshot) Value() int64 { return int64(g) }
|
||||||
|
|
||||||
|
// NilGauge is a no-op Gauge.
|
||||||
|
type NilGauge struct{}
|
||||||
|
|
||||||
|
// Snapshot is a no-op.
|
||||||
|
func (NilGauge) Snapshot() Gauge { return NilGauge{} }
|
||||||
|
|
||||||
|
// Update is a no-op.
|
||||||
|
func (NilGauge) Update(v int64) {}
|
||||||
|
|
||||||
|
// Value is a no-op.
|
||||||
|
func (NilGauge) Value() int64 { return 0 }
|
||||||
|
|
||||||
|
// StandardGauge is the standard implementation of a Gauge and uses the
|
||||||
|
// sync/atomic package to manage a single int64 value.
|
||||||
|
type StandardGauge struct {
|
||||||
|
value int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot returns a read-only copy of the gauge.
|
||||||
|
func (g *StandardGauge) Snapshot() Gauge {
|
||||||
|
return GaugeSnapshot(g.Value())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update updates the gauge's value.
|
||||||
|
func (g *StandardGauge) Update(v int64) {
|
||||||
|
atomic.StoreInt64(&g.value, v)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Value returns the gauge's current value.
|
||||||
|
func (g *StandardGauge) Value() int64 {
|
||||||
|
return atomic.LoadInt64(&g.value)
|
||||||
|
}
|
||||||
+91
@@ -0,0 +1,91 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import "sync"
|
||||||
|
|
||||||
|
// GaugeFloat64s hold a float64 value that can be set arbitrarily.
|
||||||
|
type GaugeFloat64 interface {
|
||||||
|
Snapshot() GaugeFloat64
|
||||||
|
Update(float64)
|
||||||
|
Value() float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetOrRegisterGaugeFloat64 returns an existing GaugeFloat64 or constructs and registers a
|
||||||
|
// new StandardGaugeFloat64.
|
||||||
|
func GetOrRegisterGaugeFloat64(name string, r Registry) GaugeFloat64 {
|
||||||
|
if nil == r {
|
||||||
|
r = DefaultRegistry
|
||||||
|
}
|
||||||
|
return r.GetOrRegister(name, NewGaugeFloat64()).(GaugeFloat64)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewGaugeFloat64 constructs a new StandardGaugeFloat64.
|
||||||
|
func NewGaugeFloat64() GaugeFloat64 {
|
||||||
|
if UseNilMetrics {
|
||||||
|
return NilGaugeFloat64{}
|
||||||
|
}
|
||||||
|
return &StandardGaugeFloat64{
|
||||||
|
value: 0.0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRegisteredGaugeFloat64 constructs and registers a new StandardGaugeFloat64.
|
||||||
|
func NewRegisteredGaugeFloat64(name string, r Registry) GaugeFloat64 {
|
||||||
|
c := NewGaugeFloat64()
|
||||||
|
if nil == r {
|
||||||
|
r = DefaultRegistry
|
||||||
|
}
|
||||||
|
r.Register(name, c)
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
// GaugeFloat64Snapshot is a read-only copy of another GaugeFloat64.
|
||||||
|
type GaugeFloat64Snapshot float64
|
||||||
|
|
||||||
|
// Snapshot returns the snapshot.
|
||||||
|
func (g GaugeFloat64Snapshot) Snapshot() GaugeFloat64 { return g }
|
||||||
|
|
||||||
|
// Update panics.
|
||||||
|
func (GaugeFloat64Snapshot) Update(float64) {
|
||||||
|
panic("Update called on a GaugeFloat64Snapshot")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Value returns the value at the time the snapshot was taken.
|
||||||
|
func (g GaugeFloat64Snapshot) Value() float64 { return float64(g) }
|
||||||
|
|
||||||
|
// NilGauge is a no-op Gauge.
|
||||||
|
type NilGaugeFloat64 struct{}
|
||||||
|
|
||||||
|
// Snapshot is a no-op.
|
||||||
|
func (NilGaugeFloat64) Snapshot() GaugeFloat64 { return NilGaugeFloat64{} }
|
||||||
|
|
||||||
|
// Update is a no-op.
|
||||||
|
func (NilGaugeFloat64) Update(v float64) {}
|
||||||
|
|
||||||
|
// Value is a no-op.
|
||||||
|
func (NilGaugeFloat64) Value() float64 { return 0.0 }
|
||||||
|
|
||||||
|
// StandardGaugeFloat64 is the standard implementation of a GaugeFloat64 and uses
|
||||||
|
// sync.Mutex to manage a single float64 value.
|
||||||
|
type StandardGaugeFloat64 struct {
|
||||||
|
mutex sync.Mutex
|
||||||
|
value float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot returns a read-only copy of the gauge.
|
||||||
|
func (g *StandardGaugeFloat64) Snapshot() GaugeFloat64 {
|
||||||
|
return GaugeFloat64Snapshot(g.Value())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update updates the gauge's value.
|
||||||
|
func (g *StandardGaugeFloat64) Update(v float64) {
|
||||||
|
g.mutex.Lock()
|
||||||
|
defer g.mutex.Unlock()
|
||||||
|
g.value = v
|
||||||
|
}
|
||||||
|
|
||||||
|
// Value returns the gauge's current value.
|
||||||
|
func (g *StandardGaugeFloat64) Value() float64 {
|
||||||
|
g.mutex.Lock()
|
||||||
|
defer g.mutex.Unlock()
|
||||||
|
return g.value
|
||||||
|
}
|
||||||
+113
@@ -0,0 +1,113 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"net"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GraphiteConfig provides a container with configuration parameters for
|
||||||
|
// the Graphite exporter
|
||||||
|
type GraphiteConfig struct {
|
||||||
|
Addr *net.TCPAddr // Network address to connect to
|
||||||
|
Registry Registry // Registry to be exported
|
||||||
|
FlushInterval time.Duration // Flush interval
|
||||||
|
DurationUnit time.Duration // Time conversion unit for durations
|
||||||
|
Prefix string // Prefix to be prepended to metric names
|
||||||
|
Percentiles []float64 // Percentiles to export from timers and histograms
|
||||||
|
}
|
||||||
|
|
||||||
|
// Graphite is a blocking exporter function which reports metrics in r
|
||||||
|
// to a graphite server located at addr, flushing them every d duration
|
||||||
|
// and prepending metric names with prefix.
|
||||||
|
func Graphite(r Registry, d time.Duration, prefix string, addr *net.TCPAddr) {
|
||||||
|
GraphiteWithConfig(GraphiteConfig{
|
||||||
|
Addr: addr,
|
||||||
|
Registry: r,
|
||||||
|
FlushInterval: d,
|
||||||
|
DurationUnit: time.Nanosecond,
|
||||||
|
Prefix: prefix,
|
||||||
|
Percentiles: []float64{0.5, 0.75, 0.95, 0.99, 0.999},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// GraphiteWithConfig is a blocking exporter function just like Graphite,
|
||||||
|
// but it takes a GraphiteConfig instead.
|
||||||
|
func GraphiteWithConfig(c GraphiteConfig) {
|
||||||
|
log.Printf("WARNING: This go-metrics client has been DEPRECATED! It has been moved to https://github.com/cyberdelia/go-metrics-graphite and will be removed from rcrowley/go-metrics on August 12th 2015")
|
||||||
|
for _ = range time.Tick(c.FlushInterval) {
|
||||||
|
if err := graphite(&c); nil != err {
|
||||||
|
log.Println(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GraphiteOnce performs a single submission to Graphite, returning a
|
||||||
|
// non-nil error on failed connections. This can be used in a loop
|
||||||
|
// similar to GraphiteWithConfig for custom error handling.
|
||||||
|
func GraphiteOnce(c GraphiteConfig) error {
|
||||||
|
log.Printf("WARNING: This go-metrics client has been DEPRECATED! It has been moved to https://github.com/cyberdelia/go-metrics-graphite and will be removed from rcrowley/go-metrics on August 12th 2015")
|
||||||
|
return graphite(&c)
|
||||||
|
}
|
||||||
|
|
||||||
|
func graphite(c *GraphiteConfig) error {
|
||||||
|
now := time.Now().Unix()
|
||||||
|
du := float64(c.DurationUnit)
|
||||||
|
conn, err := net.DialTCP("tcp", nil, c.Addr)
|
||||||
|
if nil != err {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
w := bufio.NewWriter(conn)
|
||||||
|
c.Registry.Each(func(name string, i interface{}) {
|
||||||
|
switch metric := i.(type) {
|
||||||
|
case Counter:
|
||||||
|
fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, metric.Count(), now)
|
||||||
|
case Gauge:
|
||||||
|
fmt.Fprintf(w, "%s.%s.value %d %d\n", c.Prefix, name, metric.Value(), now)
|
||||||
|
case GaugeFloat64:
|
||||||
|
fmt.Fprintf(w, "%s.%s.value %f %d\n", c.Prefix, name, metric.Value(), now)
|
||||||
|
case Histogram:
|
||||||
|
h := metric.Snapshot()
|
||||||
|
ps := h.Percentiles(c.Percentiles)
|
||||||
|
fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, h.Count(), now)
|
||||||
|
fmt.Fprintf(w, "%s.%s.min %d %d\n", c.Prefix, name, h.Min(), now)
|
||||||
|
fmt.Fprintf(w, "%s.%s.max %d %d\n", c.Prefix, name, h.Max(), now)
|
||||||
|
fmt.Fprintf(w, "%s.%s.mean %.2f %d\n", c.Prefix, name, h.Mean(), now)
|
||||||
|
fmt.Fprintf(w, "%s.%s.std-dev %.2f %d\n", c.Prefix, name, h.StdDev(), now)
|
||||||
|
for psIdx, psKey := range c.Percentiles {
|
||||||
|
key := strings.Replace(strconv.FormatFloat(psKey*100.0, 'f', -1, 64), ".", "", 1)
|
||||||
|
fmt.Fprintf(w, "%s.%s.%s-percentile %.2f %d\n", c.Prefix, name, key, ps[psIdx], now)
|
||||||
|
}
|
||||||
|
case Meter:
|
||||||
|
m := metric.Snapshot()
|
||||||
|
fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, m.Count(), now)
|
||||||
|
fmt.Fprintf(w, "%s.%s.one-minute %.2f %d\n", c.Prefix, name, m.Rate1(), now)
|
||||||
|
fmt.Fprintf(w, "%s.%s.five-minute %.2f %d\n", c.Prefix, name, m.Rate5(), now)
|
||||||
|
fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, m.Rate15(), now)
|
||||||
|
fmt.Fprintf(w, "%s.%s.mean %.2f %d\n", c.Prefix, name, m.RateMean(), now)
|
||||||
|
case Timer:
|
||||||
|
t := metric.Snapshot()
|
||||||
|
ps := t.Percentiles(c.Percentiles)
|
||||||
|
fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, t.Count(), now)
|
||||||
|
fmt.Fprintf(w, "%s.%s.min %d %d\n", c.Prefix, name, t.Min()/int64(du), now)
|
||||||
|
fmt.Fprintf(w, "%s.%s.max %d %d\n", c.Prefix, name, t.Max()/int64(du), now)
|
||||||
|
fmt.Fprintf(w, "%s.%s.mean %.2f %d\n", c.Prefix, name, t.Mean()/du, now)
|
||||||
|
fmt.Fprintf(w, "%s.%s.std-dev %.2f %d\n", c.Prefix, name, t.StdDev()/du, now)
|
||||||
|
for psIdx, psKey := range c.Percentiles {
|
||||||
|
key := strings.Replace(strconv.FormatFloat(psKey*100.0, 'f', -1, 64), ".", "", 1)
|
||||||
|
fmt.Fprintf(w, "%s.%s.%s-percentile %.2f %d\n", c.Prefix, name, key, ps[psIdx], now)
|
||||||
|
}
|
||||||
|
fmt.Fprintf(w, "%s.%s.one-minute %.2f %d\n", c.Prefix, name, t.Rate1(), now)
|
||||||
|
fmt.Fprintf(w, "%s.%s.five-minute %.2f %d\n", c.Prefix, name, t.Rate5(), now)
|
||||||
|
fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, t.Rate15(), now)
|
||||||
|
fmt.Fprintf(w, "%s.%s.mean-rate %.2f %d\n", c.Prefix, name, t.RateMean(), now)
|
||||||
|
}
|
||||||
|
w.Flush()
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
+61
@@ -0,0 +1,61 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
// Healthchecks hold an error value describing an arbitrary up/down status.
|
||||||
|
type Healthcheck interface {
|
||||||
|
Check()
|
||||||
|
Error() error
|
||||||
|
Healthy()
|
||||||
|
Unhealthy(error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewHealthcheck constructs a new Healthcheck which will use the given
|
||||||
|
// function to update its status.
|
||||||
|
func NewHealthcheck(f func(Healthcheck)) Healthcheck {
|
||||||
|
if UseNilMetrics {
|
||||||
|
return NilHealthcheck{}
|
||||||
|
}
|
||||||
|
return &StandardHealthcheck{nil, f}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NilHealthcheck is a no-op.
|
||||||
|
type NilHealthcheck struct{}
|
||||||
|
|
||||||
|
// Check is a no-op.
|
||||||
|
func (NilHealthcheck) Check() {}
|
||||||
|
|
||||||
|
// Error is a no-op.
|
||||||
|
func (NilHealthcheck) Error() error { return nil }
|
||||||
|
|
||||||
|
// Healthy is a no-op.
|
||||||
|
func (NilHealthcheck) Healthy() {}
|
||||||
|
|
||||||
|
// Unhealthy is a no-op.
|
||||||
|
func (NilHealthcheck) Unhealthy(error) {}
|
||||||
|
|
||||||
|
// StandardHealthcheck is the standard implementation of a Healthcheck and
|
||||||
|
// stores the status and a function to call to update the status.
|
||||||
|
type StandardHealthcheck struct {
|
||||||
|
err error
|
||||||
|
f func(Healthcheck)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check runs the healthcheck function to update the healthcheck's status.
|
||||||
|
func (h *StandardHealthcheck) Check() {
|
||||||
|
h.f(h)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Error returns the healthcheck's status, which will be nil if it is healthy.
|
||||||
|
func (h *StandardHealthcheck) Error() error {
|
||||||
|
return h.err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Healthy marks the healthcheck as healthy.
|
||||||
|
func (h *StandardHealthcheck) Healthy() {
|
||||||
|
h.err = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unhealthy marks the healthcheck as unhealthy. The error is stored and
|
||||||
|
// may be retrieved by the Error method.
|
||||||
|
func (h *StandardHealthcheck) Unhealthy(err error) {
|
||||||
|
h.err = err
|
||||||
|
}
|
||||||
+202
@@ -0,0 +1,202 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
// Histograms calculate distribution statistics from a series of int64 values.
|
||||||
|
type Histogram interface {
|
||||||
|
Clear()
|
||||||
|
Count() int64
|
||||||
|
Max() int64
|
||||||
|
Mean() float64
|
||||||
|
Min() int64
|
||||||
|
Percentile(float64) float64
|
||||||
|
Percentiles([]float64) []float64
|
||||||
|
Sample() Sample
|
||||||
|
Snapshot() Histogram
|
||||||
|
StdDev() float64
|
||||||
|
Sum() int64
|
||||||
|
Update(int64)
|
||||||
|
Variance() float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetOrRegisterHistogram returns an existing Histogram or constructs and
|
||||||
|
// registers a new StandardHistogram.
|
||||||
|
func GetOrRegisterHistogram(name string, r Registry, s Sample) Histogram {
|
||||||
|
if nil == r {
|
||||||
|
r = DefaultRegistry
|
||||||
|
}
|
||||||
|
return r.GetOrRegister(name, func() Histogram { return NewHistogram(s) }).(Histogram)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewHistogram constructs a new StandardHistogram from a Sample.
|
||||||
|
func NewHistogram(s Sample) Histogram {
|
||||||
|
if UseNilMetrics {
|
||||||
|
return NilHistogram{}
|
||||||
|
}
|
||||||
|
return &StandardHistogram{sample: s}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRegisteredHistogram constructs and registers a new StandardHistogram from
|
||||||
|
// a Sample.
|
||||||
|
func NewRegisteredHistogram(name string, r Registry, s Sample) Histogram {
|
||||||
|
c := NewHistogram(s)
|
||||||
|
if nil == r {
|
||||||
|
r = DefaultRegistry
|
||||||
|
}
|
||||||
|
r.Register(name, c)
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
// HistogramSnapshot is a read-only copy of another Histogram.
|
||||||
|
type HistogramSnapshot struct {
|
||||||
|
sample *SampleSnapshot
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clear panics.
|
||||||
|
func (*HistogramSnapshot) Clear() {
|
||||||
|
panic("Clear called on a HistogramSnapshot")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count returns the number of samples recorded at the time the snapshot was
|
||||||
|
// taken.
|
||||||
|
func (h *HistogramSnapshot) Count() int64 { return h.sample.Count() }
|
||||||
|
|
||||||
|
// Max returns the maximum value in the sample at the time the snapshot was
|
||||||
|
// taken.
|
||||||
|
func (h *HistogramSnapshot) Max() int64 { return h.sample.Max() }
|
||||||
|
|
||||||
|
// Mean returns the mean of the values in the sample at the time the snapshot
|
||||||
|
// was taken.
|
||||||
|
func (h *HistogramSnapshot) Mean() float64 { return h.sample.Mean() }
|
||||||
|
|
||||||
|
// Min returns the minimum value in the sample at the time the snapshot was
|
||||||
|
// taken.
|
||||||
|
func (h *HistogramSnapshot) Min() int64 { return h.sample.Min() }
|
||||||
|
|
||||||
|
// Percentile returns an arbitrary percentile of values in the sample at the
|
||||||
|
// time the snapshot was taken.
|
||||||
|
func (h *HistogramSnapshot) Percentile(p float64) float64 {
|
||||||
|
return h.sample.Percentile(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Percentiles returns a slice of arbitrary percentiles of values in the sample
|
||||||
|
// at the time the snapshot was taken.
|
||||||
|
func (h *HistogramSnapshot) Percentiles(ps []float64) []float64 {
|
||||||
|
return h.sample.Percentiles(ps)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sample returns the Sample underlying the histogram.
|
||||||
|
func (h *HistogramSnapshot) Sample() Sample { return h.sample }
|
||||||
|
|
||||||
|
// Snapshot returns the snapshot.
|
||||||
|
func (h *HistogramSnapshot) Snapshot() Histogram { return h }
|
||||||
|
|
||||||
|
// StdDev returns the standard deviation of the values in the sample at the
|
||||||
|
// time the snapshot was taken.
|
||||||
|
func (h *HistogramSnapshot) StdDev() float64 { return h.sample.StdDev() }
|
||||||
|
|
||||||
|
// Sum returns the sum in the sample at the time the snapshot was taken.
|
||||||
|
func (h *HistogramSnapshot) Sum() int64 { return h.sample.Sum() }
|
||||||
|
|
||||||
|
// Update panics.
|
||||||
|
func (*HistogramSnapshot) Update(int64) {
|
||||||
|
panic("Update called on a HistogramSnapshot")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Variance returns the variance of inputs at the time the snapshot was taken.
|
||||||
|
func (h *HistogramSnapshot) Variance() float64 { return h.sample.Variance() }
|
||||||
|
|
||||||
|
// NilHistogram is a no-op Histogram.
|
||||||
|
type NilHistogram struct{}
|
||||||
|
|
||||||
|
// Clear is a no-op.
|
||||||
|
func (NilHistogram) Clear() {}
|
||||||
|
|
||||||
|
// Count is a no-op.
|
||||||
|
func (NilHistogram) Count() int64 { return 0 }
|
||||||
|
|
||||||
|
// Max is a no-op.
|
||||||
|
func (NilHistogram) Max() int64 { return 0 }
|
||||||
|
|
||||||
|
// Mean is a no-op.
|
||||||
|
func (NilHistogram) Mean() float64 { return 0.0 }
|
||||||
|
|
||||||
|
// Min is a no-op.
|
||||||
|
func (NilHistogram) Min() int64 { return 0 }
|
||||||
|
|
||||||
|
// Percentile is a no-op.
|
||||||
|
func (NilHistogram) Percentile(p float64) float64 { return 0.0 }
|
||||||
|
|
||||||
|
// Percentiles is a no-op.
|
||||||
|
func (NilHistogram) Percentiles(ps []float64) []float64 {
|
||||||
|
return make([]float64, len(ps))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sample is a no-op.
|
||||||
|
func (NilHistogram) Sample() Sample { return NilSample{} }
|
||||||
|
|
||||||
|
// Snapshot is a no-op.
|
||||||
|
func (NilHistogram) Snapshot() Histogram { return NilHistogram{} }
|
||||||
|
|
||||||
|
// StdDev is a no-op.
|
||||||
|
func (NilHistogram) StdDev() float64 { return 0.0 }
|
||||||
|
|
||||||
|
// Sum is a no-op.
|
||||||
|
func (NilHistogram) Sum() int64 { return 0 }
|
||||||
|
|
||||||
|
// Update is a no-op.
|
||||||
|
func (NilHistogram) Update(v int64) {}
|
||||||
|
|
||||||
|
// Variance is a no-op.
|
||||||
|
func (NilHistogram) Variance() float64 { return 0.0 }
|
||||||
|
|
||||||
|
// StandardHistogram is the standard implementation of a Histogram and uses a
|
||||||
|
// Sample to bound its memory use.
|
||||||
|
type StandardHistogram struct {
|
||||||
|
sample Sample
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clear clears the histogram and its sample.
|
||||||
|
func (h *StandardHistogram) Clear() { h.sample.Clear() }
|
||||||
|
|
||||||
|
// Count returns the number of samples recorded since the histogram was last
|
||||||
|
// cleared.
|
||||||
|
func (h *StandardHistogram) Count() int64 { return h.sample.Count() }
|
||||||
|
|
||||||
|
// Max returns the maximum value in the sample.
|
||||||
|
func (h *StandardHistogram) Max() int64 { return h.sample.Max() }
|
||||||
|
|
||||||
|
// Mean returns the mean of the values in the sample.
|
||||||
|
func (h *StandardHistogram) Mean() float64 { return h.sample.Mean() }
|
||||||
|
|
||||||
|
// Min returns the minimum value in the sample.
|
||||||
|
func (h *StandardHistogram) Min() int64 { return h.sample.Min() }
|
||||||
|
|
||||||
|
// Percentile returns an arbitrary percentile of the values in the sample.
|
||||||
|
func (h *StandardHistogram) Percentile(p float64) float64 {
|
||||||
|
return h.sample.Percentile(p)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Percentiles returns a slice of arbitrary percentiles of the values in the
|
||||||
|
// sample.
|
||||||
|
func (h *StandardHistogram) Percentiles(ps []float64) []float64 {
|
||||||
|
return h.sample.Percentiles(ps)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sample returns the Sample underlying the histogram.
|
||||||
|
func (h *StandardHistogram) Sample() Sample { return h.sample }
|
||||||
|
|
||||||
|
// Snapshot returns a read-only copy of the histogram.
|
||||||
|
func (h *StandardHistogram) Snapshot() Histogram {
|
||||||
|
return &HistogramSnapshot{sample: h.sample.Snapshot().(*SampleSnapshot)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// StdDev returns the standard deviation of the values in the sample.
|
||||||
|
func (h *StandardHistogram) StdDev() float64 { return h.sample.StdDev() }
|
||||||
|
|
||||||
|
// Sum returns the sum in the sample.
|
||||||
|
func (h *StandardHistogram) Sum() int64 { return h.sample.Sum() }
|
||||||
|
|
||||||
|
// Update samples a new value.
|
||||||
|
func (h *StandardHistogram) Update(v int64) { h.sample.Update(v) }
|
||||||
|
|
||||||
|
// Variance returns the variance of the values in the sample.
|
||||||
|
func (h *StandardHistogram) Variance() float64 { return h.sample.Variance() }
|
||||||
+83
@@ -0,0 +1,83 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// MarshalJSON returns a byte slice containing a JSON representation of all
|
||||||
|
// the metrics in the Registry.
|
||||||
|
func (r *StandardRegistry) MarshalJSON() ([]byte, error) {
|
||||||
|
data := make(map[string]map[string]interface{})
|
||||||
|
r.Each(func(name string, i interface{}) {
|
||||||
|
values := make(map[string]interface{})
|
||||||
|
switch metric := i.(type) {
|
||||||
|
case Counter:
|
||||||
|
values["count"] = metric.Count()
|
||||||
|
case Gauge:
|
||||||
|
values["value"] = metric.Value()
|
||||||
|
case GaugeFloat64:
|
||||||
|
values["value"] = metric.Value()
|
||||||
|
case Healthcheck:
|
||||||
|
values["error"] = nil
|
||||||
|
metric.Check()
|
||||||
|
if err := metric.Error(); nil != err {
|
||||||
|
values["error"] = metric.Error().Error()
|
||||||
|
}
|
||||||
|
case Histogram:
|
||||||
|
h := metric.Snapshot()
|
||||||
|
ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
|
||||||
|
values["count"] = h.Count()
|
||||||
|
values["min"] = h.Min()
|
||||||
|
values["max"] = h.Max()
|
||||||
|
values["mean"] = h.Mean()
|
||||||
|
values["stddev"] = h.StdDev()
|
||||||
|
values["median"] = ps[0]
|
||||||
|
values["75%"] = ps[1]
|
||||||
|
values["95%"] = ps[2]
|
||||||
|
values["99%"] = ps[3]
|
||||||
|
values["99.9%"] = ps[4]
|
||||||
|
case Meter:
|
||||||
|
m := metric.Snapshot()
|
||||||
|
values["count"] = m.Count()
|
||||||
|
values["1m.rate"] = m.Rate1()
|
||||||
|
values["5m.rate"] = m.Rate5()
|
||||||
|
values["15m.rate"] = m.Rate15()
|
||||||
|
values["mean.rate"] = m.RateMean()
|
||||||
|
case Timer:
|
||||||
|
t := metric.Snapshot()
|
||||||
|
ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
|
||||||
|
values["count"] = t.Count()
|
||||||
|
values["min"] = t.Min()
|
||||||
|
values["max"] = t.Max()
|
||||||
|
values["mean"] = t.Mean()
|
||||||
|
values["stddev"] = t.StdDev()
|
||||||
|
values["median"] = ps[0]
|
||||||
|
values["75%"] = ps[1]
|
||||||
|
values["95%"] = ps[2]
|
||||||
|
values["99%"] = ps[3]
|
||||||
|
values["99.9%"] = ps[4]
|
||||||
|
values["1m.rate"] = t.Rate1()
|
||||||
|
values["5m.rate"] = t.Rate5()
|
||||||
|
values["15m.rate"] = t.Rate15()
|
||||||
|
values["mean.rate"] = t.RateMean()
|
||||||
|
}
|
||||||
|
data[name] = values
|
||||||
|
})
|
||||||
|
return json.Marshal(data)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteJSON writes metrics from the given registry periodically to the
|
||||||
|
// specified io.Writer as JSON.
|
||||||
|
func WriteJSON(r Registry, d time.Duration, w io.Writer) {
|
||||||
|
for _ = range time.Tick(d) {
|
||||||
|
WriteJSONOnce(r, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteJSONOnce writes metrics from the given registry to the specified
|
||||||
|
// io.Writer as JSON.
|
||||||
|
func WriteJSONOnce(r Registry, w io.Writer) {
|
||||||
|
json.NewEncoder(w).Encode(r)
|
||||||
|
}
|
||||||
+102
@@ -0,0 +1,102 @@
|
|||||||
|
package librato
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"net/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
const Operations = "operations"
|
||||||
|
const OperationsShort = "ops"
|
||||||
|
|
||||||
|
type LibratoClient struct {
|
||||||
|
Email, Token string
|
||||||
|
}
|
||||||
|
|
||||||
|
// property strings
|
||||||
|
const (
|
||||||
|
// display attributes
|
||||||
|
Color = "color"
|
||||||
|
DisplayMax = "display_max"
|
||||||
|
DisplayMin = "display_min"
|
||||||
|
DisplayUnitsLong = "display_units_long"
|
||||||
|
DisplayUnitsShort = "display_units_short"
|
||||||
|
DisplayStacked = "display_stacked"
|
||||||
|
DisplayTransform = "display_transform"
|
||||||
|
// special gauge display attributes
|
||||||
|
SummarizeFunction = "summarize_function"
|
||||||
|
Aggregate = "aggregate"
|
||||||
|
|
||||||
|
// metric keys
|
||||||
|
Name = "name"
|
||||||
|
Period = "period"
|
||||||
|
Description = "description"
|
||||||
|
DisplayName = "display_name"
|
||||||
|
Attributes = "attributes"
|
||||||
|
|
||||||
|
// measurement keys
|
||||||
|
MeasureTime = "measure_time"
|
||||||
|
Source = "source"
|
||||||
|
Value = "value"
|
||||||
|
|
||||||
|
// special gauge keys
|
||||||
|
Count = "count"
|
||||||
|
Sum = "sum"
|
||||||
|
Max = "max"
|
||||||
|
Min = "min"
|
||||||
|
SumSquares = "sum_squares"
|
||||||
|
|
||||||
|
// batch keys
|
||||||
|
Counters = "counters"
|
||||||
|
Gauges = "gauges"
|
||||||
|
|
||||||
|
MetricsPostUrl = "https://metrics-api.librato.com/v1/metrics"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Measurement map[string]interface{}
|
||||||
|
type Metric map[string]interface{}
|
||||||
|
|
||||||
|
type Batch struct {
|
||||||
|
Gauges []Measurement `json:"gauges,omitempty"`
|
||||||
|
Counters []Measurement `json:"counters,omitempty"`
|
||||||
|
MeasureTime int64 `json:"measure_time"`
|
||||||
|
Source string `json:"source"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (self *LibratoClient) PostMetrics(batch Batch) (err error) {
|
||||||
|
var (
|
||||||
|
js []byte
|
||||||
|
req *http.Request
|
||||||
|
resp *http.Response
|
||||||
|
)
|
||||||
|
|
||||||
|
if len(batch.Counters) == 0 && len(batch.Gauges) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if js, err = json.Marshal(batch); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if req, err = http.NewRequest("POST", MetricsPostUrl, bytes.NewBuffer(js)); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.SetBasicAuth(self.Email, self.Token)
|
||||||
|
|
||||||
|
if resp, err = http.DefaultClient.Do(req); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
var body []byte
|
||||||
|
if body, err = ioutil.ReadAll(resp.Body); err != nil {
|
||||||
|
body = []byte(fmt.Sprintf("(could not fetch response body for error: %s)", err))
|
||||||
|
}
|
||||||
|
err = fmt.Errorf("Unable to post to Librato: %d %s %s", resp.StatusCode, resp.Status, string(body))
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
+231
@@ -0,0 +1,231 @@
|
|||||||
|
package librato
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"math"
|
||||||
|
"regexp"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/rcrowley/go-metrics"
|
||||||
|
)
|
||||||
|
|
||||||
|
// a regexp for extracting the unit from time.Duration.String
|
||||||
|
var unitRegexp = regexp.MustCompile("[^\\d]+$")
|
||||||
|
|
||||||
|
// a helper that turns a time.Duration into librato display attributes for timer metrics
|
||||||
|
func translateTimerAttributes(d time.Duration) (attrs map[string]interface{}) {
|
||||||
|
attrs = make(map[string]interface{})
|
||||||
|
attrs[DisplayTransform] = fmt.Sprintf("x/%d", int64(d))
|
||||||
|
attrs[DisplayUnitsShort] = string(unitRegexp.Find([]byte(d.String())))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
type Reporter struct {
|
||||||
|
Email, Token string
|
||||||
|
Source string
|
||||||
|
Interval time.Duration
|
||||||
|
Registry metrics.Registry
|
||||||
|
Percentiles []float64 // percentiles to report on histogram metrics
|
||||||
|
TimerAttributes map[string]interface{} // units in which timers will be displayed
|
||||||
|
intervalSec int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewReporter(r metrics.Registry, d time.Duration, e string, t string, s string, p []float64, u time.Duration) *Reporter {
|
||||||
|
return &Reporter{e, t, s, d, r, p, translateTimerAttributes(u), int64(d / time.Second)}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Librato(r metrics.Registry, d time.Duration, e string, t string, s string, p []float64, u time.Duration) {
|
||||||
|
NewReporter(r, d, e, t, s, p, u).Run()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (self *Reporter) Run() {
|
||||||
|
log.Printf("WARNING: This client has been DEPRECATED! It has been moved to https://github.com/mihasya/go-metrics-librato and will be removed from rcrowley/go-metrics on August 5th 2015")
|
||||||
|
ticker := time.Tick(self.Interval)
|
||||||
|
metricsApi := &LibratoClient{self.Email, self.Token}
|
||||||
|
for now := range ticker {
|
||||||
|
var metrics Batch
|
||||||
|
var err error
|
||||||
|
if metrics, err = self.BuildRequest(now, self.Registry); err != nil {
|
||||||
|
log.Printf("ERROR constructing librato request body %s", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := metricsApi.PostMetrics(metrics); err != nil {
|
||||||
|
log.Printf("ERROR sending metrics to librato %s", err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// calculate sum of squares from data provided by metrics.Histogram
|
||||||
|
// see http://en.wikipedia.org/wiki/Standard_deviation#Rapid_calculation_methods
|
||||||
|
func sumSquares(s metrics.Sample) float64 {
|
||||||
|
count := float64(s.Count())
|
||||||
|
sumSquared := math.Pow(count*s.Mean(), 2)
|
||||||
|
sumSquares := math.Pow(count*s.StdDev(), 2) + sumSquared/count
|
||||||
|
if math.IsNaN(sumSquares) {
|
||||||
|
return 0.0
|
||||||
|
}
|
||||||
|
return sumSquares
|
||||||
|
}
|
||||||
|
func sumSquaresTimer(t metrics.Timer) float64 {
|
||||||
|
count := float64(t.Count())
|
||||||
|
sumSquared := math.Pow(count*t.Mean(), 2)
|
||||||
|
sumSquares := math.Pow(count*t.StdDev(), 2) + sumSquared/count
|
||||||
|
if math.IsNaN(sumSquares) {
|
||||||
|
return 0.0
|
||||||
|
}
|
||||||
|
return sumSquares
|
||||||
|
}
|
||||||
|
|
||||||
|
func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Batch, err error) {
|
||||||
|
snapshot = Batch{
|
||||||
|
// coerce timestamps to a stepping fn so that they line up in Librato graphs
|
||||||
|
MeasureTime: (now.Unix() / self.intervalSec) * self.intervalSec,
|
||||||
|
Source: self.Source,
|
||||||
|
}
|
||||||
|
snapshot.Gauges = make([]Measurement, 0)
|
||||||
|
snapshot.Counters = make([]Measurement, 0)
|
||||||
|
histogramGaugeCount := 1 + len(self.Percentiles)
|
||||||
|
r.Each(func(name string, metric interface{}) {
|
||||||
|
measurement := Measurement{}
|
||||||
|
measurement[Period] = self.Interval.Seconds()
|
||||||
|
switch m := metric.(type) {
|
||||||
|
case metrics.Counter:
|
||||||
|
if m.Count() > 0 {
|
||||||
|
measurement[Name] = fmt.Sprintf("%s.%s", name, "count")
|
||||||
|
measurement[Value] = float64(m.Count())
|
||||||
|
measurement[Attributes] = map[string]interface{}{
|
||||||
|
DisplayUnitsLong: Operations,
|
||||||
|
DisplayUnitsShort: OperationsShort,
|
||||||
|
DisplayMin: "0",
|
||||||
|
}
|
||||||
|
snapshot.Counters = append(snapshot.Counters, measurement)
|
||||||
|
}
|
||||||
|
case metrics.Gauge:
|
||||||
|
measurement[Name] = name
|
||||||
|
measurement[Value] = float64(m.Value())
|
||||||
|
snapshot.Gauges = append(snapshot.Gauges, measurement)
|
||||||
|
case metrics.GaugeFloat64:
|
||||||
|
measurement[Name] = name
|
||||||
|
measurement[Value] = float64(m.Value())
|
||||||
|
snapshot.Gauges = append(snapshot.Gauges, measurement)
|
||||||
|
case metrics.Histogram:
|
||||||
|
if m.Count() > 0 {
|
||||||
|
gauges := make([]Measurement, histogramGaugeCount, histogramGaugeCount)
|
||||||
|
s := m.Sample()
|
||||||
|
measurement[Name] = fmt.Sprintf("%s.%s", name, "hist")
|
||||||
|
measurement[Count] = uint64(s.Count())
|
||||||
|
measurement[Max] = float64(s.Max())
|
||||||
|
measurement[Min] = float64(s.Min())
|
||||||
|
measurement[Sum] = float64(s.Sum())
|
||||||
|
measurement[SumSquares] = sumSquares(s)
|
||||||
|
gauges[0] = measurement
|
||||||
|
for i, p := range self.Percentiles {
|
||||||
|
gauges[i+1] = Measurement{
|
||||||
|
Name: fmt.Sprintf("%s.%.2f", measurement[Name], p),
|
||||||
|
Value: s.Percentile(p),
|
||||||
|
Period: measurement[Period],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
snapshot.Gauges = append(snapshot.Gauges, gauges...)
|
||||||
|
}
|
||||||
|
case metrics.Meter:
|
||||||
|
measurement[Name] = name
|
||||||
|
measurement[Value] = float64(m.Count())
|
||||||
|
snapshot.Counters = append(snapshot.Counters, measurement)
|
||||||
|
snapshot.Gauges = append(snapshot.Gauges,
|
||||||
|
Measurement{
|
||||||
|
Name: fmt.Sprintf("%s.%s", name, "1min"),
|
||||||
|
Value: m.Rate1(),
|
||||||
|
Period: int64(self.Interval.Seconds()),
|
||||||
|
Attributes: map[string]interface{}{
|
||||||
|
DisplayUnitsLong: Operations,
|
||||||
|
DisplayUnitsShort: OperationsShort,
|
||||||
|
DisplayMin: "0",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Measurement{
|
||||||
|
Name: fmt.Sprintf("%s.%s", name, "5min"),
|
||||||
|
Value: m.Rate5(),
|
||||||
|
Period: int64(self.Interval.Seconds()),
|
||||||
|
Attributes: map[string]interface{}{
|
||||||
|
DisplayUnitsLong: Operations,
|
||||||
|
DisplayUnitsShort: OperationsShort,
|
||||||
|
DisplayMin: "0",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Measurement{
|
||||||
|
Name: fmt.Sprintf("%s.%s", name, "15min"),
|
||||||
|
Value: m.Rate15(),
|
||||||
|
Period: int64(self.Interval.Seconds()),
|
||||||
|
Attributes: map[string]interface{}{
|
||||||
|
DisplayUnitsLong: Operations,
|
||||||
|
DisplayUnitsShort: OperationsShort,
|
||||||
|
DisplayMin: "0",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
case metrics.Timer:
|
||||||
|
measurement[Name] = name
|
||||||
|
measurement[Value] = float64(m.Count())
|
||||||
|
snapshot.Counters = append(snapshot.Counters, measurement)
|
||||||
|
if m.Count() > 0 {
|
||||||
|
libratoName := fmt.Sprintf("%s.%s", name, "timer.mean")
|
||||||
|
gauges := make([]Measurement, histogramGaugeCount, histogramGaugeCount)
|
||||||
|
gauges[0] = Measurement{
|
||||||
|
Name: libratoName,
|
||||||
|
Count: uint64(m.Count()),
|
||||||
|
Sum: m.Mean() * float64(m.Count()),
|
||||||
|
Max: float64(m.Max()),
|
||||||
|
Min: float64(m.Min()),
|
||||||
|
SumSquares: sumSquaresTimer(m),
|
||||||
|
Period: int64(self.Interval.Seconds()),
|
||||||
|
Attributes: self.TimerAttributes,
|
||||||
|
}
|
||||||
|
for i, p := range self.Percentiles {
|
||||||
|
gauges[i+1] = Measurement{
|
||||||
|
Name: fmt.Sprintf("%s.timer.%2.0f", name, p*100),
|
||||||
|
Value: m.Percentile(p),
|
||||||
|
Period: int64(self.Interval.Seconds()),
|
||||||
|
Attributes: self.TimerAttributes,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
snapshot.Gauges = append(snapshot.Gauges, gauges...)
|
||||||
|
snapshot.Gauges = append(snapshot.Gauges,
|
||||||
|
Measurement{
|
||||||
|
Name: fmt.Sprintf("%s.%s", name, "rate.1min"),
|
||||||
|
Value: m.Rate1(),
|
||||||
|
Period: int64(self.Interval.Seconds()),
|
||||||
|
Attributes: map[string]interface{}{
|
||||||
|
DisplayUnitsLong: Operations,
|
||||||
|
DisplayUnitsShort: OperationsShort,
|
||||||
|
DisplayMin: "0",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Measurement{
|
||||||
|
Name: fmt.Sprintf("%s.%s", name, "rate.5min"),
|
||||||
|
Value: m.Rate5(),
|
||||||
|
Period: int64(self.Interval.Seconds()),
|
||||||
|
Attributes: map[string]interface{}{
|
||||||
|
DisplayUnitsLong: Operations,
|
||||||
|
DisplayUnitsShort: OperationsShort,
|
||||||
|
DisplayMin: "0",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Measurement{
|
||||||
|
Name: fmt.Sprintf("%s.%s", name, "rate.15min"),
|
||||||
|
Value: m.Rate15(),
|
||||||
|
Period: int64(self.Interval.Seconds()),
|
||||||
|
Attributes: map[string]interface{}{
|
||||||
|
DisplayUnitsLong: Operations,
|
||||||
|
DisplayUnitsShort: OperationsShort,
|
||||||
|
DisplayMin: "0",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return
|
||||||
|
}
|
||||||
+70
@@ -0,0 +1,70 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"log"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Output each metric in the given registry periodically using the given
|
||||||
|
// logger.
|
||||||
|
func Log(r Registry, d time.Duration, l *log.Logger) {
|
||||||
|
for _ = range time.Tick(d) {
|
||||||
|
r.Each(func(name string, i interface{}) {
|
||||||
|
switch metric := i.(type) {
|
||||||
|
case Counter:
|
||||||
|
l.Printf("counter %s\n", name)
|
||||||
|
l.Printf(" count: %9d\n", metric.Count())
|
||||||
|
case Gauge:
|
||||||
|
l.Printf("gauge %s\n", name)
|
||||||
|
l.Printf(" value: %9d\n", metric.Value())
|
||||||
|
case GaugeFloat64:
|
||||||
|
l.Printf("gauge %s\n", name)
|
||||||
|
l.Printf(" value: %f\n", metric.Value())
|
||||||
|
case Healthcheck:
|
||||||
|
metric.Check()
|
||||||
|
l.Printf("healthcheck %s\n", name)
|
||||||
|
l.Printf(" error: %v\n", metric.Error())
|
||||||
|
case Histogram:
|
||||||
|
h := metric.Snapshot()
|
||||||
|
ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
|
||||||
|
l.Printf("histogram %s\n", name)
|
||||||
|
l.Printf(" count: %9d\n", h.Count())
|
||||||
|
l.Printf(" min: %9d\n", h.Min())
|
||||||
|
l.Printf(" max: %9d\n", h.Max())
|
||||||
|
l.Printf(" mean: %12.2f\n", h.Mean())
|
||||||
|
l.Printf(" stddev: %12.2f\n", h.StdDev())
|
||||||
|
l.Printf(" median: %12.2f\n", ps[0])
|
||||||
|
l.Printf(" 75%%: %12.2f\n", ps[1])
|
||||||
|
l.Printf(" 95%%: %12.2f\n", ps[2])
|
||||||
|
l.Printf(" 99%%: %12.2f\n", ps[3])
|
||||||
|
l.Printf(" 99.9%%: %12.2f\n", ps[4])
|
||||||
|
case Meter:
|
||||||
|
m := metric.Snapshot()
|
||||||
|
l.Printf("meter %s\n", name)
|
||||||
|
l.Printf(" count: %9d\n", m.Count())
|
||||||
|
l.Printf(" 1-min rate: %12.2f\n", m.Rate1())
|
||||||
|
l.Printf(" 5-min rate: %12.2f\n", m.Rate5())
|
||||||
|
l.Printf(" 15-min rate: %12.2f\n", m.Rate15())
|
||||||
|
l.Printf(" mean rate: %12.2f\n", m.RateMean())
|
||||||
|
case Timer:
|
||||||
|
t := metric.Snapshot()
|
||||||
|
ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
|
||||||
|
l.Printf("timer %s\n", name)
|
||||||
|
l.Printf(" count: %9d\n", t.Count())
|
||||||
|
l.Printf(" min: %9d\n", t.Min())
|
||||||
|
l.Printf(" max: %9d\n", t.Max())
|
||||||
|
l.Printf(" mean: %12.2f\n", t.Mean())
|
||||||
|
l.Printf(" stddev: %12.2f\n", t.StdDev())
|
||||||
|
l.Printf(" median: %12.2f\n", ps[0])
|
||||||
|
l.Printf(" 75%%: %12.2f\n", ps[1])
|
||||||
|
l.Printf(" 95%%: %12.2f\n", ps[2])
|
||||||
|
l.Printf(" 99%%: %12.2f\n", ps[3])
|
||||||
|
l.Printf(" 99.9%%: %12.2f\n", ps[4])
|
||||||
|
l.Printf(" 1-min rate: %12.2f\n", t.Rate1())
|
||||||
|
l.Printf(" 5-min rate: %12.2f\n", t.Rate5())
|
||||||
|
l.Printf(" 15-min rate: %12.2f\n", t.Rate15())
|
||||||
|
l.Printf(" mean rate: %12.2f\n", t.RateMean())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
+285
@@ -0,0 +1,285 @@
|
|||||||
|
Memory usage
|
||||||
|
============
|
||||||
|
|
||||||
|
(Highly unscientific.)
|
||||||
|
|
||||||
|
Command used to gather static memory usage:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
grep ^Vm "/proc/$(ps fax | grep [m]etrics-bench | awk '{print $1}')/status"
|
||||||
|
```
|
||||||
|
|
||||||
|
Program used to gather baseline memory usage:
|
||||||
|
|
||||||
|
```go
|
||||||
|
package main
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
time.Sleep(600e9)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Baseline
|
||||||
|
--------
|
||||||
|
|
||||||
|
```
|
||||||
|
VmPeak: 42604 kB
|
||||||
|
VmSize: 42604 kB
|
||||||
|
VmLck: 0 kB
|
||||||
|
VmHWM: 1120 kB
|
||||||
|
VmRSS: 1120 kB
|
||||||
|
VmData: 35460 kB
|
||||||
|
VmStk: 136 kB
|
||||||
|
VmExe: 1020 kB
|
||||||
|
VmLib: 1848 kB
|
||||||
|
VmPTE: 36 kB
|
||||||
|
VmSwap: 0 kB
|
||||||
|
```
|
||||||
|
|
||||||
|
Program used to gather metric memory usage (with other metrics being similar):
|
||||||
|
|
||||||
|
```go
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"metrics"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
fmt.Sprintf("foo")
|
||||||
|
metrics.NewRegistry()
|
||||||
|
time.Sleep(600e9)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
1000 counters registered
|
||||||
|
------------------------
|
||||||
|
|
||||||
|
```
|
||||||
|
VmPeak: 44016 kB
|
||||||
|
VmSize: 44016 kB
|
||||||
|
VmLck: 0 kB
|
||||||
|
VmHWM: 1928 kB
|
||||||
|
VmRSS: 1928 kB
|
||||||
|
VmData: 36868 kB
|
||||||
|
VmStk: 136 kB
|
||||||
|
VmExe: 1024 kB
|
||||||
|
VmLib: 1848 kB
|
||||||
|
VmPTE: 40 kB
|
||||||
|
VmSwap: 0 kB
|
||||||
|
```
|
||||||
|
|
||||||
|
**1.412 kB virtual, TODO 0.808 kB resident per counter.**
|
||||||
|
|
||||||
|
100000 counters registered
|
||||||
|
--------------------------
|
||||||
|
|
||||||
|
```
|
||||||
|
VmPeak: 55024 kB
|
||||||
|
VmSize: 55024 kB
|
||||||
|
VmLck: 0 kB
|
||||||
|
VmHWM: 12440 kB
|
||||||
|
VmRSS: 12440 kB
|
||||||
|
VmData: 47876 kB
|
||||||
|
VmStk: 136 kB
|
||||||
|
VmExe: 1024 kB
|
||||||
|
VmLib: 1848 kB
|
||||||
|
VmPTE: 64 kB
|
||||||
|
VmSwap: 0 kB
|
||||||
|
```
|
||||||
|
|
||||||
|
**0.1242 kB virtual, 0.1132 kB resident per counter.**
|
||||||
|
|
||||||
|
1000 gauges registered
|
||||||
|
----------------------
|
||||||
|
|
||||||
|
```
|
||||||
|
VmPeak: 44012 kB
|
||||||
|
VmSize: 44012 kB
|
||||||
|
VmLck: 0 kB
|
||||||
|
VmHWM: 1928 kB
|
||||||
|
VmRSS: 1928 kB
|
||||||
|
VmData: 36868 kB
|
||||||
|
VmStk: 136 kB
|
||||||
|
VmExe: 1020 kB
|
||||||
|
VmLib: 1848 kB
|
||||||
|
VmPTE: 40 kB
|
||||||
|
VmSwap: 0 kB
|
||||||
|
```
|
||||||
|
|
||||||
|
**1.408 kB virtual, 0.808 kB resident per counter.**
|
||||||
|
|
||||||
|
100000 gauges registered
|
||||||
|
------------------------
|
||||||
|
|
||||||
|
```
|
||||||
|
VmPeak: 55020 kB
|
||||||
|
VmSize: 55020 kB
|
||||||
|
VmLck: 0 kB
|
||||||
|
VmHWM: 12432 kB
|
||||||
|
VmRSS: 12432 kB
|
||||||
|
VmData: 47876 kB
|
||||||
|
VmStk: 136 kB
|
||||||
|
VmExe: 1020 kB
|
||||||
|
VmLib: 1848 kB
|
||||||
|
VmPTE: 60 kB
|
||||||
|
VmSwap: 0 kB
|
||||||
|
```
|
||||||
|
|
||||||
|
**0.12416 kB virtual, 0.11312 resident per gauge.**
|
||||||
|
|
||||||
|
1000 histograms with a uniform sample size of 1028
|
||||||
|
--------------------------------------------------
|
||||||
|
|
||||||
|
```
|
||||||
|
VmPeak: 72272 kB
|
||||||
|
VmSize: 72272 kB
|
||||||
|
VmLck: 0 kB
|
||||||
|
VmHWM: 16204 kB
|
||||||
|
VmRSS: 16204 kB
|
||||||
|
VmData: 65100 kB
|
||||||
|
VmStk: 136 kB
|
||||||
|
VmExe: 1048 kB
|
||||||
|
VmLib: 1848 kB
|
||||||
|
VmPTE: 80 kB
|
||||||
|
VmSwap: 0 kB
|
||||||
|
```
|
||||||
|
|
||||||
|
**29.668 kB virtual, TODO 15.084 resident per histogram.**
|
||||||
|
|
||||||
|
10000 histograms with a uniform sample size of 1028
|
||||||
|
---------------------------------------------------
|
||||||
|
|
||||||
|
```
|
||||||
|
VmPeak: 256912 kB
|
||||||
|
VmSize: 256912 kB
|
||||||
|
VmLck: 0 kB
|
||||||
|
VmHWM: 146204 kB
|
||||||
|
VmRSS: 146204 kB
|
||||||
|
VmData: 249740 kB
|
||||||
|
VmStk: 136 kB
|
||||||
|
VmExe: 1048 kB
|
||||||
|
VmLib: 1848 kB
|
||||||
|
VmPTE: 448 kB
|
||||||
|
VmSwap: 0 kB
|
||||||
|
```
|
||||||
|
|
||||||
|
**21.4308 kB virtual, 14.5084 kB resident per histogram.**
|
||||||
|
|
||||||
|
50000 histograms with a uniform sample size of 1028
|
||||||
|
---------------------------------------------------
|
||||||
|
|
||||||
|
```
|
||||||
|
VmPeak: 908112 kB
|
||||||
|
VmSize: 908112 kB
|
||||||
|
VmLck: 0 kB
|
||||||
|
VmHWM: 645832 kB
|
||||||
|
VmRSS: 645588 kB
|
||||||
|
VmData: 900940 kB
|
||||||
|
VmStk: 136 kB
|
||||||
|
VmExe: 1048 kB
|
||||||
|
VmLib: 1848 kB
|
||||||
|
VmPTE: 1716 kB
|
||||||
|
VmSwap: 1544 kB
|
||||||
|
```
|
||||||
|
|
||||||
|
**17.31016 kB virtual, 12.88936 kB resident per histogram.**
|
||||||
|
|
||||||
|
1000 histograms with an exponentially-decaying sample size of 1028 and alpha of 0.015
|
||||||
|
-------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
```
|
||||||
|
VmPeak: 62480 kB
|
||||||
|
VmSize: 62480 kB
|
||||||
|
VmLck: 0 kB
|
||||||
|
VmHWM: 11572 kB
|
||||||
|
VmRSS: 11572 kB
|
||||||
|
VmData: 55308 kB
|
||||||
|
VmStk: 136 kB
|
||||||
|
VmExe: 1048 kB
|
||||||
|
VmLib: 1848 kB
|
||||||
|
VmPTE: 64 kB
|
||||||
|
VmSwap: 0 kB
|
||||||
|
```
|
||||||
|
|
||||||
|
**19.876 kB virtual, 10.452 kB resident per histogram.**
|
||||||
|
|
||||||
|
10000 histograms with an exponentially-decaying sample size of 1028 and alpha of 0.015
|
||||||
|
--------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
```
|
||||||
|
VmPeak: 153296 kB
|
||||||
|
VmSize: 153296 kB
|
||||||
|
VmLck: 0 kB
|
||||||
|
VmHWM: 101176 kB
|
||||||
|
VmRSS: 101176 kB
|
||||||
|
VmData: 146124 kB
|
||||||
|
VmStk: 136 kB
|
||||||
|
VmExe: 1048 kB
|
||||||
|
VmLib: 1848 kB
|
||||||
|
VmPTE: 240 kB
|
||||||
|
VmSwap: 0 kB
|
||||||
|
```
|
||||||
|
|
||||||
|
**11.0692 kB virtual, 10.0056 kB resident per histogram.**
|
||||||
|
|
||||||
|
50000 histograms with an exponentially-decaying sample size of 1028 and alpha of 0.015
|
||||||
|
--------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
```
|
||||||
|
VmPeak: 557264 kB
|
||||||
|
VmSize: 557264 kB
|
||||||
|
VmLck: 0 kB
|
||||||
|
VmHWM: 501056 kB
|
||||||
|
VmRSS: 501056 kB
|
||||||
|
VmData: 550092 kB
|
||||||
|
VmStk: 136 kB
|
||||||
|
VmExe: 1048 kB
|
||||||
|
VmLib: 1848 kB
|
||||||
|
VmPTE: 1032 kB
|
||||||
|
VmSwap: 0 kB
|
||||||
|
```
|
||||||
|
|
||||||
|
**10.2932 kB virtual, 9.99872 kB resident per histogram.**
|
||||||
|
|
||||||
|
1000 meters
|
||||||
|
-----------
|
||||||
|
|
||||||
|
```
|
||||||
|
VmPeak: 74504 kB
|
||||||
|
VmSize: 74504 kB
|
||||||
|
VmLck: 0 kB
|
||||||
|
VmHWM: 24124 kB
|
||||||
|
VmRSS: 24124 kB
|
||||||
|
VmData: 67340 kB
|
||||||
|
VmStk: 136 kB
|
||||||
|
VmExe: 1040 kB
|
||||||
|
VmLib: 1848 kB
|
||||||
|
VmPTE: 92 kB
|
||||||
|
VmSwap: 0 kB
|
||||||
|
```
|
||||||
|
|
||||||
|
**31.9 kB virtual, 23.004 kB resident per meter.**
|
||||||
|
|
||||||
|
10000 meters
|
||||||
|
------------
|
||||||
|
|
||||||
|
```
|
||||||
|
VmPeak: 278920 kB
|
||||||
|
VmSize: 278920 kB
|
||||||
|
VmLck: 0 kB
|
||||||
|
VmHWM: 227300 kB
|
||||||
|
VmRSS: 227300 kB
|
||||||
|
VmData: 271756 kB
|
||||||
|
VmStk: 136 kB
|
||||||
|
VmExe: 1040 kB
|
||||||
|
VmLib: 1848 kB
|
||||||
|
VmPTE: 488 kB
|
||||||
|
VmSwap: 0 kB
|
||||||
|
```
|
||||||
|
|
||||||
|
**23.6316 kB virtual, 22.618 kB resident per meter.**
|
||||||
+233
@@ -0,0 +1,233 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Meters count events to produce exponentially-weighted moving average rates
|
||||||
|
// at one-, five-, and fifteen-minutes and a mean rate.
|
||||||
|
type Meter interface {
|
||||||
|
Count() int64
|
||||||
|
Mark(int64)
|
||||||
|
Rate1() float64
|
||||||
|
Rate5() float64
|
||||||
|
Rate15() float64
|
||||||
|
RateMean() float64
|
||||||
|
Snapshot() Meter
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetOrRegisterMeter returns an existing Meter or constructs and registers a
|
||||||
|
// new StandardMeter.
|
||||||
|
func GetOrRegisterMeter(name string, r Registry) Meter {
|
||||||
|
if nil == r {
|
||||||
|
r = DefaultRegistry
|
||||||
|
}
|
||||||
|
return r.GetOrRegister(name, NewMeter).(Meter)
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewMeter constructs a new StandardMeter and launches a goroutine.
|
||||||
|
func NewMeter() Meter {
|
||||||
|
if UseNilMetrics {
|
||||||
|
return NilMeter{}
|
||||||
|
}
|
||||||
|
m := newStandardMeter()
|
||||||
|
arbiter.Lock()
|
||||||
|
defer arbiter.Unlock()
|
||||||
|
arbiter.meters = append(arbiter.meters, m)
|
||||||
|
if !arbiter.started {
|
||||||
|
arbiter.started = true
|
||||||
|
go arbiter.tick()
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewMeter constructs and registers a new StandardMeter and launches a
|
||||||
|
// goroutine.
|
||||||
|
func NewRegisteredMeter(name string, r Registry) Meter {
|
||||||
|
c := NewMeter()
|
||||||
|
if nil == r {
|
||||||
|
r = DefaultRegistry
|
||||||
|
}
|
||||||
|
r.Register(name, c)
|
||||||
|
return c
|
||||||
|
}
|
||||||
|
|
||||||
|
// MeterSnapshot is a read-only copy of another Meter.
|
||||||
|
type MeterSnapshot struct {
|
||||||
|
count int64
|
||||||
|
rate1, rate5, rate15, rateMean float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count returns the count of events at the time the snapshot was taken.
|
||||||
|
func (m *MeterSnapshot) Count() int64 { return m.count }
|
||||||
|
|
||||||
|
// Mark panics.
|
||||||
|
func (*MeterSnapshot) Mark(n int64) {
|
||||||
|
panic("Mark called on a MeterSnapshot")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rate1 returns the one-minute moving average rate of events per second at the
|
||||||
|
// time the snapshot was taken.
|
||||||
|
func (m *MeterSnapshot) Rate1() float64 { return m.rate1 }
|
||||||
|
|
||||||
|
// Rate5 returns the five-minute moving average rate of events per second at
|
||||||
|
// the time the snapshot was taken.
|
||||||
|
func (m *MeterSnapshot) Rate5() float64 { return m.rate5 }
|
||||||
|
|
||||||
|
// Rate15 returns the fifteen-minute moving average rate of events per second
|
||||||
|
// at the time the snapshot was taken.
|
||||||
|
func (m *MeterSnapshot) Rate15() float64 { return m.rate15 }
|
||||||
|
|
||||||
|
// RateMean returns the meter's mean rate of events per second at the time the
|
||||||
|
// snapshot was taken.
|
||||||
|
func (m *MeterSnapshot) RateMean() float64 { return m.rateMean }
|
||||||
|
|
||||||
|
// Snapshot returns the snapshot.
|
||||||
|
func (m *MeterSnapshot) Snapshot() Meter { return m }
|
||||||
|
|
||||||
|
// NilMeter is a no-op Meter.
|
||||||
|
type NilMeter struct{}
|
||||||
|
|
||||||
|
// Count is a no-op.
|
||||||
|
func (NilMeter) Count() int64 { return 0 }
|
||||||
|
|
||||||
|
// Mark is a no-op.
|
||||||
|
func (NilMeter) Mark(n int64) {}
|
||||||
|
|
||||||
|
// Rate1 is a no-op.
|
||||||
|
func (NilMeter) Rate1() float64 { return 0.0 }
|
||||||
|
|
||||||
|
// Rate5 is a no-op.
|
||||||
|
func (NilMeter) Rate5() float64 { return 0.0 }
|
||||||
|
|
||||||
|
// Rate15is a no-op.
|
||||||
|
func (NilMeter) Rate15() float64 { return 0.0 }
|
||||||
|
|
||||||
|
// RateMean is a no-op.
|
||||||
|
func (NilMeter) RateMean() float64 { return 0.0 }
|
||||||
|
|
||||||
|
// Snapshot is a no-op.
|
||||||
|
func (NilMeter) Snapshot() Meter { return NilMeter{} }
|
||||||
|
|
||||||
|
// StandardMeter is the standard implementation of a Meter.
|
||||||
|
type StandardMeter struct {
|
||||||
|
lock sync.RWMutex
|
||||||
|
snapshot *MeterSnapshot
|
||||||
|
a1, a5, a15 EWMA
|
||||||
|
startTime time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func newStandardMeter() *StandardMeter {
|
||||||
|
return &StandardMeter{
|
||||||
|
snapshot: &MeterSnapshot{},
|
||||||
|
a1: NewEWMA1(),
|
||||||
|
a5: NewEWMA5(),
|
||||||
|
a15: NewEWMA15(),
|
||||||
|
startTime: time.Now(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count returns the number of events recorded.
|
||||||
|
func (m *StandardMeter) Count() int64 {
|
||||||
|
m.lock.RLock()
|
||||||
|
count := m.snapshot.count
|
||||||
|
m.lock.RUnlock()
|
||||||
|
return count
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mark records the occurance of n events.
|
||||||
|
func (m *StandardMeter) Mark(n int64) {
|
||||||
|
m.lock.Lock()
|
||||||
|
defer m.lock.Unlock()
|
||||||
|
m.snapshot.count += n
|
||||||
|
m.a1.Update(n)
|
||||||
|
m.a5.Update(n)
|
||||||
|
m.a15.Update(n)
|
||||||
|
m.updateSnapshot()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rate1 returns the one-minute moving average rate of events per second.
|
||||||
|
func (m *StandardMeter) Rate1() float64 {
|
||||||
|
m.lock.RLock()
|
||||||
|
rate1 := m.snapshot.rate1
|
||||||
|
m.lock.RUnlock()
|
||||||
|
return rate1
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rate5 returns the five-minute moving average rate of events per second.
|
||||||
|
func (m *StandardMeter) Rate5() float64 {
|
||||||
|
m.lock.RLock()
|
||||||
|
rate5 := m.snapshot.rate5
|
||||||
|
m.lock.RUnlock()
|
||||||
|
return rate5
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rate15 returns the fifteen-minute moving average rate of events per second.
|
||||||
|
func (m *StandardMeter) Rate15() float64 {
|
||||||
|
m.lock.RLock()
|
||||||
|
rate15 := m.snapshot.rate15
|
||||||
|
m.lock.RUnlock()
|
||||||
|
return rate15
|
||||||
|
}
|
||||||
|
|
||||||
|
// RateMean returns the meter's mean rate of events per second.
|
||||||
|
func (m *StandardMeter) RateMean() float64 {
|
||||||
|
m.lock.RLock()
|
||||||
|
rateMean := m.snapshot.rateMean
|
||||||
|
m.lock.RUnlock()
|
||||||
|
return rateMean
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot returns a read-only copy of the meter.
|
||||||
|
func (m *StandardMeter) Snapshot() Meter {
|
||||||
|
m.lock.RLock()
|
||||||
|
snapshot := *m.snapshot
|
||||||
|
m.lock.RUnlock()
|
||||||
|
return &snapshot
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *StandardMeter) updateSnapshot() {
|
||||||
|
// should run with write lock held on m.lock
|
||||||
|
snapshot := m.snapshot
|
||||||
|
snapshot.rate1 = m.a1.Rate()
|
||||||
|
snapshot.rate5 = m.a5.Rate()
|
||||||
|
snapshot.rate15 = m.a15.Rate()
|
||||||
|
snapshot.rateMean = float64(snapshot.count) / time.Since(m.startTime).Seconds()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *StandardMeter) tick() {
|
||||||
|
m.lock.Lock()
|
||||||
|
defer m.lock.Unlock()
|
||||||
|
m.a1.Tick()
|
||||||
|
m.a5.Tick()
|
||||||
|
m.a15.Tick()
|
||||||
|
m.updateSnapshot()
|
||||||
|
}
|
||||||
|
|
||||||
|
type meterArbiter struct {
|
||||||
|
sync.RWMutex
|
||||||
|
started bool
|
||||||
|
meters []*StandardMeter
|
||||||
|
ticker *time.Ticker
|
||||||
|
}
|
||||||
|
|
||||||
|
var arbiter = meterArbiter{ticker: time.NewTicker(5e9)}
|
||||||
|
|
||||||
|
// Ticks meters on the scheduled interval
|
||||||
|
func (ma *meterArbiter) tick() {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ma.ticker.C:
|
||||||
|
ma.tickMeters()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (ma *meterArbiter) tickMeters() {
|
||||||
|
ma.RLock()
|
||||||
|
defer ma.RUnlock()
|
||||||
|
for _, meter := range ma.meters {
|
||||||
|
meter.tick()
|
||||||
|
}
|
||||||
|
}
|
||||||
+13
@@ -0,0 +1,13 @@
|
|||||||
|
// Go port of Coda Hale's Metrics library
|
||||||
|
//
|
||||||
|
// <https://github.com/rcrowley/go-metrics>
|
||||||
|
//
|
||||||
|
// Coda Hale's original work: <https://github.com/codahale/metrics>
|
||||||
|
package metrics
|
||||||
|
|
||||||
|
// UseNilMetrics is checked by the constructor functions for all of the
|
||||||
|
// standard metrics. If it is true, the metric returned is a stub.
|
||||||
|
//
|
||||||
|
// This global kill-switch helps quantify the observer effect and makes
|
||||||
|
// for less cluttered pprof profiles.
|
||||||
|
var UseNilMetrics bool = false
|
||||||
+119
@@ -0,0 +1,119 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
var shortHostName string = ""
|
||||||
|
|
||||||
|
// OpenTSDBConfig provides a container with configuration parameters for
|
||||||
|
// the OpenTSDB exporter
|
||||||
|
type OpenTSDBConfig struct {
|
||||||
|
Addr *net.TCPAddr // Network address to connect to
|
||||||
|
Registry Registry // Registry to be exported
|
||||||
|
FlushInterval time.Duration // Flush interval
|
||||||
|
DurationUnit time.Duration // Time conversion unit for durations
|
||||||
|
Prefix string // Prefix to be prepended to metric names
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenTSDB is a blocking exporter function which reports metrics in r
|
||||||
|
// to a TSDB server located at addr, flushing them every d duration
|
||||||
|
// and prepending metric names with prefix.
|
||||||
|
func OpenTSDB(r Registry, d time.Duration, prefix string, addr *net.TCPAddr) {
|
||||||
|
OpenTSDBWithConfig(OpenTSDBConfig{
|
||||||
|
Addr: addr,
|
||||||
|
Registry: r,
|
||||||
|
FlushInterval: d,
|
||||||
|
DurationUnit: time.Nanosecond,
|
||||||
|
Prefix: prefix,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// OpenTSDBWithConfig is a blocking exporter function just like OpenTSDB,
|
||||||
|
// but it takes a OpenTSDBConfig instead.
|
||||||
|
func OpenTSDBWithConfig(c OpenTSDBConfig) {
|
||||||
|
for _ = range time.Tick(c.FlushInterval) {
|
||||||
|
if err := openTSDB(&c); nil != err {
|
||||||
|
log.Println(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func getShortHostname() string {
|
||||||
|
if shortHostName == "" {
|
||||||
|
host, _ := os.Hostname()
|
||||||
|
if index := strings.Index(host, "."); index > 0 {
|
||||||
|
shortHostName = host[:index]
|
||||||
|
} else {
|
||||||
|
shortHostName = host
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return shortHostName
|
||||||
|
}
|
||||||
|
|
||||||
|
func openTSDB(c *OpenTSDBConfig) error {
|
||||||
|
shortHostname := getShortHostname()
|
||||||
|
now := time.Now().Unix()
|
||||||
|
du := float64(c.DurationUnit)
|
||||||
|
conn, err := net.DialTCP("tcp", nil, c.Addr)
|
||||||
|
if nil != err {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
w := bufio.NewWriter(conn)
|
||||||
|
c.Registry.Each(func(name string, i interface{}) {
|
||||||
|
switch metric := i.(type) {
|
||||||
|
case Counter:
|
||||||
|
fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, metric.Count(), shortHostname)
|
||||||
|
case Gauge:
|
||||||
|
fmt.Fprintf(w, "put %s.%s.value %d %d host=%s\n", c.Prefix, name, now, metric.Value(), shortHostname)
|
||||||
|
case GaugeFloat64:
|
||||||
|
fmt.Fprintf(w, "put %s.%s.value %d %f host=%s\n", c.Prefix, name, now, metric.Value(), shortHostname)
|
||||||
|
case Histogram:
|
||||||
|
h := metric.Snapshot()
|
||||||
|
ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
|
||||||
|
fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, h.Count(), shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.min %d %d host=%s\n", c.Prefix, name, now, h.Min(), shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.max %d %d host=%s\n", c.Prefix, name, now, h.Max(), shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.mean %d %.2f host=%s\n", c.Prefix, name, now, h.Mean(), shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.std-dev %d %.2f host=%s\n", c.Prefix, name, now, h.StdDev(), shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.50-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[0], shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.75-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[1], shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.95-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[2], shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.99-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[3], shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.999-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[4], shortHostname)
|
||||||
|
case Meter:
|
||||||
|
m := metric.Snapshot()
|
||||||
|
fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, m.Count(), shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.one-minute %d %.2f host=%s\n", c.Prefix, name, now, m.Rate1(), shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.five-minute %d %.2f host=%s\n", c.Prefix, name, now, m.Rate5(), shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.fifteen-minute %d %.2f host=%s\n", c.Prefix, name, now, m.Rate15(), shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.mean %d %.2f host=%s\n", c.Prefix, name, now, m.RateMean(), shortHostname)
|
||||||
|
case Timer:
|
||||||
|
t := metric.Snapshot()
|
||||||
|
ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
|
||||||
|
fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, t.Count(), shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.min %d %d host=%s\n", c.Prefix, name, now, t.Min()/int64(du), shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.max %d %d host=%s\n", c.Prefix, name, now, t.Max()/int64(du), shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.mean %d %.2f host=%s\n", c.Prefix, name, now, t.Mean()/du, shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.std-dev %d %.2f host=%s\n", c.Prefix, name, now, t.StdDev()/du, shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.50-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[0]/du, shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.75-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[1]/du, shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.95-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[2]/du, shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.99-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[3]/du, shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.999-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[4]/du, shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.one-minute %d %.2f host=%s\n", c.Prefix, name, now, t.Rate1(), shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.five-minute %d %.2f host=%s\n", c.Prefix, name, now, t.Rate5(), shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.fifteen-minute %d %.2f host=%s\n", c.Prefix, name, now, t.Rate15(), shortHostname)
|
||||||
|
fmt.Fprintf(w, "put %s.%s.mean-rate %d %.2f host=%s\n", c.Prefix, name, now, t.RateMean(), shortHostname)
|
||||||
|
}
|
||||||
|
w.Flush()
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
}
|
||||||
+240
@@ -0,0 +1,240 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"sync"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DuplicateMetric is the error returned by Registry.Register when a metric
|
||||||
|
// already exists. If you mean to Register that metric you must first
|
||||||
|
// Unregister the existing metric.
|
||||||
|
type DuplicateMetric string
|
||||||
|
|
||||||
|
func (err DuplicateMetric) Error() string {
|
||||||
|
return fmt.Sprintf("duplicate metric: %s", string(err))
|
||||||
|
}
|
||||||
|
|
||||||
|
// A Registry holds references to a set of metrics by name and can iterate
|
||||||
|
// over them, calling callback functions provided by the user.
|
||||||
|
//
|
||||||
|
// This is an interface so as to encourage other structs to implement
|
||||||
|
// the Registry API as appropriate.
|
||||||
|
type Registry interface {
|
||||||
|
|
||||||
|
// Call the given function for each registered metric.
|
||||||
|
Each(func(string, interface{}))
|
||||||
|
|
||||||
|
// Get the metric by the given name or nil if none is registered.
|
||||||
|
Get(string) interface{}
|
||||||
|
|
||||||
|
// Gets an existing metric or registers the given one.
|
||||||
|
// The interface can be the metric to register if not found in registry,
|
||||||
|
// or a function returning the metric for lazy instantiation.
|
||||||
|
GetOrRegister(string, interface{}) interface{}
|
||||||
|
|
||||||
|
// Register the given metric under the given name.
|
||||||
|
Register(string, interface{}) error
|
||||||
|
|
||||||
|
// Run all registered healthchecks.
|
||||||
|
RunHealthchecks()
|
||||||
|
|
||||||
|
// Unregister the metric with the given name.
|
||||||
|
Unregister(string)
|
||||||
|
|
||||||
|
// Unregister all metrics. (Mostly for testing.)
|
||||||
|
UnregisterAll()
|
||||||
|
}
|
||||||
|
|
||||||
|
// The standard implementation of a Registry is a mutex-protected map
|
||||||
|
// of names to metrics.
|
||||||
|
type StandardRegistry struct {
|
||||||
|
metrics map[string]interface{}
|
||||||
|
mutex sync.Mutex
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a new registry.
|
||||||
|
func NewRegistry() Registry {
|
||||||
|
return &StandardRegistry{metrics: make(map[string]interface{})}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Call the given function for each registered metric.
|
||||||
|
func (r *StandardRegistry) Each(f func(string, interface{})) {
|
||||||
|
for name, i := range r.registered() {
|
||||||
|
f(name, i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the metric by the given name or nil if none is registered.
|
||||||
|
func (r *StandardRegistry) Get(name string) interface{} {
|
||||||
|
r.mutex.Lock()
|
||||||
|
defer r.mutex.Unlock()
|
||||||
|
return r.metrics[name]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Gets an existing metric or creates and registers a new one. Threadsafe
|
||||||
|
// alternative to calling Get and Register on failure.
|
||||||
|
// The interface can be the metric to register if not found in registry,
|
||||||
|
// or a function returning the metric for lazy instantiation.
|
||||||
|
func (r *StandardRegistry) GetOrRegister(name string, i interface{}) interface{} {
|
||||||
|
r.mutex.Lock()
|
||||||
|
defer r.mutex.Unlock()
|
||||||
|
if metric, ok := r.metrics[name]; ok {
|
||||||
|
return metric
|
||||||
|
}
|
||||||
|
if v := reflect.ValueOf(i); v.Kind() == reflect.Func {
|
||||||
|
i = v.Call(nil)[0].Interface()
|
||||||
|
}
|
||||||
|
r.register(name, i)
|
||||||
|
return i
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register the given metric under the given name. Returns a DuplicateMetric
|
||||||
|
// if a metric by the given name is already registered.
|
||||||
|
func (r *StandardRegistry) Register(name string, i interface{}) error {
|
||||||
|
r.mutex.Lock()
|
||||||
|
defer r.mutex.Unlock()
|
||||||
|
return r.register(name, i)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run all registered healthchecks.
|
||||||
|
func (r *StandardRegistry) RunHealthchecks() {
|
||||||
|
r.mutex.Lock()
|
||||||
|
defer r.mutex.Unlock()
|
||||||
|
for _, i := range r.metrics {
|
||||||
|
if h, ok := i.(Healthcheck); ok {
|
||||||
|
h.Check()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unregister the metric with the given name.
|
||||||
|
func (r *StandardRegistry) Unregister(name string) {
|
||||||
|
r.mutex.Lock()
|
||||||
|
defer r.mutex.Unlock()
|
||||||
|
delete(r.metrics, name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unregister all metrics. (Mostly for testing.)
|
||||||
|
func (r *StandardRegistry) UnregisterAll() {
|
||||||
|
r.mutex.Lock()
|
||||||
|
defer r.mutex.Unlock()
|
||||||
|
for name, _ := range r.metrics {
|
||||||
|
delete(r.metrics, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *StandardRegistry) register(name string, i interface{}) error {
|
||||||
|
if _, ok := r.metrics[name]; ok {
|
||||||
|
return DuplicateMetric(name)
|
||||||
|
}
|
||||||
|
switch i.(type) {
|
||||||
|
case Counter, Gauge, GaugeFloat64, Healthcheck, Histogram, Meter, Timer:
|
||||||
|
r.metrics[name] = i
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *StandardRegistry) registered() map[string]interface{} {
|
||||||
|
r.mutex.Lock()
|
||||||
|
defer r.mutex.Unlock()
|
||||||
|
metrics := make(map[string]interface{}, len(r.metrics))
|
||||||
|
for name, i := range r.metrics {
|
||||||
|
metrics[name] = i
|
||||||
|
}
|
||||||
|
return metrics
|
||||||
|
}
|
||||||
|
|
||||||
|
type PrefixedRegistry struct {
|
||||||
|
underlying Registry
|
||||||
|
prefix string
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewPrefixedRegistry(prefix string) Registry {
|
||||||
|
return &PrefixedRegistry{
|
||||||
|
underlying: NewRegistry(),
|
||||||
|
prefix: prefix,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Call the given function for each registered metric.
|
||||||
|
func (r *PrefixedRegistry) Each(fn func(string, interface{})) {
|
||||||
|
r.underlying.Each(fn)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the metric by the given name or nil if none is registered.
|
||||||
|
func (r *PrefixedRegistry) Get(name string) interface{} {
|
||||||
|
return r.underlying.Get(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Gets an existing metric or registers the given one.
|
||||||
|
// The interface can be the metric to register if not found in registry,
|
||||||
|
// or a function returning the metric for lazy instantiation.
|
||||||
|
func (r *PrefixedRegistry) GetOrRegister(name string, metric interface{}) interface{} {
|
||||||
|
realName := r.prefix + name
|
||||||
|
return r.underlying.GetOrRegister(realName, metric)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register the given metric under the given name. The name will be prefixed.
|
||||||
|
func (r *PrefixedRegistry) Register(name string, metric interface{}) error {
|
||||||
|
realName := r.prefix + name
|
||||||
|
return r.underlying.Register(realName, metric)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run all registered healthchecks.
|
||||||
|
func (r *PrefixedRegistry) RunHealthchecks() {
|
||||||
|
r.underlying.RunHealthchecks()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unregister the metric with the given name. The name will be prefixed.
|
||||||
|
func (r *PrefixedRegistry) Unregister(name string) {
|
||||||
|
realName := r.prefix + name
|
||||||
|
r.underlying.Unregister(realName)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unregister all metrics. (Mostly for testing.)
|
||||||
|
func (r *PrefixedRegistry) UnregisterAll() {
|
||||||
|
r.underlying.UnregisterAll()
|
||||||
|
}
|
||||||
|
|
||||||
|
var DefaultRegistry Registry = NewRegistry()
|
||||||
|
|
||||||
|
// Call the given function for each registered metric.
|
||||||
|
func Each(f func(string, interface{})) {
|
||||||
|
DefaultRegistry.Each(f)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get the metric by the given name or nil if none is registered.
|
||||||
|
func Get(name string) interface{} {
|
||||||
|
return DefaultRegistry.Get(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Gets an existing metric or creates and registers a new one. Threadsafe
|
||||||
|
// alternative to calling Get and Register on failure.
|
||||||
|
func GetOrRegister(name string, i interface{}) interface{} {
|
||||||
|
return DefaultRegistry.GetOrRegister(name, i)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register the given metric under the given name. Returns a DuplicateMetric
|
||||||
|
// if a metric by the given name is already registered.
|
||||||
|
func Register(name string, i interface{}) error {
|
||||||
|
return DefaultRegistry.Register(name, i)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register the given metric under the given name. Panics if a metric by the
|
||||||
|
// given name is already registered.
|
||||||
|
func MustRegister(name string, i interface{}) {
|
||||||
|
if err := Register(name, i); err != nil {
|
||||||
|
panic(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run all registered healthchecks.
|
||||||
|
func RunHealthchecks() {
|
||||||
|
DefaultRegistry.RunHealthchecks()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Unregister the metric with the given name.
|
||||||
|
func Unregister(name string) {
|
||||||
|
DefaultRegistry.Unregister(name)
|
||||||
|
}
|
||||||
+200
@@ -0,0 +1,200 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"runtime"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
memStats runtime.MemStats
|
||||||
|
runtimeMetrics struct {
|
||||||
|
MemStats struct {
|
||||||
|
Alloc Gauge
|
||||||
|
BuckHashSys Gauge
|
||||||
|
DebugGC Gauge
|
||||||
|
EnableGC Gauge
|
||||||
|
Frees Gauge
|
||||||
|
HeapAlloc Gauge
|
||||||
|
HeapIdle Gauge
|
||||||
|
HeapInuse Gauge
|
||||||
|
HeapObjects Gauge
|
||||||
|
HeapReleased Gauge
|
||||||
|
HeapSys Gauge
|
||||||
|
LastGC Gauge
|
||||||
|
Lookups Gauge
|
||||||
|
Mallocs Gauge
|
||||||
|
MCacheInuse Gauge
|
||||||
|
MCacheSys Gauge
|
||||||
|
MSpanInuse Gauge
|
||||||
|
MSpanSys Gauge
|
||||||
|
NextGC Gauge
|
||||||
|
NumGC Gauge
|
||||||
|
PauseNs Histogram
|
||||||
|
PauseTotalNs Gauge
|
||||||
|
StackInuse Gauge
|
||||||
|
StackSys Gauge
|
||||||
|
Sys Gauge
|
||||||
|
TotalAlloc Gauge
|
||||||
|
}
|
||||||
|
NumCgoCall Gauge
|
||||||
|
NumGoroutine Gauge
|
||||||
|
ReadMemStats Timer
|
||||||
|
}
|
||||||
|
frees uint64
|
||||||
|
lookups uint64
|
||||||
|
mallocs uint64
|
||||||
|
numGC uint32
|
||||||
|
numCgoCalls int64
|
||||||
|
)
|
||||||
|
|
||||||
|
// Capture new values for the Go runtime statistics exported in
|
||||||
|
// runtime.MemStats. This is designed to be called as a goroutine.
|
||||||
|
func CaptureRuntimeMemStats(r Registry, d time.Duration) {
|
||||||
|
for _ = range time.Tick(d) {
|
||||||
|
CaptureRuntimeMemStatsOnce(r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Capture new values for the Go runtime statistics exported in
|
||||||
|
// runtime.MemStats. This is designed to be called in a background
|
||||||
|
// goroutine. Giving a registry which has not been given to
|
||||||
|
// RegisterRuntimeMemStats will panic.
|
||||||
|
//
|
||||||
|
// Be very careful with this because runtime.ReadMemStats calls the C
|
||||||
|
// functions runtime·semacquire(&runtime·worldsema) and runtime·stoptheworld()
|
||||||
|
// and that last one does what it says on the tin.
|
||||||
|
func CaptureRuntimeMemStatsOnce(r Registry) {
|
||||||
|
t := time.Now()
|
||||||
|
runtime.ReadMemStats(&memStats) // This takes 50-200us.
|
||||||
|
runtimeMetrics.ReadMemStats.UpdateSince(t)
|
||||||
|
|
||||||
|
runtimeMetrics.MemStats.Alloc.Update(int64(memStats.Alloc))
|
||||||
|
runtimeMetrics.MemStats.BuckHashSys.Update(int64(memStats.BuckHashSys))
|
||||||
|
if memStats.DebugGC {
|
||||||
|
runtimeMetrics.MemStats.DebugGC.Update(1)
|
||||||
|
} else {
|
||||||
|
runtimeMetrics.MemStats.DebugGC.Update(0)
|
||||||
|
}
|
||||||
|
if memStats.EnableGC {
|
||||||
|
runtimeMetrics.MemStats.EnableGC.Update(1)
|
||||||
|
} else {
|
||||||
|
runtimeMetrics.MemStats.EnableGC.Update(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
runtimeMetrics.MemStats.Frees.Update(int64(memStats.Frees - frees))
|
||||||
|
runtimeMetrics.MemStats.HeapAlloc.Update(int64(memStats.HeapAlloc))
|
||||||
|
runtimeMetrics.MemStats.HeapIdle.Update(int64(memStats.HeapIdle))
|
||||||
|
runtimeMetrics.MemStats.HeapInuse.Update(int64(memStats.HeapInuse))
|
||||||
|
runtimeMetrics.MemStats.HeapObjects.Update(int64(memStats.HeapObjects))
|
||||||
|
runtimeMetrics.MemStats.HeapReleased.Update(int64(memStats.HeapReleased))
|
||||||
|
runtimeMetrics.MemStats.HeapSys.Update(int64(memStats.HeapSys))
|
||||||
|
runtimeMetrics.MemStats.LastGC.Update(int64(memStats.LastGC))
|
||||||
|
runtimeMetrics.MemStats.Lookups.Update(int64(memStats.Lookups - lookups))
|
||||||
|
runtimeMetrics.MemStats.Mallocs.Update(int64(memStats.Mallocs - mallocs))
|
||||||
|
runtimeMetrics.MemStats.MCacheInuse.Update(int64(memStats.MCacheInuse))
|
||||||
|
runtimeMetrics.MemStats.MCacheSys.Update(int64(memStats.MCacheSys))
|
||||||
|
runtimeMetrics.MemStats.MSpanInuse.Update(int64(memStats.MSpanInuse))
|
||||||
|
runtimeMetrics.MemStats.MSpanSys.Update(int64(memStats.MSpanSys))
|
||||||
|
runtimeMetrics.MemStats.NextGC.Update(int64(memStats.NextGC))
|
||||||
|
runtimeMetrics.MemStats.NumGC.Update(int64(memStats.NumGC - numGC))
|
||||||
|
|
||||||
|
// <https://code.google.com/p/go/source/browse/src/pkg/runtime/mgc0.c>
|
||||||
|
i := numGC % uint32(len(memStats.PauseNs))
|
||||||
|
ii := memStats.NumGC % uint32(len(memStats.PauseNs))
|
||||||
|
if memStats.NumGC-numGC >= uint32(len(memStats.PauseNs)) {
|
||||||
|
for i = 0; i < uint32(len(memStats.PauseNs)); i++ {
|
||||||
|
runtimeMetrics.MemStats.PauseNs.Update(int64(memStats.PauseNs[i]))
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if i > ii {
|
||||||
|
for ; i < uint32(len(memStats.PauseNs)); i++ {
|
||||||
|
runtimeMetrics.MemStats.PauseNs.Update(int64(memStats.PauseNs[i]))
|
||||||
|
}
|
||||||
|
i = 0
|
||||||
|
}
|
||||||
|
for ; i < ii; i++ {
|
||||||
|
runtimeMetrics.MemStats.PauseNs.Update(int64(memStats.PauseNs[i]))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
frees = memStats.Frees
|
||||||
|
lookups = memStats.Lookups
|
||||||
|
mallocs = memStats.Mallocs
|
||||||
|
numGC = memStats.NumGC
|
||||||
|
|
||||||
|
runtimeMetrics.MemStats.PauseTotalNs.Update(int64(memStats.PauseTotalNs))
|
||||||
|
runtimeMetrics.MemStats.StackInuse.Update(int64(memStats.StackInuse))
|
||||||
|
runtimeMetrics.MemStats.StackSys.Update(int64(memStats.StackSys))
|
||||||
|
runtimeMetrics.MemStats.Sys.Update(int64(memStats.Sys))
|
||||||
|
runtimeMetrics.MemStats.TotalAlloc.Update(int64(memStats.TotalAlloc))
|
||||||
|
|
||||||
|
currentNumCgoCalls := numCgoCall()
|
||||||
|
runtimeMetrics.NumCgoCall.Update(currentNumCgoCalls - numCgoCalls)
|
||||||
|
numCgoCalls = currentNumCgoCalls
|
||||||
|
|
||||||
|
runtimeMetrics.NumGoroutine.Update(int64(runtime.NumGoroutine()))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register runtimeMetrics for the Go runtime statistics exported in runtime and
|
||||||
|
// specifically runtime.MemStats. The runtimeMetrics are named by their
|
||||||
|
// fully-qualified Go symbols, i.e. runtime.MemStats.Alloc.
|
||||||
|
func RegisterRuntimeMemStats(r Registry) {
|
||||||
|
runtimeMetrics.MemStats.Alloc = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.BuckHashSys = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.DebugGC = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.EnableGC = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.Frees = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.HeapAlloc = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.HeapIdle = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.HeapInuse = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.HeapObjects = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.HeapReleased = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.HeapSys = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.LastGC = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.Lookups = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.Mallocs = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.MCacheInuse = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.MCacheSys = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.MSpanInuse = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.MSpanSys = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.NextGC = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.NumGC = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.PauseNs = NewHistogram(NewExpDecaySample(1028, 0.015))
|
||||||
|
runtimeMetrics.MemStats.PauseTotalNs = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.StackInuse = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.StackSys = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.Sys = NewGauge()
|
||||||
|
runtimeMetrics.MemStats.TotalAlloc = NewGauge()
|
||||||
|
runtimeMetrics.NumCgoCall = NewGauge()
|
||||||
|
runtimeMetrics.NumGoroutine = NewGauge()
|
||||||
|
runtimeMetrics.ReadMemStats = NewTimer()
|
||||||
|
|
||||||
|
r.Register("runtime.MemStats.Alloc", runtimeMetrics.MemStats.Alloc)
|
||||||
|
r.Register("runtime.MemStats.BuckHashSys", runtimeMetrics.MemStats.BuckHashSys)
|
||||||
|
r.Register("runtime.MemStats.DebugGC", runtimeMetrics.MemStats.DebugGC)
|
||||||
|
r.Register("runtime.MemStats.EnableGC", runtimeMetrics.MemStats.EnableGC)
|
||||||
|
r.Register("runtime.MemStats.Frees", runtimeMetrics.MemStats.Frees)
|
||||||
|
r.Register("runtime.MemStats.HeapAlloc", runtimeMetrics.MemStats.HeapAlloc)
|
||||||
|
r.Register("runtime.MemStats.HeapIdle", runtimeMetrics.MemStats.HeapIdle)
|
||||||
|
r.Register("runtime.MemStats.HeapInuse", runtimeMetrics.MemStats.HeapInuse)
|
||||||
|
r.Register("runtime.MemStats.HeapObjects", runtimeMetrics.MemStats.HeapObjects)
|
||||||
|
r.Register("runtime.MemStats.HeapReleased", runtimeMetrics.MemStats.HeapReleased)
|
||||||
|
r.Register("runtime.MemStats.HeapSys", runtimeMetrics.MemStats.HeapSys)
|
||||||
|
r.Register("runtime.MemStats.LastGC", runtimeMetrics.MemStats.LastGC)
|
||||||
|
r.Register("runtime.MemStats.Lookups", runtimeMetrics.MemStats.Lookups)
|
||||||
|
r.Register("runtime.MemStats.Mallocs", runtimeMetrics.MemStats.Mallocs)
|
||||||
|
r.Register("runtime.MemStats.MCacheInuse", runtimeMetrics.MemStats.MCacheInuse)
|
||||||
|
r.Register("runtime.MemStats.MCacheSys", runtimeMetrics.MemStats.MCacheSys)
|
||||||
|
r.Register("runtime.MemStats.MSpanInuse", runtimeMetrics.MemStats.MSpanInuse)
|
||||||
|
r.Register("runtime.MemStats.MSpanSys", runtimeMetrics.MemStats.MSpanSys)
|
||||||
|
r.Register("runtime.MemStats.NextGC", runtimeMetrics.MemStats.NextGC)
|
||||||
|
r.Register("runtime.MemStats.NumGC", runtimeMetrics.MemStats.NumGC)
|
||||||
|
r.Register("runtime.MemStats.PauseNs", runtimeMetrics.MemStats.PauseNs)
|
||||||
|
r.Register("runtime.MemStats.PauseTotalNs", runtimeMetrics.MemStats.PauseTotalNs)
|
||||||
|
r.Register("runtime.MemStats.StackInuse", runtimeMetrics.MemStats.StackInuse)
|
||||||
|
r.Register("runtime.MemStats.StackSys", runtimeMetrics.MemStats.StackSys)
|
||||||
|
r.Register("runtime.MemStats.Sys", runtimeMetrics.MemStats.Sys)
|
||||||
|
r.Register("runtime.MemStats.TotalAlloc", runtimeMetrics.MemStats.TotalAlloc)
|
||||||
|
r.Register("runtime.NumCgoCall", runtimeMetrics.NumCgoCall)
|
||||||
|
r.Register("runtime.NumGoroutine", runtimeMetrics.NumGoroutine)
|
||||||
|
r.Register("runtime.ReadMemStats", runtimeMetrics.ReadMemStats)
|
||||||
|
}
|
||||||
+10
@@ -0,0 +1,10 @@
|
|||||||
|
// +build cgo
|
||||||
|
// +build !appengine
|
||||||
|
|
||||||
|
package metrics
|
||||||
|
|
||||||
|
import "runtime"
|
||||||
|
|
||||||
|
func numCgoCall() int64 {
|
||||||
|
return runtime.NumCgoCall()
|
||||||
|
}
|
||||||
+7
@@ -0,0 +1,7 @@
|
|||||||
|
// +build !cgo appengine
|
||||||
|
|
||||||
|
package metrics
|
||||||
|
|
||||||
|
func numCgoCall() int64 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
+609
@@ -0,0 +1,609 @@
|
|||||||
|
package metrics
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math"
|
||||||
|
"math/rand"
|
||||||
|
"sort"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const rescaleThreshold = time.Hour
|
||||||
|
|
||||||
|
// Samples maintain a statistically-significant selection of values from
|
||||||
|
// a stream.
|
||||||
|
type Sample interface {
|
||||||
|
Clear()
|
||||||
|
Count() int64
|
||||||
|
Max() int64
|
||||||
|
Mean() float64
|
||||||
|
Min() int64
|
||||||
|
Percentile(float64) float64
|
||||||
|
Percentiles([]float64) []float64
|
||||||
|
Size() int
|
||||||
|
Snapshot() Sample
|
||||||
|
StdDev() float64
|
||||||
|
Sum() int64
|
||||||
|
Update(int64)
|
||||||
|
Values() []int64
|
||||||
|
Variance() float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExpDecaySample is an exponentially-decaying sample using a forward-decaying
|
||||||
|
// priority reservoir. See Cormode et al's "Forward Decay: A Practical Time
|
||||||
|
// Decay Model for Streaming Systems".
|
||||||
|
//
|
||||||
|
// <http://www.research.att.com/people/Cormode_Graham/library/publications/CormodeShkapenyukSrivastavaXu09.pdf>
|
||||||
|
type ExpDecaySample struct {
|
||||||
|
alpha float64
|
||||||
|
count int64
|
||||||
|
mutex sync.Mutex
|
||||||
|
reservoirSize int
|
||||||
|
t0, t1 time.Time
|
||||||
|
values *expDecaySampleHeap
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewExpDecaySample constructs a new exponentially-decaying sample with the
|
||||||
|
// given reservoir size and alpha.
|
||||||
|
func NewExpDecaySample(reservoirSize int, alpha float64) Sample {
|
||||||
|
if UseNilMetrics {
|
||||||
|
return NilSample{}
|
||||||
|
}
|
||||||
|
s := &ExpDecaySample{
|
||||||
|
alpha: alpha,
|
||||||
|
reservoirSize: reservoirSize,
|
||||||
|
t0: time.Now(),
|
||||||
|
values: newExpDecaySampleHeap(reservoirSize),
|
||||||
|
}
|
||||||
|
s.t1 = s.t0.Add(rescaleThreshold)
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clear clears all samples.
|
||||||
|
func (s *ExpDecaySample) Clear() {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
s.count = 0
|
||||||
|
s.t0 = time.Now()
|
||||||
|
s.t1 = s.t0.Add(rescaleThreshold)
|
||||||
|
s.values.Clear()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count returns the number of samples recorded, which may exceed the
|
||||||
|
// reservoir size.
|
||||||
|
func (s *ExpDecaySample) Count() int64 {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
return s.count
|
||||||
|
}
|
||||||
|
|
||||||
|
// Max returns the maximum value in the sample, which may not be the maximum
|
||||||
|
// value ever to be part of the sample.
|
||||||
|
func (s *ExpDecaySample) Max() int64 {
|
||||||
|
return SampleMax(s.Values())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mean returns the mean of the values in the sample.
|
||||||
|
func (s *ExpDecaySample) Mean() float64 {
|
||||||
|
return SampleMean(s.Values())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Min returns the minimum value in the sample, which may not be the minimum
|
||||||
|
// value ever to be part of the sample.
|
||||||
|
func (s *ExpDecaySample) Min() int64 {
|
||||||
|
return SampleMin(s.Values())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Percentile returns an arbitrary percentile of values in the sample.
|
||||||
|
func (s *ExpDecaySample) Percentile(p float64) float64 {
|
||||||
|
return SamplePercentile(s.Values(), p)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Percentiles returns a slice of arbitrary percentiles of values in the
|
||||||
|
// sample.
|
||||||
|
func (s *ExpDecaySample) Percentiles(ps []float64) []float64 {
|
||||||
|
return SamplePercentiles(s.Values(), ps)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Size returns the size of the sample, which is at most the reservoir size.
|
||||||
|
func (s *ExpDecaySample) Size() int {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
return s.values.Size()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot returns a read-only copy of the sample.
|
||||||
|
func (s *ExpDecaySample) Snapshot() Sample {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
vals := s.values.Values()
|
||||||
|
values := make([]int64, len(vals))
|
||||||
|
for i, v := range vals {
|
||||||
|
values[i] = v.v
|
||||||
|
}
|
||||||
|
return &SampleSnapshot{
|
||||||
|
count: s.count,
|
||||||
|
values: values,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// StdDev returns the standard deviation of the values in the sample.
|
||||||
|
func (s *ExpDecaySample) StdDev() float64 {
|
||||||
|
return SampleStdDev(s.Values())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sum returns the sum of the values in the sample.
|
||||||
|
func (s *ExpDecaySample) Sum() int64 {
|
||||||
|
return SampleSum(s.Values())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update samples a new value.
|
||||||
|
func (s *ExpDecaySample) Update(v int64) {
|
||||||
|
s.update(time.Now(), v)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Values returns a copy of the values in the sample.
|
||||||
|
func (s *ExpDecaySample) Values() []int64 {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
vals := s.values.Values()
|
||||||
|
values := make([]int64, len(vals))
|
||||||
|
for i, v := range vals {
|
||||||
|
values[i] = v.v
|
||||||
|
}
|
||||||
|
return values
|
||||||
|
}
|
||||||
|
|
||||||
|
// Variance returns the variance of the values in the sample.
|
||||||
|
func (s *ExpDecaySample) Variance() float64 {
|
||||||
|
return SampleVariance(s.Values())
|
||||||
|
}
|
||||||
|
|
||||||
|
// update samples a new value at a particular timestamp. This is a method all
|
||||||
|
// its own to facilitate testing.
|
||||||
|
func (s *ExpDecaySample) update(t time.Time, v int64) {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
s.count++
|
||||||
|
if s.values.Size() == s.reservoirSize {
|
||||||
|
s.values.Pop()
|
||||||
|
}
|
||||||
|
s.values.Push(expDecaySample{
|
||||||
|
k: math.Exp(t.Sub(s.t0).Seconds()*s.alpha) / rand.Float64(),
|
||||||
|
v: v,
|
||||||
|
})
|
||||||
|
if t.After(s.t1) {
|
||||||
|
values := s.values.Values()
|
||||||
|
t0 := s.t0
|
||||||
|
s.values.Clear()
|
||||||
|
s.t0 = t
|
||||||
|
s.t1 = s.t0.Add(rescaleThreshold)
|
||||||
|
for _, v := range values {
|
||||||
|
v.k = v.k * math.Exp(-s.alpha*s.t0.Sub(t0).Seconds())
|
||||||
|
s.values.Push(v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NilSample is a no-op Sample.
|
||||||
|
type NilSample struct{}
|
||||||
|
|
||||||
|
// Clear is a no-op.
|
||||||
|
func (NilSample) Clear() {}
|
||||||
|
|
||||||
|
// Count is a no-op.
|
||||||
|
func (NilSample) Count() int64 { return 0 }
|
||||||
|
|
||||||
|
// Max is a no-op.
|
||||||
|
func (NilSample) Max() int64 { return 0 }
|
||||||
|
|
||||||
|
// Mean is a no-op.
|
||||||
|
func (NilSample) Mean() float64 { return 0.0 }
|
||||||
|
|
||||||
|
// Min is a no-op.
|
||||||
|
func (NilSample) Min() int64 { return 0 }
|
||||||
|
|
||||||
|
// Percentile is a no-op.
|
||||||
|
func (NilSample) Percentile(p float64) float64 { return 0.0 }
|
||||||
|
|
||||||
|
// Percentiles is a no-op.
|
||||||
|
func (NilSample) Percentiles(ps []float64) []float64 {
|
||||||
|
return make([]float64, len(ps))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Size is a no-op.
|
||||||
|
func (NilSample) Size() int { return 0 }
|
||||||
|
|
||||||
|
// Sample is a no-op.
|
||||||
|
func (NilSample) Snapshot() Sample { return NilSample{} }
|
||||||
|
|
||||||
|
// StdDev is a no-op.
|
||||||
|
func (NilSample) StdDev() float64 { return 0.0 }
|
||||||
|
|
||||||
|
// Sum is a no-op.
|
||||||
|
func (NilSample) Sum() int64 { return 0 }
|
||||||
|
|
||||||
|
// Update is a no-op.
|
||||||
|
func (NilSample) Update(v int64) {}
|
||||||
|
|
||||||
|
// Values is a no-op.
|
||||||
|
func (NilSample) Values() []int64 { return []int64{} }
|
||||||
|
|
||||||
|
// Variance is a no-op.
|
||||||
|
func (NilSample) Variance() float64 { return 0.0 }
|
||||||
|
|
||||||
|
// SampleMax returns the maximum value of the slice of int64.
|
||||||
|
func SampleMax(values []int64) int64 {
|
||||||
|
if 0 == len(values) {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
var max int64 = math.MinInt64
|
||||||
|
for _, v := range values {
|
||||||
|
if max < v {
|
||||||
|
max = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return max
|
||||||
|
}
|
||||||
|
|
||||||
|
// SampleMean returns the mean value of the slice of int64.
|
||||||
|
func SampleMean(values []int64) float64 {
|
||||||
|
if 0 == len(values) {
|
||||||
|
return 0.0
|
||||||
|
}
|
||||||
|
return float64(SampleSum(values)) / float64(len(values))
|
||||||
|
}
|
||||||
|
|
||||||
|
// SampleMin returns the minimum value of the slice of int64.
|
||||||
|
func SampleMin(values []int64) int64 {
|
||||||
|
if 0 == len(values) {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
var min int64 = math.MaxInt64
|
||||||
|
for _, v := range values {
|
||||||
|
if min > v {
|
||||||
|
min = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return min
|
||||||
|
}
|
||||||
|
|
||||||
|
// SamplePercentiles returns an arbitrary percentile of the slice of int64.
|
||||||
|
func SamplePercentile(values int64Slice, p float64) float64 {
|
||||||
|
return SamplePercentiles(values, []float64{p})[0]
|
||||||
|
}
|
||||||
|
|
||||||
|
// SamplePercentiles returns a slice of arbitrary percentiles of the slice of
|
||||||
|
// int64.
|
||||||
|
func SamplePercentiles(values int64Slice, ps []float64) []float64 {
|
||||||
|
scores := make([]float64, len(ps))
|
||||||
|
size := len(values)
|
||||||
|
if size > 0 {
|
||||||
|
sort.Sort(values)
|
||||||
|
for i, p := range ps {
|
||||||
|
pos := p * float64(size+1)
|
||||||
|
if pos < 1.0 {
|
||||||
|
scores[i] = float64(values[0])
|
||||||
|
} else if pos >= float64(size) {
|
||||||
|
scores[i] = float64(values[size-1])
|
||||||
|
} else {
|
||||||
|
lower := float64(values[int(pos)-1])
|
||||||
|
upper := float64(values[int(pos)])
|
||||||
|
scores[i] = lower + (pos-math.Floor(pos))*(upper-lower)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return scores
|
||||||
|
}
|
||||||
|
|
||||||
|
// SampleSnapshot is a read-only copy of another Sample.
|
||||||
|
type SampleSnapshot struct {
|
||||||
|
count int64
|
||||||
|
values []int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clear panics.
|
||||||
|
func (*SampleSnapshot) Clear() {
|
||||||
|
panic("Clear called on a SampleSnapshot")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count returns the count of inputs at the time the snapshot was taken.
|
||||||
|
func (s *SampleSnapshot) Count() int64 { return s.count }
|
||||||
|
|
||||||
|
// Max returns the maximal value at the time the snapshot was taken.
|
||||||
|
func (s *SampleSnapshot) Max() int64 { return SampleMax(s.values) }
|
||||||
|
|
||||||
|
// Mean returns the mean value at the time the snapshot was taken.
|
||||||
|
func (s *SampleSnapshot) Mean() float64 { return SampleMean(s.values) }
|
||||||
|
|
||||||
|
// Min returns the minimal value at the time the snapshot was taken.
|
||||||
|
func (s *SampleSnapshot) Min() int64 { return SampleMin(s.values) }
|
||||||
|
|
||||||
|
// Percentile returns an arbitrary percentile of values at the time the
|
||||||
|
// snapshot was taken.
|
||||||
|
func (s *SampleSnapshot) Percentile(p float64) float64 {
|
||||||
|
return SamplePercentile(s.values, p)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Percentiles returns a slice of arbitrary percentiles of values at the time
|
||||||
|
// the snapshot was taken.
|
||||||
|
func (s *SampleSnapshot) Percentiles(ps []float64) []float64 {
|
||||||
|
return SamplePercentiles(s.values, ps)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Size returns the size of the sample at the time the snapshot was taken.
|
||||||
|
func (s *SampleSnapshot) Size() int { return len(s.values) }
|
||||||
|
|
||||||
|
// Snapshot returns the snapshot.
|
||||||
|
func (s *SampleSnapshot) Snapshot() Sample { return s }
|
||||||
|
|
||||||
|
// StdDev returns the standard deviation of values at the time the snapshot was
|
||||||
|
// taken.
|
||||||
|
func (s *SampleSnapshot) StdDev() float64 { return SampleStdDev(s.values) }
|
||||||
|
|
||||||
|
// Sum returns the sum of values at the time the snapshot was taken.
|
||||||
|
func (s *SampleSnapshot) Sum() int64 { return SampleSum(s.values) }
|
||||||
|
|
||||||
|
// Update panics.
|
||||||
|
func (*SampleSnapshot) Update(int64) {
|
||||||
|
panic("Update called on a SampleSnapshot")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Values returns a copy of the values in the sample.
|
||||||
|
func (s *SampleSnapshot) Values() []int64 {
|
||||||
|
values := make([]int64, len(s.values))
|
||||||
|
copy(values, s.values)
|
||||||
|
return values
|
||||||
|
}
|
||||||
|
|
||||||
|
// Variance returns the variance of values at the time the snapshot was taken.
|
||||||
|
func (s *SampleSnapshot) Variance() float64 { return SampleVariance(s.values) }
|
||||||
|
|
||||||
|
// SampleStdDev returns the standard deviation of the slice of int64.
|
||||||
|
func SampleStdDev(values []int64) float64 {
|
||||||
|
return math.Sqrt(SampleVariance(values))
|
||||||
|
}
|
||||||
|
|
||||||
|
// SampleSum returns the sum of the slice of int64.
|
||||||
|
func SampleSum(values []int64) int64 {
|
||||||
|
var sum int64
|
||||||
|
for _, v := range values {
|
||||||
|
sum += v
|
||||||
|
}
|
||||||
|
return sum
|
||||||
|
}
|
||||||
|
|
||||||
|
// SampleVariance returns the variance of the slice of int64.
|
||||||
|
func SampleVariance(values []int64) float64 {
|
||||||
|
if 0 == len(values) {
|
||||||
|
return 0.0
|
||||||
|
}
|
||||||
|
m := SampleMean(values)
|
||||||
|
var sum float64
|
||||||
|
for _, v := range values {
|
||||||
|
d := float64(v) - m
|
||||||
|
sum += d * d
|
||||||
|
}
|
||||||
|
return sum / float64(len(values))
|
||||||
|
}
|
||||||
|
|
||||||
|
// A uniform sample using Vitter's Algorithm R.
|
||||||
|
//
|
||||||
|
// <http://www.cs.umd.edu/~samir/498/vitter.pdf>
|
||||||
|
type UniformSample struct {
|
||||||
|
count int64
|
||||||
|
mutex sync.Mutex
|
||||||
|
reservoirSize int
|
||||||
|
values []int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewUniformSample constructs a new uniform sample with the given reservoir
|
||||||
|
// size.
|
||||||
|
func NewUniformSample(reservoirSize int) Sample {
|
||||||
|
if UseNilMetrics {
|
||||||
|
return NilSample{}
|
||||||
|
}
|
||||||
|
return &UniformSample{
|
||||||
|
reservoirSize: reservoirSize,
|
||||||
|
values: make([]int64, 0, reservoirSize),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clear clears all samples.
|
||||||
|
func (s *UniformSample) Clear() {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
s.count = 0
|
||||||
|
s.values = make([]int64, 0, s.reservoirSize)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count returns the number of samples recorded, which may exceed the
|
||||||
|
// reservoir size.
|
||||||
|
func (s *UniformSample) Count() int64 {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
return s.count
|
||||||
|
}
|
||||||
|
|
||||||
|
// Max returns the maximum value in the sample, which may not be the maximum
|
||||||
|
// value ever to be part of the sample.
|
||||||
|
func (s *UniformSample) Max() int64 {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
return SampleMax(s.values)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mean returns the mean of the values in the sample.
|
||||||
|
func (s *UniformSample) Mean() float64 {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
return SampleMean(s.values)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Min returns the minimum value in the sample, which may not be the minimum
|
||||||
|
// value ever to be part of the sample.
|
||||||
|
func (s *UniformSample) Min() int64 {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
return SampleMin(s.values)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Percentile returns an arbitrary percentile of values in the sample.
|
||||||
|
func (s *UniformSample) Percentile(p float64) float64 {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
return SamplePercentile(s.values, p)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Percentiles returns a slice of arbitrary percentiles of values in the
|
||||||
|
// sample.
|
||||||
|
func (s *UniformSample) Percentiles(ps []float64) []float64 {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
return SamplePercentiles(s.values, ps)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Size returns the size of the sample, which is at most the reservoir size.
|
||||||
|
func (s *UniformSample) Size() int {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
return len(s.values)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot returns a read-only copy of the sample.
|
||||||
|
func (s *UniformSample) Snapshot() Sample {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
values := make([]int64, len(s.values))
|
||||||
|
copy(values, s.values)
|
||||||
|
return &SampleSnapshot{
|
||||||
|
count: s.count,
|
||||||
|
values: values,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// StdDev returns the standard deviation of the values in the sample.
|
||||||
|
func (s *UniformSample) StdDev() float64 {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
return SampleStdDev(s.values)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sum returns the sum of the values in the sample.
|
||||||
|
func (s *UniformSample) Sum() int64 {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
return SampleSum(s.values)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Update samples a new value.
|
||||||
|
func (s *UniformSample) Update(v int64) {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
s.count++
|
||||||
|
if len(s.values) < s.reservoirSize {
|
||||||
|
s.values = append(s.values, v)
|
||||||
|
} else {
|
||||||
|
r := rand.Int63n(s.count)
|
||||||
|
if r < int64(len(s.values)) {
|
||||||
|
s.values[int(r)] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Values returns a copy of the values in the sample.
|
||||||
|
func (s *UniformSample) Values() []int64 {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
values := make([]int64, len(s.values))
|
||||||
|
copy(values, s.values)
|
||||||
|
return values
|
||||||
|
}
|
||||||
|
|
||||||
|
// Variance returns the variance of the values in the sample.
|
||||||
|
func (s *UniformSample) Variance() float64 {
|
||||||
|
s.mutex.Lock()
|
||||||
|
defer s.mutex.Unlock()
|
||||||
|
return SampleVariance(s.values)
|
||||||
|
}
|
||||||
|
|
||||||
|
// expDecaySample represents an individual sample in a heap.
|
||||||
|
type expDecaySample struct {
|
||||||
|
k float64
|
||||||
|
v int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func newExpDecaySampleHeap(reservoirSize int) *expDecaySampleHeap {
|
||||||
|
return &expDecaySampleHeap{make([]expDecaySample, 0, reservoirSize)}
|
||||||
|
}
|
||||||
|
|
||||||
|
// expDecaySampleHeap is a min-heap of expDecaySamples.
|
||||||
|
// The internal implementation is copied from the standard library's container/heap
|
||||||
|
type expDecaySampleHeap struct {
|
||||||
|
s []expDecaySample
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *expDecaySampleHeap) Clear() {
|
||||||
|
h.s = h.s[:0]
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *expDecaySampleHeap) Push(s expDecaySample) {
|
||||||
|
n := len(h.s)
|
||||||
|
h.s = h.s[0 : n+1]
|
||||||
|
h.s[n] = s
|
||||||
|
h.up(n)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *expDecaySampleHeap) Pop() expDecaySample {
|
||||||
|
n := len(h.s) - 1
|
||||||
|
h.s[0], h.s[n] = h.s[n], h.s[0]
|
||||||
|
h.down(0, n)
|
||||||
|
|
||||||
|
n = len(h.s)
|
||||||
|
s := h.s[n-1]
|
||||||
|
h.s = h.s[0 : n-1]
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *expDecaySampleHeap) Size() int {
|
||||||
|
return len(h.s)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *expDecaySampleHeap) Values() []expDecaySample {
|
||||||
|
return h.s
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *expDecaySampleHeap) up(j int) {
|
||||||
|
for {
|
||||||
|
i := (j - 1) / 2 // parent
|
||||||
|
if i == j || !(h.s[j].k < h.s[i].k) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
h.s[i], h.s[j] = h.s[j], h.s[i]
|
||||||
|
j = i
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *expDecaySampleHeap) down(i, n int) {
|
||||||
|
for {
|
||||||
|
j1 := 2*i + 1
|
||||||
|
if j1 >= n || j1 < 0 { // j1 < 0 after int overflow
|
||||||
|
break
|
||||||
|
}
|
||||||
|
j := j1 // left child
|
||||||
|
if j2 := j1 + 1; j2 < n && !(h.s[j1].k < h.s[j2].k) {
|
||||||
|
j = j2 // = 2*i + 2 // right child
|
||||||
|
}
|
||||||
|
if !(h.s[j].k < h.s[i].k) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
h.s[i], h.s[j] = h.s[j], h.s[i]
|
||||||
|
i = j
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type int64Slice []int64
|
||||||
|
|
||||||
|
func (p int64Slice) Len() int { return len(p) }
|
||||||
|
func (p int64Slice) Less(i, j int) bool { return p[i] < p[j] }
|
||||||
|
func (p int64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
|
||||||
Loaded 100 of 1078 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user