gui: Temporarily disable the usage reporting prompt (ref #3301)

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3292
LGTM: AudriusButkevicius, calmh

.gitattributes

@@ -2,8 +2,7 @@
 *   text=auto
 
 # Except the dependencies, which we leave alone
-Godeps/**   -text=auto
+vendor/**   -text=auto
 
 # Diffs on these files are meaningless
-gui.files.go    -diff
 *.svg           -diff

.gitignore

@@ -1,18 +1,17 @@
-./syncthing
+/syncthing
+/discosrv
 syncthing.exe
+discosrv.exe
 *.tar.gz
 *.zip
 *.asc
-*.sublime*
-.idea/
 .jshintrc
 coverage.out
 files/pidx
 bin
 perfstats*.csv
 coverage.xml
-!gui/scripts/syncthing
-.DS_Store
-syncthing.md5
-syncthing.exe.md5
+syncthing.sig
 RELEASE
+deb
+lib/auto/gui.files.go

AUTHORS

@@ -1,55 +1,97 @@
 # This is the official list of Syncthing authors for copyright purposes.
+# The format is:
+#
+#    Name Name Name (nickname) <email1@example.com> <email2@example.com>
+#
+# The NICKS list is auto generated from this file.
 
-Aaron Bieber <qbit@deftly.net>
-Alexander Graf <register-github@alex-graf.de>
-Andrew Dunham <andrew@du.nham.ca>
-Audrius Butkevicius <audrius.butkevicius@gmail.com>
-Arthur Axel fREW Schmidt <frew@afoolishmanifesto.com> <frioux@gmail.com>
-Ben Curthoys <ben@bencurthoys.com>
-Ben Schulz <ueomkail@gmail.com> <uok@users.noreply.github.com>
-Ben Sidhom <bsidhom@gmail.com>
-Brandon Philips <brandon@ifup.org>
-Brendan Long <self@brendanlong.com>
-Caleb Callaway <enlightened.despot@gmail.com>
-Carsten Hagemann <moter8@gmail.com>
-Cathryne Linenweaver <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
-Chris Joel <chris@scriptolo.gy>
-Colin Kennedy <moshen.colin@gmail.com>
-Daniel Martí <mvdan@mvdan.cc>
-Dennis Wilson <dw@risu.io>
-Dominik Heidler <dominik@heidler.eu>
-Elias Jarlebring <jarlebring@gmail.com>
-Emil Hessman <emil@hessman.se>
-Federico Castagnini <federico.castagnini@gmail.com>
-Felix Ableitner <me@nutomic.com>
-Felix Unterpaintner <bigbear2nd@gmail.com>
-Francois-Xavier Gsell <fxgsell@gmail.com>
-Gilli Sigurdsson <gilli@vx.is>
-Jakob Borg <jakob@nym.se>
-James Patterson <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
-Jaroslav Malec <dzardacz@gmail.com>
-Jens Diemer <github.com@jensdiemer.de> <git@jensdiemer.de>
-Jochen Voss <voss@seehuhn.de>
-Johan Vromans <jvromans@squirrel.nl>
-Karol Różycki <rozycki.karol@gmail.com>
-Ken'ichi Kamada <kamada@nanohz.org>
-Lode Hoste <zillode@zillode.be>
-Marcin Dziadus <dziadus.marcin@gmail.com>
-Marc Laporte <marc@marclaporte.com> <marc@laporte.name>
-Marc Pujol <kilburn@la3.org>
-Michael Jephcote <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
-Michael Tilli <pyfisch@gmail.com>
-Pascal Jungblut <github@pascalj.com> <mail@pascal-jungblut.com>
-Peter Hoeg <peter@speartail.com>
-Philippe Schommers <philippe@schommers.be>
-Phill Luby <phill.luby@newredo.com>
-Piotr Bejda <piotrb10@gmail.com>
-Ryan Sullivan <kayoticsully@gmail.com>
-Sergey Mishin <ralder@yandex.ru>
-Stefan Tatschner <stefan@sevenbyte.org>
-Tim Abell <tim@timwise.co.uk>
-Tobias Nygren <tnn@nygren.pp.se>
-Tomas Cerveny <kozec@kozec.com>
-Tully Robinson <tully@tojr.org>
-Veeti Paananen <veeti.paananen@rojekti.fi>
-Vil Brekin <vilbrekin@gmail.com>
+Aaron Bieber (qbit) <qbit@deftly.net>
+Adam Piggott (simplypeachy) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com>
+Alessandro G. (alessandro.g89) <alessandro.g89@gmail.com>
+Alexander Graf (alex2108) <register-github@alex-graf.de>
+Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
+Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
+Andrew Dunham (andrew-d) <andrew@du.nham.ca>
+Andrey D (scienmind) <scintertech@cryptolab.net>
+Antony Male (canton7) <antony.male@gmail.com>
+Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
+Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com>
+Bart De Vries (mogwa1) <devriesb@gmail.com>
+Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
+Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
+Ben Sidhom (bsidhom) <bsidhom@gmail.com>
+Benny Ng (tpng) <benny.tpng@gmail.com>
+Brandon Philips (philips) <brandon@ifup.org>
+Brendan Long (brendanlong) <self@brendanlong.com>
+Brian R. Becker (brbecker) <brbecker@gmail.com>
+Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
+Carsten Hagemann (Moter8) <moter8@gmail.com>
+Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
+Chris Howie (cdhowie) <me@chrishowie.com>
+Chris Joel (cdata) <chris@scriptolo.gy>
+Colin Kennedy (moshen) <moshen.colin@gmail.com>
+Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
+Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
+Daniel Martí (mvdan) <mvdan@mvdan.cc>
+David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
+Denis A. (dva) <denisva@gmail.com>
+Dennis Wilson (snnd) <dw@risu.io>
+Dominik Heidler (asdil12) <dominik@heidler.eu>
+Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
+Emil Hessman (ceh) <emil@hessman.se>
+Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
+Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
+Felix Ableitner (Nutomic) <me@nutomic.com>
+Felix Unterpaintner (bigbear2nd) <bigbear2nd@gmail.com>
+Francois-Xavier Gsell (zukoo) <fxgsell@gmail.com>
+Frank Isemann (fti7) <frank@isemann.name>
+Gilli Sigurdsson (gillisig) <gilli@vx.is>
+Jaakko Hannikainen (jgke) <jgke@jgke.fi>
+Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
+Jake Peterson (acogdev) <jake@acogdev.com>
+Jakob Borg (calmh) <jakob@nym.se>
+James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
+Jaroslav Malec (dzarda) <dzardacz@gmail.com>
+Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
+Jochen Voss (seehuhn) <voss@seehuhn.de>
+Johan Vromans (sciurius) <jvromans@squirrel.nl>
+Karol Różycki (krozycki) <rozycki.karol@gmail.com>
+Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
+Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
+Kevin Allen (ironmig) <kma1660@gmail.com>
+Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
+Laurent Etiemble (letiemble) <laurent.etiemble@gmail.com> <laurent.etiemble@monobjc.net>
+Lode Hoste (Zillode) <zillode@zillode.be>
+Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
+Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
+Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
+Marc Pujol (kilburn) <kilburn@la3.org>
+Marcin Dziadus (marcindziadus) <dziadus.marcin@gmail.com>
+Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
+Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
+Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
+Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
+Michael Ploujnikov (plouj) <ploujj@gmail.com>
+Michael Tilli (pyfisch) <pyfisch@gmail.com>
+Nate Morrison (nrm21) <natemorrison@gmail.com>
+Pascal Jungblut (pascalj) <github@pascalj.com> <mail@pascal-jungblut.com>
+Peter Hoeg (peterhoeg) <peter@speartail.com>
+Philippe Schommers (filoozoom) <philippe@schommers.be>
+Phill Luby (pluby) <phill.luby@newredo.com>
+Piotr Bejda (piobpl) <piotrb10@gmail.com>
+Ryan Sullivan (KayoticSully) <kayoticsully@gmail.com>
+Scott Klupfel (kluppy) <kluppy@going2blue.com>
+Sergey Mishin (ralder) <ralder@yandex.ru>
+Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
+Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
+Tim Abell (timabell) <tim@timwise.co.uk>
+Tobias Nygren (tnn2) <tnn@nygren.pp.se>
+Tomas Cerveny (kozec) <kozec@kozec.com>
+Tully Robinson (tojrobinson) <tully@tojr.org>
+Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
+Veeti Paananen (veeti) <veeti.paananen@rojekti.fi>
+Victor Buinsky (buinsky) <vix_booja@tut.by>
+Vil Brekin (Vilbrekin) <vilbrekin@gmail.com>
+William A. Kennington III (wkennington) <william@wkennington.com>
+Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de>
+Yannic A. (eipiminus1) <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>

CONTRIBUTING.md

@@ -32,109 +32,32 @@ latest info on Transifex.
 ## Contributing Code
 
 Every contribution is welcome. If you want to contribute but are unsure
-where to start, any open issues are fair game! Be prepared for a
-[certain amount of review](https://github.com/syncthing/syncthing/wiki/FAQ#why-are-you-being-so-hard-on-my-pull-request);
-it's all in the name of quality. :) Following the points below will make this
-a smoother process.
+where to start, any open issues are fair game! See the [Contribution
+Guidelines](http://docs.syncthing.net/dev/contributing.html) for the full
+story on committing code.
 
-Individuals making significant and valuable contributions are given
-commit-access to the project. If you make a significant contribution and
-are not considered for commit-access, please contact any of the
-Syncthing core team members.
+## Contributing Documentation
 
-All nontrivial contributions should go through the pull request
-mechanism for internal review. Determining what is "nontrivial" is left
-at the discretion of the contributor.
-
-### Authorship
-
-All code authors are listed in the AUTHORS file. Commits must be made
-with the same name and email as listed in the AUTHORS file. To
-accomplish this, ensure that your git configuration is set correctly
-prior to making your first commit;
-
-    $ git config --global user.name "Jane Doe"
-    $ git config --global user.email janedoe@example.com
-
-You must be reachable on the given email address. If you do not wish to
-use your real name for whatever reason, using a nickname or pseudonym is
-perfectly acceptable.
-
-### Core Team
-
-The Syncthing core team currently consists of the following members;
-
- - Jakob Borg (@calmh)
- - Audrius Butkevicius (@AudriusButkevicius)
-
-## Coding Style
-
-- Follow the conventions laid out in [Effective Go](https://golang.org/doc/effective_go.html)
-  as much as makes sense.
-
-- All text files use Unix line endings.
-
-- Each commit should be `go fmt` clean.
-
-- The commit message subject should be a single short sentence
-  describing the change, starting with a capital letter.
-
-- Commits that resolve an existing issue must include the issue number
-  as `(fixes #123)` at the end of the commit message subject.
-
-- Imports are grouped per `goimports` standard; that is, standard
-  library first, then third party libraries after a blank line.
-
-- A contribution solving a single issue or introducing a single new
-  feature should probably be a single commit based on the current
-  `master` branch. You may be asked to "rebase" or "squash" your pull
-  request to make sure this is the case, especially if there have been
-  amendments during review.
+Updates to the [documentation site](http://docs.syncthing.net/) can be
+made as pull requests on the [documentation
+repository](https://github.com/syncthing/docs).
 
 ## Licensing
 
-All contributions are made under the same MPLv2 license as the rest of
-the project, except documentation, user interface text and translation
-strings which are licensed under the Creative Commons Attribution 4.0
-International License. You retain the copyright to code you have
-written.
+All contributions are made available under the same license as the already
+existing material being contributed to. For most of the project and unless
+otherwise stated this means MPLv2, but there are exceptions:
 
-When accepting your first contribution, the maintainer of the project
-will ensure that you are added to the AUTHORS file. You are welcome to
-add yourself as a separate commit in your first pull request.
+- Certain commands (under cmd/...) may have a separate license, indicated by
+  the presence of a LICENSE file in the corresponding directory.
 
-## Building
+- The documentation (man/...) is licensed under the Creative Commons
+  Attribution 4.0 International License.
 
-[See the documentation](https://github.com/syncthing/syncthing/wiki/Building)
-on how to get started with a build environment.
+- Projects under vendor/... are copyright by and licensed from their
+  respective original authors. Contributions should be made to the original
+  project, not here.
 
-## Branches
+Regardless of the license in effect, you retain the copyright to your
+contribution.
 
- - `master` is the main branch containing good code that will end up in
-   the next release. You should base your work on it. It won't ever be
-   rebased or force-pushed to.
-
- - `vx.y` branches exist to make patch releases on otherwise obsolete
-   minor releases. Should only contain fixes cherry picked from master.
-   Don't base any work on them.
-
- - Other branches are probably topic branches and may be subject to
-   rebasing. Don't base any work on them unless you specifically know
-   otherwise.
-
-## Tags
-
-All releases are tagged semver style as `vx.y.z`. Release tags are
-signed by GPG key BCE524C7.
-
-## Tests
-
-Yes please!
-
-## Documentation
-
-[Over here!](https://github.com/syncthing/syncthing/wiki)
-
-## License
-
-MPLv2

Godeps/Godeps.json

@@ -1,77 +0,0 @@
-{
-	"ImportPath": "github.com/syncthing/syncthing",
-	"GoVersion": "go1.4",
-	"Packages": [
-		"./cmd/..."
-	],
-	"Deps": [
-		{
-			"ImportPath": "github.com/bkaradzic/go-lz4",
-			"Rev": "93a831dcee242be64a9cc9803dda84af25932de7"
-		},
-		{
-			"ImportPath": "github.com/calmh/logger",
-			"Rev": "4d4e2801954c5581e4c2a80a3d3beb3b3645fd04"
-		},
-		{
-			"ImportPath": "github.com/calmh/luhn",
-			"Rev": "0c8388ff95fa92d4094011e5a04fc99dea3d1632"
-		},
-		{
-			"ImportPath": "github.com/calmh/xdr",
-			"Rev": "5f7208e86762911861c94f1849eddbfc0a60cbf0"
-		},
-		{
-			"ImportPath": "github.com/juju/ratelimit",
-			"Rev": "c5abe513796336ee2869745bff0638508450e9c5"
-		},
-		{
-			"ImportPath": "github.com/kardianos/osext",
-			"Rev": "efacde03154693404c65e7aa7d461ac9014acd0c"
-		},
-		{
-			"ImportPath": "github.com/syncthing/protocol",
-			"Rev": "e7db2648034fb71b051902a02bc25d4468ed492e"
-		},
-		{
-			"ImportPath": "github.com/syndtr/goleveldb/leveldb",
-			"Rev": "87e4e645d80ae9c537e8f2dee52b28036a5dd75e"
-		},
-		{
-			"ImportPath": "github.com/syndtr/gosnappy/snappy",
-			"Rev": "156a073208e131d7d2e212cb749feae7c339e846"
-		},
-		{
-			"ImportPath": "github.com/thejerf/suture",
-			"Rev": "ff19fb384c3fe30f42717967eaa69da91e5f317c"
-		},
-		{
-			"ImportPath": "github.com/vitrun/qart/coding",
-			"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
-		},
-		{
-			"ImportPath": "github.com/vitrun/qart/gf256",
-			"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
-		},
-		{
-			"ImportPath": "github.com/vitrun/qart/qr",
-			"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
-		},
-		{
-			"ImportPath": "golang.org/x/crypto/bcrypt",
-			"Rev": "c57d4a71915a248dbad846d60825145062b4c18e"
-		},
-		{
-			"ImportPath": "golang.org/x/crypto/blowfish",
-			"Rev": "c57d4a71915a248dbad846d60825145062b4c18e"
-		},
-		{
-			"ImportPath": "golang.org/x/text/transform",
-			"Rev": "2076e9cab4147459c82bc81169e46c139d358547"
-		},
-		{
-			"ImportPath": "golang.org/x/text/unicode/norm",
-			"Rev": "2076e9cab4147459c82bc81169e46c139d358547"
-		}
-	]
-}

Godeps/Readme

@@ -1,5 +0,0 @@
-This directory tree is generated automatically by godep.
-
-Please do not edit.
-
-See https://github.com/tools/godep for more information.

Godeps/_workspace/.gitignore

@@ -1,2 +0,0 @@
-/pkg
-/bin

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/.gitignore

@@ -1 +0,0 @@
-/lz4-example/lz4-example

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/.travis.yml

@@ -1,7 +0,0 @@
-language: go
-
-go:
-  - 1.1
-  - 1.2
-  - 1.3
-  - tip

Godeps/_workspace/src/github.com/calmh/logger/README.md

@@ -1,15 +0,0 @@
-logger
-======
-
-A small wrapper around `log` to provide log levels.
-
-Documentation
--------------
-
-http://godoc.org/github.com/calmh/logger
-
-License
--------
-
-MIT
-

Godeps/_workspace/src/github.com/calmh/logger/logger.go

@@ -1,179 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-// Package logger implements a standardized logger with callback functionality
-package logger
-
-import (
-	"fmt"
-	"log"
-	"os"
-	"strings"
-	"sync"
-)
-
-type LogLevel int
-
-const (
-	LevelDebug LogLevel = iota
-	LevelVerbose
-	LevelInfo
-	LevelOK
-	LevelWarn
-	LevelFatal
-	NumLevels
-)
-
-// A MessageHandler is called with the log level and message text.
-type MessageHandler func(l LogLevel, msg string)
-
-type Logger struct {
-	logger   *log.Logger
-	handlers [NumLevels][]MessageHandler
-	mut      sync.Mutex
-}
-
-// The default logger logs to standard output with a time prefix.
-var DefaultLogger = New()
-
-func New() *Logger {
-	return &Logger{
-		logger: log.New(os.Stdout, "", log.Ltime),
-	}
-}
-
-// AddHandler registers a new MessageHandler to receive messages with the
-// specified log level or above.
-func (l *Logger) AddHandler(level LogLevel, h MessageHandler) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	l.handlers[level] = append(l.handlers[level], h)
-}
-
-// See log.SetFlags
-func (l *Logger) SetFlags(flag int) {
-	l.logger.SetFlags(flag)
-}
-
-// See log.SetPrefix
-func (l *Logger) SetPrefix(prefix string) {
-	l.logger.SetPrefix(prefix)
-}
-
-func (l *Logger) callHandlers(level LogLevel, s string) {
-	for _, h := range l.handlers[level] {
-		h(level, strings.TrimSpace(s))
-	}
-}
-
-// Debugln logs a line with a DEBUG prefix.
-func (l *Logger) Debugln(vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintln(vals...)
-	l.logger.Output(2, "DEBUG: "+s)
-	l.callHandlers(LevelDebug, s)
-}
-
-// Debugf logs a formatted line with a DEBUG prefix.
-func (l *Logger) Debugf(format string, vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintf(format, vals...)
-	l.logger.Output(2, "DEBUG: "+s)
-	l.callHandlers(LevelDebug, s)
-}
-
-// Infoln logs a line with a VERBOSE prefix.
-func (l *Logger) Verboseln(vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintln(vals...)
-	l.logger.Output(2, "VERBOSE: "+s)
-	l.callHandlers(LevelVerbose, s)
-}
-
-// Infof logs a formatted line with a VERBOSE prefix.
-func (l *Logger) Verbosef(format string, vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintf(format, vals...)
-	l.logger.Output(2, "VERBOSE: "+s)
-	l.callHandlers(LevelVerbose, s)
-}
-
-// Infoln logs a line with an INFO prefix.
-func (l *Logger) Infoln(vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintln(vals...)
-	l.logger.Output(2, "INFO: "+s)
-	l.callHandlers(LevelInfo, s)
-}
-
-// Infof logs a formatted line with an INFO prefix.
-func (l *Logger) Infof(format string, vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintf(format, vals...)
-	l.logger.Output(2, "INFO: "+s)
-	l.callHandlers(LevelInfo, s)
-}
-
-// Okln logs a line with an OK prefix.
-func (l *Logger) Okln(vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintln(vals...)
-	l.logger.Output(2, "OK: "+s)
-	l.callHandlers(LevelOK, s)
-}
-
-// Okf logs a formatted line with an OK prefix.
-func (l *Logger) Okf(format string, vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintf(format, vals...)
-	l.logger.Output(2, "OK: "+s)
-	l.callHandlers(LevelOK, s)
-}
-
-// Warnln logs a formatted line with a WARNING prefix.
-func (l *Logger) Warnln(vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintln(vals...)
-	l.logger.Output(2, "WARNING: "+s)
-	l.callHandlers(LevelWarn, s)
-}
-
-// Warnf logs a formatted line with a WARNING prefix.
-func (l *Logger) Warnf(format string, vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintf(format, vals...)
-	l.logger.Output(2, "WARNING: "+s)
-	l.callHandlers(LevelWarn, s)
-}
-
-// Fatalln logs a line with a FATAL prefix and exits the process with exit
-// code 1.
-func (l *Logger) Fatalln(vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintln(vals...)
-	l.logger.Output(2, "FATAL: "+s)
-	l.callHandlers(LevelFatal, s)
-	os.Exit(1)
-}
-
-// Fatalf logs a formatted line with a FATAL prefix and exits the process with
-// exit code 1.
-func (l *Logger) Fatalf(format string, vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintf(format, vals...)
-	l.logger.Output(2, "FATAL: "+s)
-	l.callHandlers(LevelFatal, s)
-	os.Exit(1)
-}

Godeps/_workspace/src/github.com/calmh/logger/logger_test.go

@@ -1,58 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-package logger
-
-import (
-	"strings"
-	"testing"
-)
-
-func TestAPI(t *testing.T) {
-	l := New()
-	l.SetFlags(0)
-	l.SetPrefix("testing")
-
-	debug := 0
-	l.AddHandler(LevelDebug, checkFunc(t, LevelDebug, "test 0", &debug))
-	info := 0
-	l.AddHandler(LevelInfo, checkFunc(t, LevelInfo, "test 1", &info))
-	warn := 0
-	l.AddHandler(LevelWarn, checkFunc(t, LevelWarn, "test 2", &warn))
-	ok := 0
-	l.AddHandler(LevelOK, checkFunc(t, LevelOK, "test 3", &ok))
-
-	l.Debugf("test %d", 0)
-	l.Debugln("test", 0)
-	l.Infof("test %d", 1)
-	l.Infoln("test", 1)
-	l.Warnf("test %d", 2)
-	l.Warnln("test", 2)
-	l.Okf("test %d", 3)
-	l.Okln("test", 3)
-
-	if debug != 2 {
-		t.Errorf("Debug handler called %d != 2 times", debug)
-	}
-	if info != 2 {
-		t.Errorf("Info handler called %d != 2 times", info)
-	}
-	if warn != 2 {
-		t.Errorf("Warn handler called %d != 2 times", warn)
-	}
-	if ok != 2 {
-		t.Errorf("Ok handler called %d != 2 times", ok)
-	}
-}
-
-func checkFunc(t *testing.T, expectl LogLevel, expectmsg string, counter *int) func(LogLevel, string) {
-	return func(l LogLevel, msg string) {
-		*counter++
-		if l != expectl {
-			t.Errorf("Incorrect message level %d != %d", l, expectl)
-		}
-		if !strings.HasSuffix(msg, expectmsg) {
-			t.Errorf("%q does not end with %q", msg, expectmsg)
-		}
-	}
-}

Godeps/_workspace/src/github.com/calmh/xdr/.gitignore

@@ -1 +0,0 @@
-coverage.out

Godeps/_workspace/src/github.com/calmh/xdr/.travis.yml

@@ -1,19 +0,0 @@
-language: go
-go:
-- tip
-
-install:
-- export PATH=$PATH:$HOME/gopath/bin
-- go get code.google.com/p/go.tools/cmd/cover
-- go get github.com/mattn/goveralls
-
-script:
-- ./generate.sh
-- go test -coverprofile=coverage.out
-
-after_success:
-- goveralls -coverprofile=coverage.out -service=travis-ci -package=calmh/xdr -repotoken="$COVERALLS_TOKEN"
-
-env:
-  global:
-    secure: SmgnrGfp2zLrA44ChRMpjPeujubt9veZ8Fx/OseMWECmacyV5N/TuDhzIbwo6QwV4xB0sBacoPzvxQbJRVjNKsPiSu72UbcQmQ7flN4Tf7nW09tSh1iW8NgrpBCq/3UYLoBu2iPBEBKm93IK0aGNAKs6oEkB0fU27iTVBwiTXOY=

Godeps/_workspace/src/github.com/calmh/xdr/bench_test.go

@@ -1,117 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-package xdr_test
-
-import (
-	"io"
-	"io/ioutil"
-	"testing"
-
-	"github.com/calmh/xdr"
-)
-
-type XDRBenchStruct struct {
-	I1  uint64
-	I2  uint32
-	I3  uint16
-	I4  uint8
-	Bs0 []byte // max:128
-	Bs1 []byte
-	S0  string // max:128
-	S1  string
-}
-
-var res []byte // no to be optimized away
-var s = XDRBenchStruct{
-	I1:  42,
-	I2:  43,
-	I3:  44,
-	I4:  45,
-	Bs0: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18},
-	Bs1: []byte{11, 12, 13, 14, 15, 16, 17, 18, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
-	S0:  "Hello World! String one.",
-	S1:  "Hello World! String two.",
-}
-var e []byte
-
-func init() {
-	e, _ = s.MarshalXDR()
-}
-
-func BenchmarkThisMarshal(b *testing.B) {
-	for i := 0; i < b.N; i++ {
-		res, _ = s.MarshalXDR()
-	}
-}
-
-func BenchmarkThisUnmarshal(b *testing.B) {
-	var t XDRBenchStruct
-	for i := 0; i < b.N; i++ {
-		err := t.UnmarshalXDR(e)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func BenchmarkThisEncode(b *testing.B) {
-	for i := 0; i < b.N; i++ {
-		_, err := s.EncodeXDR(ioutil.Discard)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func BenchmarkThisEncoder(b *testing.B) {
-	w := xdr.NewWriter(ioutil.Discard)
-	for i := 0; i < b.N; i++ {
-		_, err := s.EncodeXDRInto(w)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-type repeatReader struct {
-	data []byte
-}
-
-func (r *repeatReader) Read(bs []byte) (n int, err error) {
-	if len(bs) > len(r.data) {
-		err = io.EOF
-	}
-	n = copy(bs, r.data)
-	r.data = r.data[n:]
-	return n, err
-}
-
-func (r *repeatReader) Reset(bs []byte) {
-	r.data = bs
-}
-
-func BenchmarkThisDecode(b *testing.B) {
-	rr := &repeatReader{e}
-	var t XDRBenchStruct
-	for i := 0; i < b.N; i++ {
-		err := t.DecodeXDR(rr)
-		if err != nil {
-			b.Fatal(err)
-		}
-		rr.Reset(e)
-	}
-}
-
-func BenchmarkThisDecoder(b *testing.B) {
-	rr := &repeatReader{e}
-	r := xdr.NewReader(rr)
-	var t XDRBenchStruct
-	for i := 0; i < b.N; i++ {
-		err := t.DecodeXDRFrom(r)
-		if err != nil {
-			b.Fatal(err)
-		}
-		rr.Reset(e)
-	}
-}

Godeps/_workspace/src/github.com/calmh/xdr/bench_xdr_test.go

@@ -1,201 +0,0 @@
-// ************************************************************
-// This file is automatically generated by genxdr. Do not edit.
-// ************************************************************
-
-package xdr_test
-
-import (
-	"bytes"
-	"io"
-
-	"github.com/calmh/xdr"
-)
-
-/*
-
-XDRBenchStruct Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                                                               |
-+                         I1 (64 bits)                          +
-|                                                               |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                              I2                               |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|            0x0000             |              I3               |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                        uint8 Structure                        \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                         Length of Bs0                         |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                     Bs0 (variable length)                     \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                         Length of Bs1                         |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                     Bs1 (variable length)                     \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                         Length of S0                          |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                     S0 (variable length)                      \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                         Length of S1                          |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                     S1 (variable length)                      \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct XDRBenchStruct {
-	unsigned hyper I1;
-	unsigned int I2;
-	unsigned int I3;
-	uint8 I4;
-	opaque Bs0<128>;
-	opaque Bs1<>;
-	string S0<128>;
-	string S1<>;
-}
-
-*/
-
-func (o XDRBenchStruct) EncodeXDR(w io.Writer) (int, error) {
-	var xw = xdr.NewWriter(w)
-	return o.EncodeXDRInto(xw)
-}
-
-func (o XDRBenchStruct) MarshalXDR() ([]byte, error) {
-	return o.AppendXDR(make([]byte, 0, 128))
-}
-
-func (o XDRBenchStruct) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o XDRBenchStruct) AppendXDR(bs []byte) ([]byte, error) {
-	var aw = xdr.AppendWriter(bs)
-	var xw = xdr.NewWriter(&aw)
-	_, err := o.EncodeXDRInto(xw)
-	return []byte(aw), err
-}
-
-func (o XDRBenchStruct) EncodeXDRInto(xw *xdr.Writer) (int, error) {
-	xw.WriteUint64(o.I1)
-	xw.WriteUint32(o.I2)
-	xw.WriteUint16(o.I3)
-	xw.WriteUint8(o.I4)
-	if l := len(o.Bs0); l > 128 {
-		return xw.Tot(), xdr.ElementSizeExceeded("Bs0", l, 128)
-	}
-	xw.WriteBytes(o.Bs0)
-	xw.WriteBytes(o.Bs1)
-	if l := len(o.S0); l > 128 {
-		return xw.Tot(), xdr.ElementSizeExceeded("S0", l, 128)
-	}
-	xw.WriteString(o.S0)
-	xw.WriteString(o.S1)
-	return xw.Tot(), xw.Error()
-}
-
-func (o *XDRBenchStruct) DecodeXDR(r io.Reader) error {
-	xr := xdr.NewReader(r)
-	return o.DecodeXDRFrom(xr)
-}
-
-func (o *XDRBenchStruct) UnmarshalXDR(bs []byte) error {
-	var br = bytes.NewReader(bs)
-	var xr = xdr.NewReader(br)
-	return o.DecodeXDRFrom(xr)
-}
-
-func (o *XDRBenchStruct) DecodeXDRFrom(xr *xdr.Reader) error {
-	o.I1 = xr.ReadUint64()
-	o.I2 = xr.ReadUint32()
-	o.I3 = xr.ReadUint16()
-	o.I4 = xr.ReadUint8()
-	o.Bs0 = xr.ReadBytesMax(128)
-	o.Bs1 = xr.ReadBytes()
-	o.S0 = xr.ReadStringMax(128)
-	o.S1 = xr.ReadString()
-	return xr.Error()
-}
-
-/*
-
-repeatReader Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                        Length of data                         |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                    data (variable length)                     \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct repeatReader {
-	opaque data<>;
-}
-
-*/
-
-func (o repeatReader) EncodeXDR(w io.Writer) (int, error) {
-	var xw = xdr.NewWriter(w)
-	return o.EncodeXDRInto(xw)
-}
-
-func (o repeatReader) MarshalXDR() ([]byte, error) {
-	return o.AppendXDR(make([]byte, 0, 128))
-}
-
-func (o repeatReader) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o repeatReader) AppendXDR(bs []byte) ([]byte, error) {
-	var aw = xdr.AppendWriter(bs)
-	var xw = xdr.NewWriter(&aw)
-	_, err := o.EncodeXDRInto(xw)
-	return []byte(aw), err
-}
-
-func (o repeatReader) EncodeXDRInto(xw *xdr.Writer) (int, error) {
-	xw.WriteBytes(o.data)
-	return xw.Tot(), xw.Error()
-}
-
-func (o *repeatReader) DecodeXDR(r io.Reader) error {
-	xr := xdr.NewReader(r)
-	return o.DecodeXDRFrom(xr)
-}
-
-func (o *repeatReader) UnmarshalXDR(bs []byte) error {
-	var br = bytes.NewReader(bs)
-	var xr = xdr.NewReader(br)
-	return o.DecodeXDRFrom(xr)
-}
-
-func (o *repeatReader) DecodeXDRFrom(xr *xdr.Reader) error {
-	o.data = xr.ReadBytes()
-	return xr.Error()
-}

Godeps/_workspace/src/github.com/calmh/xdr/debug.go

@@ -1,16 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-package xdr
-
-import (
-	"log"
-	"os"
-)
-
-var (
-	debug = len(os.Getenv("XDRTRACE")) > 0
-	dl    = log.New(os.Stdout, "xdr: ", log.Lshortfile|log.Ltime|log.Lmicroseconds)
-)
-
-const maxDebugBytes = 32

Godeps/_workspace/src/github.com/calmh/xdr/encdec_test.go

@@ -1,79 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-package xdr_test
-
-import (
-	"bytes"
-	"math/rand"
-	"reflect"
-	"testing"
-	"testing/quick"
-
-	"github.com/calmh/xdr"
-)
-
-// Contains all supported types
-type TestStruct struct {
-	I    int
-	I8   int8
-	UI8  uint8
-	I16  int16
-	UI16 uint16
-	I32  int32
-	UI32 uint32
-	I64  int64
-	UI64 uint64
-	BS   []byte // max:1024
-	S    string // max:1024
-	C    Opaque
-	SS   []string // max:1024
-}
-
-type Opaque [32]byte
-
-func (u *Opaque) EncodeXDRInto(w *xdr.Writer) (int, error) {
-	return w.WriteRaw(u[:])
-}
-
-func (u *Opaque) DecodeXDRFrom(r *xdr.Reader) (int, error) {
-	return r.ReadRaw(u[:])
-}
-
-func (Opaque) Generate(rand *rand.Rand, size int) reflect.Value {
-	var u Opaque
-	for i := range u[:] {
-		u[i] = byte(rand.Int())
-	}
-	return reflect.ValueOf(u)
-}
-
-func TestEncDec(t *testing.T) {
-	fn := func(t0 TestStruct) bool {
-		bs, err := t0.MarshalXDR()
-		if err != nil {
-			t.Fatal(err)
-		}
-		var t1 TestStruct
-		err = t1.UnmarshalXDR(bs)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		// Not comparing with DeepEqual since we'll unmarshal nil slices as empty
-		if t0.I != t1.I ||
-			t0.I16 != t1.I16 || t0.UI16 != t1.UI16 ||
-			t0.I32 != t1.I32 || t0.UI32 != t1.UI32 ||
-			t0.I64 != t1.I64 || t0.UI64 != t1.UI64 ||
-			bytes.Compare(t0.BS, t1.BS) != 0 ||
-			t0.S != t1.S || t0.C != t1.C {
-			t.Logf("%#v", t0)
-			t.Logf("%#v", t1)
-			return false
-		}
-		return true
-	}
-	if err := quick.Check(fn, nil); err != nil {
-		t.Error(err)
-	}
-}

Godeps/_workspace/src/github.com/calmh/xdr/encdec_xdr_test.go

@@ -1,185 +0,0 @@
-// ************************************************************
-// This file is automatically generated by genxdr. Do not edit.
-// ************************************************************
-
-package xdr_test
-
-import (
-	"bytes"
-	"io"
-
-	"github.com/calmh/xdr"
-)
-
-/*
-
-TestStruct Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                         int Structure                         \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                        int8 Structure                         \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                        uint8 Structure                        \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|            0x0000             |              I16              |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|            0x0000             |             UI16              |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                              I32                              |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             UI32                              |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                                                               |
-+                         I64 (64 bits)                         +
-|                                                               |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                                                               |
-+                        UI64 (64 bits)                         +
-|                                                               |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                         Length of BS                          |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                     BS (variable length)                      \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                          Length of S                          |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                      S (variable length)                      \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                       Opaque Structure                        \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                         Number of SS                          |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                         Length of SS                          |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                     SS (variable length)                      \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct TestStruct {
-	int I;
-	int8 I8;
-	uint8 UI8;
-	int I16;
-	unsigned int UI16;
-	int I32;
-	unsigned int UI32;
-	hyper I64;
-	unsigned hyper UI64;
-	opaque BS<1024>;
-	string S<1024>;
-	Opaque C;
-	string SS<1024>;
-}
-
-*/
-
-func (o TestStruct) EncodeXDR(w io.Writer) (int, error) {
-	var xw = xdr.NewWriter(w)
-	return o.EncodeXDRInto(xw)
-}
-
-func (o TestStruct) MarshalXDR() ([]byte, error) {
-	return o.AppendXDR(make([]byte, 0, 128))
-}
-
-func (o TestStruct) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o TestStruct) AppendXDR(bs []byte) ([]byte, error) {
-	var aw = xdr.AppendWriter(bs)
-	var xw = xdr.NewWriter(&aw)
-	_, err := o.EncodeXDRInto(xw)
-	return []byte(aw), err
-}
-
-func (o TestStruct) EncodeXDRInto(xw *xdr.Writer) (int, error) {
-	xw.WriteUint64(uint64(o.I))
-	xw.WriteUint8(uint8(o.I8))
-	xw.WriteUint8(o.UI8)
-	xw.WriteUint16(uint16(o.I16))
-	xw.WriteUint16(o.UI16)
-	xw.WriteUint32(uint32(o.I32))
-	xw.WriteUint32(o.UI32)
-	xw.WriteUint64(uint64(o.I64))
-	xw.WriteUint64(o.UI64)
-	if l := len(o.BS); l > 1024 {
-		return xw.Tot(), xdr.ElementSizeExceeded("BS", l, 1024)
-	}
-	xw.WriteBytes(o.BS)
-	if l := len(o.S); l > 1024 {
-		return xw.Tot(), xdr.ElementSizeExceeded("S", l, 1024)
-	}
-	xw.WriteString(o.S)
-	_, err := o.C.EncodeXDRInto(xw)
-	if err != nil {
-		return xw.Tot(), err
-	}
-	if l := len(o.SS); l > 1024 {
-		return xw.Tot(), xdr.ElementSizeExceeded("SS", l, 1024)
-	}
-	xw.WriteUint32(uint32(len(o.SS)))
-	for i := range o.SS {
-		xw.WriteString(o.SS[i])
-	}
-	return xw.Tot(), xw.Error()
-}
-
-func (o *TestStruct) DecodeXDR(r io.Reader) error {
-	xr := xdr.NewReader(r)
-	return o.DecodeXDRFrom(xr)
-}
-
-func (o *TestStruct) UnmarshalXDR(bs []byte) error {
-	var br = bytes.NewReader(bs)
-	var xr = xdr.NewReader(br)
-	return o.DecodeXDRFrom(xr)
-}
-
-func (o *TestStruct) DecodeXDRFrom(xr *xdr.Reader) error {
-	o.I = int(xr.ReadUint64())
-	o.I8 = int8(xr.ReadUint8())
-	o.UI8 = xr.ReadUint8()
-	o.I16 = int16(xr.ReadUint16())
-	o.UI16 = xr.ReadUint16()
-	o.I32 = int32(xr.ReadUint32())
-	o.UI32 = xr.ReadUint32()
-	o.I64 = int64(xr.ReadUint64())
-	o.UI64 = xr.ReadUint64()
-	o.BS = xr.ReadBytesMax(1024)
-	o.S = xr.ReadStringMax(1024)
-	(&o.C).DecodeXDRFrom(xr)
-	_SSSize := int(xr.ReadUint32())
-	if _SSSize < 0 {
-		return xdr.ElementSizeExceeded("SS", _SSSize, 1024)
-	}
-	if _SSSize > 1024 {
-		return xdr.ElementSizeExceeded("SS", _SSSize, 1024)
-	}
-	o.SS = make([]string, _SSSize)
-	for i := range o.SS {
-		o.SS[i] = xr.ReadString()
-	}
-	return xr.Error()
-}

Godeps/_workspace/src/github.com/calmh/xdr/pad_ipdr.go

@@ -1,10 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-// +build ipdr
-
-package xdr
-
-func pad(l int) int {
-	return 0
-}

Godeps/_workspace/src/github.com/calmh/xdr/pad_xdr.go

@@ -1,14 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-// +build !ipdr
-
-package xdr
-
-func pad(l int) int {
-	d := l % 4
-	if d == 0 {
-		return 0
-	}
-	return 4 - d
-}

Godeps/_workspace/src/github.com/calmh/xdr/reader.go

@@ -1,171 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
-
-package xdr
-
-import (
-	"fmt"
-	"io"
-	"reflect"
-	"unsafe"
-)
-
-type Reader struct {
-	r   io.Reader
-	err error
-	b   [8]byte
-}
-
-func NewReader(r io.Reader) *Reader {
-	return &Reader{
-		r: r,
-	}
-}
-
-func (r *Reader) ReadRaw(bs []byte) (int, error) {
-	if r.err != nil {
-		return 0, r.err
-	}
-
-	var n int
-	n, r.err = io.ReadFull(r.r, bs)
-	return n, r.err
-}
-
-func (r *Reader) ReadString() string {
-	return r.ReadStringMax(0)
-}
-
-func (r *Reader) ReadStringMax(max int) string {
-	buf := r.ReadBytesMaxInto(max, nil)
-	bh := (*reflect.SliceHeader)(unsafe.Pointer(&buf))
-	sh := reflect.StringHeader{
-		Data: bh.Data,
-		Len:  bh.Len,
-	}
-	return *((*string)(unsafe.Pointer(&sh)))
-}
-
-func (r *Reader) ReadBytes() []byte {
-	return r.ReadBytesInto(nil)
-}
-
-func (r *Reader) ReadBytesMax(max int) []byte {
-	return r.ReadBytesMaxInto(max, nil)
-}
-
-func (r *Reader) ReadBytesInto(dst []byte) []byte {
-	return r.ReadBytesMaxInto(0, dst)
-}
-
-func (r *Reader) ReadBytesMaxInto(max int, dst []byte) []byte {
-	if r.err != nil {
-		return nil
-	}
-
-	l := int(r.ReadUint32())
-	if r.err != nil {
-		return nil
-	}
-	if l < 0 || max > 0 && l > max {
-		// l may be negative on 32 bit builds
-		r.err = ElementSizeExceeded("bytes field", l, max)
-		return nil
-	}
-
-	if fullLen := l + pad(l); fullLen > len(dst) {
-		dst = make([]byte, fullLen)
-	} else {
-		dst = dst[:fullLen]
-	}
-
-	var n int
-	n, r.err = io.ReadFull(r.r, dst)
-	if r.err != nil {
-		if debug {
-			dl.Printf("rd bytes (%d): %v", len(dst), r.err)
-		}
-		return nil
-	}
-
-	if debug {
-		if n > maxDebugBytes {
-			dl.Printf("rd bytes (%d): %x...", len(dst), dst[:maxDebugBytes])
-		} else {
-			dl.Printf("rd bytes (%d): %x", len(dst), dst)
-		}
-	}
-	return dst[:l]
-}
-
-func (r *Reader) ReadBool() bool {
-	return r.ReadUint8() != 0
-}
-
-func (r *Reader) ReadUint32() uint32 {
-	if r.err != nil {
-		return 0
-	}
-
-	_, r.err = io.ReadFull(r.r, r.b[:4])
-	if r.err != nil {
-		if debug {
-			dl.Printf("rd uint32: %v", r.err)
-		}
-		return 0
-	}
-
-	v := uint32(r.b[3]) | uint32(r.b[2])<<8 | uint32(r.b[1])<<16 | uint32(r.b[0])<<24
-
-	if debug {
-		dl.Printf("rd uint32=%d (0x%08x)", v, v)
-	}
-	return v
-}
-
-func (r *Reader) ReadUint64() uint64 {
-	if r.err != nil {
-		return 0
-	}
-
-	_, r.err = io.ReadFull(r.r, r.b[:8])
-	if r.err != nil {
-		if debug {
-			dl.Printf("rd uint64: %v", r.err)
-		}
-		return 0
-	}
-
-	v := uint64(r.b[7]) | uint64(r.b[6])<<8 | uint64(r.b[5])<<16 | uint64(r.b[4])<<24 |
-		uint64(r.b[3])<<32 | uint64(r.b[2])<<40 | uint64(r.b[1])<<48 | uint64(r.b[0])<<56
-
-	if debug {
-		dl.Printf("rd uint64=%d (0x%016x)", v, v)
-	}
-	return v
-}
-
-type XDRError struct {
-	op  string
-	err error
-}
-
-func (e XDRError) Error() string {
-	return "xdr " + e.op + ": " + e.err.Error()
-}
-
-func (e XDRError) IsEOF() bool {
-	return e.err == io.EOF
-}
-
-func (r *Reader) Error() error {
-	if r.err == nil {
-		return nil
-	}
-	return XDRError{"read", r.err}
-}
-
-func ElementSizeExceeded(field string, size, limit int) error {
-	return fmt.Errorf("%s exceeds size limit; %d > %d", field, size, limit)
-}

Godeps/_workspace/src/github.com/calmh/xdr/reader_ipdr.go

@@ -1,49 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
-
-// +build ipdr
-
-package xdr
-
-import "io"
-
-func (r *Reader) ReadUint8() uint8 {
-	if r.err != nil {
-		return 0
-	}
-
-	_, r.err = io.ReadFull(r.r, r.b[:1])
-	if r.err != nil {
-		if debug {
-			dl.Printf("rd uint8: %v", r.err)
-		}
-		return 0
-	}
-
-	if debug {
-		dl.Printf("rd uint8=%d (0x%02x)", r.b[0], r.b[0])
-	}
-	return r.b[0]
-}
-
-func (r *Reader) ReadUint16() uint16 {
-	if r.err != nil {
-		return 0
-	}
-
-	_, r.err = io.ReadFull(r.r, r.b[:2])
-	if r.err != nil {
-		if debug {
-			dl.Printf("rd uint16: %v", r.err)
-		}
-		return 0
-	}
-
-	v := uint16(r.b[1]) | uint16(r.b[0])<<8
-
-	if debug {
-		dl.Printf("rd uint16=%d (0x%04x)", v, v)
-	}
-	return v
-}

Godeps/_workspace/src/github.com/calmh/xdr/reader_xdr.go

@@ -1,15 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
-
-// +build !ipdr
-
-package xdr
-
-func (r *Reader) ReadUint8() uint8 {
-	return uint8(r.ReadUint32())
-}
-
-func (r *Reader) ReadUint16() uint16 {
-	return uint16(r.ReadUint32())
-}

Godeps/_workspace/src/github.com/calmh/xdr/refl_test.go

@@ -1,44 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-// +build refl
-
-package xdr_test
-
-import (
-	"bytes"
-	"testing"
-
-	refl "github.com/davecgh/go-xdr/xdr"
-)
-
-func TestCompareMarshals(t *testing.T) {
-	e0 := s.MarshalXDR()
-	e1, err := refl.Marshal(s)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if bytes.Compare(e0, e1) != 0 {
-		t.Fatalf("Encoding mismatch;\n\t%x (this)\n\t%x (refl)", e0, e1)
-	}
-}
-
-func BenchmarkReflMarshal(b *testing.B) {
-	var err error
-	for i := 0; i < b.N; i++ {
-		res, err = refl.Marshal(s)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func BenchmarkReflUnmarshal(b *testing.B) {
-	var t XDRBenchStruct
-	for i := 0; i < b.N; i++ {
-		_, err := refl.Unmarshal(e, &t)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}

Godeps/_workspace/src/github.com/calmh/xdr/writer.go

@@ -1,146 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-package xdr
-
-import (
-	"io"
-	"reflect"
-	"unsafe"
-)
-
-var padBytes = []byte{0, 0, 0}
-
-type Writer struct {
-	w   io.Writer
-	tot int
-	err error
-	b   [8]byte
-}
-
-type AppendWriter []byte
-
-func (w *AppendWriter) Write(bs []byte) (int, error) {
-	*w = append(*w, bs...)
-	return len(bs), nil
-}
-
-func NewWriter(w io.Writer) *Writer {
-	return &Writer{
-		w: w,
-	}
-}
-
-func (w *Writer) WriteRaw(bs []byte) (int, error) {
-	if w.err != nil {
-		return 0, w.err
-	}
-
-	var n int
-	n, w.err = w.w.Write(bs)
-	return n, w.err
-}
-
-func (w *Writer) WriteString(s string) (int, error) {
-	sh := *((*reflect.StringHeader)(unsafe.Pointer(&s)))
-	bh := reflect.SliceHeader{
-		Data: sh.Data,
-		Len:  sh.Len,
-		Cap:  sh.Len,
-	}
-	return w.WriteBytes(*(*[]byte)(unsafe.Pointer(&bh)))
-}
-
-func (w *Writer) WriteBytes(bs []byte) (int, error) {
-	if w.err != nil {
-		return 0, w.err
-	}
-
-	w.WriteUint32(uint32(len(bs)))
-	if w.err != nil {
-		return 0, w.err
-	}
-
-	if debug {
-		if len(bs) > maxDebugBytes {
-			dl.Printf("wr bytes (%d): %x...", len(bs), bs[:maxDebugBytes])
-		} else {
-			dl.Printf("wr bytes (%d): %x", len(bs), bs)
-		}
-	}
-
-	var l, n int
-	n, w.err = w.w.Write(bs)
-	l += n
-
-	if p := pad(len(bs)); w.err == nil && p > 0 {
-		n, w.err = w.w.Write(padBytes[:p])
-		l += n
-	}
-
-	w.tot += l
-	return l, w.err
-}
-
-func (w *Writer) WriteBool(v bool) (int, error) {
-	if v {
-		return w.WriteUint8(1)
-	} else {
-		return w.WriteUint8(0)
-	}
-}
-
-func (w *Writer) WriteUint32(v uint32) (int, error) {
-	if w.err != nil {
-		return 0, w.err
-	}
-
-	if debug {
-		dl.Printf("wr uint32=%d", v)
-	}
-
-	w.b[0] = byte(v >> 24)
-	w.b[1] = byte(v >> 16)
-	w.b[2] = byte(v >> 8)
-	w.b[3] = byte(v)
-
-	var l int
-	l, w.err = w.w.Write(w.b[:4])
-	w.tot += l
-	return l, w.err
-}
-
-func (w *Writer) WriteUint64(v uint64) (int, error) {
-	if w.err != nil {
-		return 0, w.err
-	}
-
-	if debug {
-		dl.Printf("wr uint64=%d", v)
-	}
-
-	w.b[0] = byte(v >> 56)
-	w.b[1] = byte(v >> 48)
-	w.b[2] = byte(v >> 40)
-	w.b[3] = byte(v >> 32)
-	w.b[4] = byte(v >> 24)
-	w.b[5] = byte(v >> 16)
-	w.b[6] = byte(v >> 8)
-	w.b[7] = byte(v)
-
-	var l int
-	l, w.err = w.w.Write(w.b[:8])
-	w.tot += l
-	return l, w.err
-}
-
-func (w *Writer) Tot() int {
-	return w.tot
-}
-
-func (w *Writer) Error() error {
-	if w.err == nil {
-		return nil
-	}
-	return XDRError{"write", w.err}
-}

Godeps/_workspace/src/github.com/calmh/xdr/writer_ipdr.go

@@ -1,41 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-// +build ipdr
-
-package xdr
-
-func (w *Writer) WriteUint8(v uint8) (int, error) {
-	if w.err != nil {
-		return 0, w.err
-	}
-
-	if debug {
-		dl.Printf("wr uint8=%d", v)
-	}
-
-	w.b[0] = byte(v)
-
-	var l int
-	l, w.err = w.w.Write(w.b[:1])
-	w.tot += l
-	return l, w.err
-}
-
-func (w *Writer) WriteUint16(v uint16) (int, error) {
-	if w.err != nil {
-		return 0, w.err
-	}
-
-	if debug {
-		dl.Printf("wr uint8=%d", v)
-	}
-
-	w.b[0] = byte(v >> 8)
-	w.b[1] = byte(v)
-
-	var l int
-	l, w.err = w.w.Write(w.b[:2])
-	w.tot += l
-	return l, w.err
-}

Godeps/_workspace/src/github.com/calmh/xdr/writer_xdr.go

@@ -1,14 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-// +build !ipdr
-
-package xdr
-
-func (w *Writer) WriteUint8(v uint8) (int, error) {
-	return w.WriteUint32(uint32(v))
-}
-
-func (w *Writer) WriteUint16(v uint16) (int, error) {
-	return w.WriteUint32(uint32(v))
-}

Godeps/_workspace/src/github.com/calmh/xdr/xdr_test.go

@@ -1,93 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-package xdr
-
-import (
-	"bytes"
-	"strings"
-	"testing"
-	"testing/quick"
-)
-
-func TestBytesNil(t *testing.T) {
-	fn := func(bs []byte) bool {
-		var b = new(bytes.Buffer)
-		var w = NewWriter(b)
-		var r = NewReader(b)
-		w.WriteBytes(bs)
-		w.WriteBytes(bs)
-		r.ReadBytes()
-		res := r.ReadBytes()
-		return bytes.Compare(bs, res) == 0
-	}
-	if err := quick.Check(fn, nil); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestBytesGiven(t *testing.T) {
-	fn := func(bs []byte) bool {
-		var b = new(bytes.Buffer)
-		var w = NewWriter(b)
-		var r = NewReader(b)
-		w.WriteBytes(bs)
-		w.WriteBytes(bs)
-		res := make([]byte, 12)
-		res = r.ReadBytesInto(res)
-		res = r.ReadBytesInto(res)
-		return bytes.Compare(bs, res) == 0
-	}
-	if err := quick.Check(fn, nil); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestReadBytesMaxInto(t *testing.T) {
-	var max = 64
-	for tot := 32; tot < 128; tot++ {
-		for diff := -32; diff <= 32; diff++ {
-			var b = new(bytes.Buffer)
-			var r = NewReader(b)
-			var w = NewWriter(b)
-
-			var toWrite = make([]byte, tot)
-			w.WriteBytes(toWrite)
-
-			var buf = make([]byte, tot+diff)
-			var bs = r.ReadBytesMaxInto(max, buf)
-
-			if tot <= max {
-				if read := len(bs); read != tot {
-					t.Errorf("Incorrect read bytes, wrote=%d, buf=%d, max=%d, read=%d", tot, tot+diff, max, read)
-				}
-			} else if !strings.Contains(r.err.Error(), "exceeds size") {
-				t.Errorf("Unexpected non-ErrElementSizeExceeded error for wrote=%d, max=%d: %v", tot, max, r.err)
-			}
-		}
-	}
-}
-
-func TestReadStringMax(t *testing.T) {
-	for tot := 42; tot < 72; tot++ {
-		for max := 0; max < 128; max++ {
-			var b = new(bytes.Buffer)
-			var r = NewReader(b)
-			var w = NewWriter(b)
-
-			var toWrite = make([]byte, tot)
-			w.WriteBytes(toWrite)
-
-			var str = r.ReadStringMax(max)
-			var read = len(str)
-
-			if max == 0 || tot <= max {
-				if read != tot {
-					t.Errorf("Incorrect read bytes, wrote=%d, max=%d, read=%d", tot, max, read)
-				}
-			} else if !strings.Contains(r.err.Error(), "exceeds size") {
-				t.Errorf("Unexpected non-ErrElementSizeExceeded error for wrote=%d, max=%d, read=%d: %v", tot, max, read, r.err)
-			}
-		}
-	}
-}

Godeps/_workspace/src/github.com/syncthing/protocol/debug.go

@@ -1,15 +0,0 @@
-// Copyright (C) 2014 The Protocol Authors.
-
-package protocol
-
-import (
-	"os"
-	"strings"
-
-	"github.com/calmh/logger"
-)
-
-var (
-	debug = strings.Contains(os.Getenv("STTRACE"), "protocol") || os.Getenv("STTRACE") == "all"
-	l     = logger.DefaultLogger
-)

Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_unix.go

@@ -1,31 +0,0 @@
-// Copyright (C) 2014 The Protocol Authors.
-
-// +build !windows,!darwin
-
-package protocol
-
-// Normal Unixes uses NFC and slashes, which is the wire format.
-
-type nativeModel struct {
-	next Model
-}
-
-func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
-	m.next.Index(deviceID, folder, files, flags, options)
-}
-
-func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
-	m.next.IndexUpdate(deviceID, folder, files, flags, options)
-}
-
-func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
-	return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options)
-}
-
-func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
-	m.next.ClusterConfig(deviceID, config)
-}
-
-func (m nativeModel) Close(deviceID DeviceID, err error) {
-	m.next.Close(deviceID, err)
-}

Godeps/_workspace/src/github.com/syncthing/protocol/protocol_test.go

@@ -1,382 +0,0 @@
-// Copyright (C) 2014 The Protocol Authors.
-
-package protocol
-
-import (
-	"bytes"
-	"encoding/hex"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"reflect"
-	"strings"
-	"testing"
-	"testing/quick"
-
-	"github.com/calmh/xdr"
-)
-
-var (
-	c0ID = NewDeviceID([]byte{1})
-	c1ID = NewDeviceID([]byte{2})
-)
-
-func TestHeaderFunctions(t *testing.T) {
-	f := func(ver, id, typ int) bool {
-		ver = int(uint(ver) % 16)
-		id = int(uint(id) % 4096)
-		typ = int(uint(typ) % 256)
-		h0 := header{version: ver, msgID: id, msgType: typ}
-		h1 := decodeHeader(encodeHeader(h0))
-		return h0 == h1
-	}
-	if err := quick.Check(f, nil); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestHeaderLayout(t *testing.T) {
-	var e, a uint32
-
-	// Version are the first four bits
-	e = 0xf0000000
-	a = encodeHeader(header{version: 0xf})
-	if a != e {
-		t.Errorf("Header layout incorrect; %08x != %08x", a, e)
-	}
-
-	// Message ID are the following 12 bits
-	e = 0x0fff0000
-	a = encodeHeader(header{msgID: 0xfff})
-	if a != e {
-		t.Errorf("Header layout incorrect; %08x != %08x", a, e)
-	}
-
-	// Type are the last 8 bits before reserved
-	e = 0x0000ff00
-	a = encodeHeader(header{msgType: 0xff})
-	if a != e {
-		t.Errorf("Header layout incorrect; %08x != %08x", a, e)
-	}
-}
-
-func TestPing(t *testing.T) {
-	ar, aw := io.Pipe()
-	br, bw := io.Pipe()
-
-	c0 := NewConnection(c0ID, ar, bw, nil, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
-	c1 := NewConnection(c1ID, br, aw, nil, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
-
-	if ok := c0.ping(); !ok {
-		t.Error("c0 ping failed")
-	}
-	if ok := c1.ping(); !ok {
-		t.Error("c1 ping failed")
-	}
-}
-
-func TestPingErr(t *testing.T) {
-	e := errors.New("something broke")
-
-	for i := 0; i < 16; i++ {
-		for j := 0; j < 16; j++ {
-			m0 := newTestModel()
-			m1 := newTestModel()
-
-			ar, aw := io.Pipe()
-			br, bw := io.Pipe()
-			eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e}
-			ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e}
-
-			c0 := NewConnection(c0ID, ar, ebw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
-			NewConnection(c1ID, br, eaw, m1, "name", CompressAlways)
-
-			res := c0.ping()
-			if (i < 8 || j < 8) && res {
-				t.Errorf("Unexpected ping success; i=%d, j=%d", i, j)
-			} else if (i >= 12 && j >= 12) && !res {
-				t.Errorf("Unexpected ping fail; i=%d, j=%d", i, j)
-			}
-		}
-	}
-}
-
-// func TestRequestResponseErr(t *testing.T) {
-// 	e := errors.New("something broke")
-
-// 	var pass bool
-// 	for i := 0; i < 48; i++ {
-// 		for j := 0; j < 38; j++ {
-// 			m0 := newTestModel()
-// 			m0.data = []byte("response data")
-// 			m1 := newTestModel()
-
-// 			ar, aw := io.Pipe()
-// 			br, bw := io.Pipe()
-// 			eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e}
-// 			ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e}
-
-// 			NewConnection(c0ID, ar, ebw, m0, nil)
-// 			c1 := NewConnection(c1ID, br, eaw, m1, nil).(wireFormatConnection).next.(*rawConnection)
-
-// 			d, err := c1.Request("default", "tn", 1234, 5678)
-// 			if err == e || err == ErrClosed {
-// 				t.Logf("Error at %d+%d bytes", i, j)
-// 				if !m1.isClosed() {
-// 					t.Fatal("c1 not closed")
-// 				}
-// 				if !m0.isClosed() {
-// 					t.Fatal("c0 not closed")
-// 				}
-// 				continue
-// 			}
-// 			if err != nil {
-// 				t.Fatal(err)
-// 			}
-// 			if string(d) != "response data" {
-// 				t.Fatalf("Incorrect response data %q", string(d))
-// 			}
-// 			if m0.folder != "default" {
-// 				t.Fatalf("Incorrect folder %q", m0.folder)
-// 			}
-// 			if m0.name != "tn" {
-// 				t.Fatalf("Incorrect name %q", m0.name)
-// 			}
-// 			if m0.offset != 1234 {
-// 				t.Fatalf("Incorrect offset %d", m0.offset)
-// 			}
-// 			if m0.size != 5678 {
-// 				t.Fatalf("Incorrect size %d", m0.size)
-// 			}
-// 			t.Logf("Pass at %d+%d bytes", i, j)
-// 			pass = true
-// 		}
-// 	}
-// 	if !pass {
-// 		t.Fatal("Never passed")
-// 	}
-// }
-
-func TestVersionErr(t *testing.T) {
-	m0 := newTestModel()
-	m1 := newTestModel()
-
-	ar, aw := io.Pipe()
-	br, bw := io.Pipe()
-
-	c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
-	NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
-
-	w := xdr.NewWriter(c0.cw)
-	w.WriteUint32(encodeHeader(header{
-		version: 2,
-		msgID:   0,
-		msgType: 0,
-	}))
-	w.WriteUint32(0) // Avoids reader closing due to EOF
-
-	if !m1.isClosed() {
-		t.Error("Connection should close due to unknown version")
-	}
-}
-
-func TestTypeErr(t *testing.T) {
-	m0 := newTestModel()
-	m1 := newTestModel()
-
-	ar, aw := io.Pipe()
-	br, bw := io.Pipe()
-
-	c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
-	NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
-
-	w := xdr.NewWriter(c0.cw)
-	w.WriteUint32(encodeHeader(header{
-		version: 0,
-		msgID:   0,
-		msgType: 42,
-	}))
-	w.WriteUint32(0) // Avoids reader closing due to EOF
-
-	if !m1.isClosed() {
-		t.Error("Connection should close due to unknown message type")
-	}
-}
-
-func TestClose(t *testing.T) {
-	m0 := newTestModel()
-	m1 := newTestModel()
-
-	ar, aw := io.Pipe()
-	br, bw := io.Pipe()
-
-	c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
-	NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
-
-	c0.close(nil)
-
-	<-c0.closed
-	if !m0.isClosed() {
-		t.Fatal("Connection should be closed")
-	}
-
-	// None of these should panic, some should return an error
-
-	if c0.ping() {
-		t.Error("Ping should not return true")
-	}
-
-	c0.Index("default", nil, 0, nil)
-	c0.Index("default", nil, 0, nil)
-
-	if _, err := c0.Request("default", "foo", 0, 0, nil, 0, nil); err == nil {
-		t.Error("Request should return an error")
-	}
-}
-
-func TestElementSizeExceededNested(t *testing.T) {
-	m := ClusterConfigMessage{
-		Folders: []Folder{
-			{ID: "longstringlongstringlongstringinglongstringlongstringlonlongstringlongstringlon"},
-		},
-	}
-	_, err := m.EncodeXDR(ioutil.Discard)
-	if err == nil {
-		t.Errorf("ID length %d > max 64, but no error", len(m.Folders[0].ID))
-	}
-}
-
-func TestMarshalIndexMessage(t *testing.T) {
-	var quickCfg = &quick.Config{MaxCountScale: 10}
-	if testing.Short() {
-		quickCfg = nil
-	}
-
-	f := func(m1 IndexMessage) bool {
-		for _, f := range m1.Files {
-			for i := range f.Blocks {
-				f.Blocks[i].Offset = 0
-				if len(f.Blocks[i].Hash) == 0 {
-					f.Blocks[i].Hash = nil
-				}
-			}
-		}
-
-		return testMarshal(t, "index", &m1, &IndexMessage{})
-	}
-
-	if err := quick.Check(f, quickCfg); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestMarshalRequestMessage(t *testing.T) {
-	var quickCfg = &quick.Config{MaxCountScale: 10}
-	if testing.Short() {
-		quickCfg = nil
-	}
-
-	f := func(m1 RequestMessage) bool {
-		return testMarshal(t, "request", &m1, &RequestMessage{})
-	}
-
-	if err := quick.Check(f, quickCfg); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestMarshalResponseMessage(t *testing.T) {
-	var quickCfg = &quick.Config{MaxCountScale: 10}
-	if testing.Short() {
-		quickCfg = nil
-	}
-
-	f := func(m1 ResponseMessage) bool {
-		if len(m1.Data) == 0 {
-			m1.Data = nil
-		}
-		return testMarshal(t, "response", &m1, &ResponseMessage{})
-	}
-
-	if err := quick.Check(f, quickCfg); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestMarshalClusterConfigMessage(t *testing.T) {
-	var quickCfg = &quick.Config{MaxCountScale: 10}
-	if testing.Short() {
-		quickCfg = nil
-	}
-
-	f := func(m1 ClusterConfigMessage) bool {
-		return testMarshal(t, "clusterconfig", &m1, &ClusterConfigMessage{})
-	}
-
-	if err := quick.Check(f, quickCfg); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestMarshalCloseMessage(t *testing.T) {
-	var quickCfg = &quick.Config{MaxCountScale: 10}
-	if testing.Short() {
-		quickCfg = nil
-	}
-
-	f := func(m1 CloseMessage) bool {
-		return testMarshal(t, "close", &m1, &CloseMessage{})
-	}
-
-	if err := quick.Check(f, quickCfg); err != nil {
-		t.Error(err)
-	}
-}
-
-type message interface {
-	EncodeXDR(io.Writer) (int, error)
-	DecodeXDR(io.Reader) error
-}
-
-func testMarshal(t *testing.T, prefix string, m1, m2 message) bool {
-	var buf bytes.Buffer
-
-	failed := func(bc []byte) {
-		bs, _ := json.MarshalIndent(m1, "", "  ")
-		ioutil.WriteFile(prefix+"-1.txt", bs, 0644)
-		bs, _ = json.MarshalIndent(m2, "", "  ")
-		ioutil.WriteFile(prefix+"-2.txt", bs, 0644)
-		if len(bc) > 0 {
-			f, _ := os.Create(prefix + "-data.txt")
-			fmt.Fprint(f, hex.Dump(bc))
-			f.Close()
-		}
-	}
-
-	_, err := m1.EncodeXDR(&buf)
-	if err != nil && strings.Contains(err.Error(), "exceeds size") {
-		return true
-	}
-	if err != nil {
-		failed(nil)
-		t.Fatal(err)
-	}
-
-	bc := make([]byte, len(buf.Bytes()))
-	copy(bc, buf.Bytes())
-
-	err = m2.DecodeXDR(&buf)
-	if err != nil {
-		failed(bc)
-		t.Fatal(err)
-	}
-
-	ok := reflect.DeepEqual(m1, m2)
-	if !ok {
-		failed(bc)
-	}
-	return ok
-}

Godeps/_workspace/src/github.com/syncthing/protocol/vector_xdr.go

@@ -1,38 +0,0 @@
-// Copyright (C) 2015 The Protocol Authors.
-
-package protocol
-
-// This stuff is hacked up manually because genxdr doesn't support 'type
-// Vector []Counter' declarations and it was tricky when I tried to add it...
-
-type xdrWriter interface {
-	WriteUint32(uint32) (int, error)
-	WriteUint64(uint64) (int, error)
-}
-type xdrReader interface {
-	ReadUint32() uint32
-	ReadUint64() uint64
-}
-
-// EncodeXDRInto encodes the vector as an XDR object into the given XDR
-// encoder.
-func (v Vector) EncodeXDRInto(w xdrWriter) (int, error) {
-	w.WriteUint32(uint32(len(v)))
-	for i := range v {
-		w.WriteUint64(v[i].ID)
-		w.WriteUint64(v[i].Value)
-	}
-	return 4 + 16*len(v), nil
-}
-
-// DecodeXDRFrom decodes the XDR objects from the given reader into itself.
-func (v *Vector) DecodeXDRFrom(r xdrReader) error {
-	l := int(r.ReadUint32())
-	n := make(Vector, l)
-	for i := range n {
-		n[i].ID = r.ReadUint64()
-		n[i].Value = r.ReadUint64()
-	}
-	*v = n
-	return nil
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/external_test.go

@@ -1,58 +0,0 @@
-// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package leveldb
-
-import (
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/syndtr/goleveldb/leveldb/opt"
-	"github.com/syndtr/goleveldb/leveldb/testutil"
-)
-
-var _ = testutil.Defer(func() {
-	Describe("Leveldb external", func() {
-		o := &opt.Options{
-			DisableBlockCache:      true,
-			BlockRestartInterval:   5,
-			BlockSize:              80,
-			Compression:            opt.NoCompression,
-			OpenFilesCacheCapacity: -1,
-			Strict:                 opt.StrictAll,
-			WriteBuffer:            1000,
-			CompactionTableSize:    2000,
-		}
-
-		Describe("write test", func() {
-			It("should do write correctly", func(done Done) {
-				db := newTestingDB(o, nil, nil)
-				t := testutil.DBTesting{
-					DB:      db,
-					Deleted: testutil.KeyValue_Generate(nil, 500, 1, 50, 5, 5).Clone(),
-				}
-				testutil.DoDBTesting(&t)
-				db.TestClose()
-				done <- true
-			}, 20.0)
-		})
-
-		Describe("read test", func() {
-			testutil.AllKeyValueTesting(nil, nil, func(kv testutil.KeyValue) testutil.DB {
-				// Building the DB.
-				db := newTestingDB(o, nil, nil)
-				kv.IterateShuffled(nil, func(i int, key, value []byte) {
-					err := db.TestPut(key, value)
-					Expect(err).NotTo(HaveOccurred())
-				})
-
-				return db
-			}, func(db testutil.DB) {
-				db.(*testingDB).TestClose()
-			})
-		})
-	})
-})

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go

@@ -1,455 +0,0 @@
-// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package leveldb
-
-import (
-	"fmt"
-	"io"
-	"os"
-	"sync"
-	"sync/atomic"
-
-	"github.com/syndtr/goleveldb/leveldb/errors"
-	"github.com/syndtr/goleveldb/leveldb/iterator"
-	"github.com/syndtr/goleveldb/leveldb/journal"
-	"github.com/syndtr/goleveldb/leveldb/opt"
-	"github.com/syndtr/goleveldb/leveldb/storage"
-	"github.com/syndtr/goleveldb/leveldb/util"
-)
-
-type ErrManifestCorrupted struct {
-	Field  string
-	Reason string
-}
-
-func (e *ErrManifestCorrupted) Error() string {
-	return fmt.Sprintf("leveldb: manifest corrupted (field '%s'): %s", e.Field, e.Reason)
-}
-
-func newErrManifestCorrupted(f storage.File, field, reason string) error {
-	return errors.NewErrCorrupted(f, &ErrManifestCorrupted{field, reason})
-}
-
-// session represent a persistent database session.
-type session struct {
-	// Need 64-bit alignment.
-	stNextFileNum    uint64 // current unused file number
-	stJournalNum     uint64 // current journal file number; need external synchronization
-	stPrevJournalNum uint64 // prev journal file number; no longer used; for compatibility with older version of leveldb
-	stSeqNum         uint64 // last mem compacted seq; need external synchronization
-	stTempFileNum    uint64
-
-	stor     storage.Storage
-	storLock util.Releaser
-	o        *cachedOptions
-	icmp     *iComparer
-	tops     *tOps
-
-	manifest       *journal.Writer
-	manifestWriter storage.Writer
-	manifestFile   storage.File
-
-	stCompPtrs []iKey   // compaction pointers; need external synchronization
-	stVersion  *version // current version
-	vmu        sync.Mutex
-}
-
-// Creates new initialized session instance.
-func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) {
-	if stor == nil {
-		return nil, os.ErrInvalid
-	}
-	storLock, err := stor.Lock()
-	if err != nil {
-		return
-	}
-	s = &session{
-		stor:       stor,
-		storLock:   storLock,
-		stCompPtrs: make([]iKey, o.GetNumLevel()),
-	}
-	s.setOptions(o)
-	s.tops = newTableOps(s)
-	s.setVersion(newVersion(s))
-	s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed")
-	return
-}
-
-// Close session.
-func (s *session) close() {
-	s.tops.close()
-	if s.manifest != nil {
-		s.manifest.Close()
-	}
-	if s.manifestWriter != nil {
-		s.manifestWriter.Close()
-	}
-	s.manifest = nil
-	s.manifestWriter = nil
-	s.manifestFile = nil
-	s.stVersion = nil
-}
-
-// Release session lock.
-func (s *session) release() {
-	s.storLock.Release()
-}
-
-// Create a new database session; need external synchronization.
-func (s *session) create() error {
-	// create manifest
-	return s.newManifest(nil, nil)
-}
-
-// Recover a database session; need external synchronization.
-func (s *session) recover() (err error) {
-	defer func() {
-		if os.IsNotExist(err) {
-			// Don't return os.ErrNotExist if the underlying storage contains
-			// other files that belong to LevelDB. So the DB won't get trashed.
-			if files, _ := s.stor.GetFiles(storage.TypeAll); len(files) > 0 {
-				err = &errors.ErrCorrupted{File: &storage.FileInfo{Type: storage.TypeManifest}, Err: &errors.ErrMissingFiles{}}
-			}
-		}
-	}()
-
-	m, err := s.stor.GetManifest()
-	if err != nil {
-		return
-	}
-
-	reader, err := m.Open()
-	if err != nil {
-		return
-	}
-	defer reader.Close()
-	strict := s.o.GetStrict(opt.StrictManifest)
-	jr := journal.NewReader(reader, dropper{s, m}, strict, true)
-
-	staging := s.stVersion.newStaging()
-	rec := &sessionRecord{numLevel: s.o.GetNumLevel()}
-	for {
-		var r io.Reader
-		r, err = jr.Next()
-		if err != nil {
-			if err == io.EOF {
-				err = nil
-				break
-			}
-			return errors.SetFile(err, m)
-		}
-
-		err = rec.decode(r)
-		if err == nil {
-			// save compact pointers
-			for _, r := range rec.compPtrs {
-				s.stCompPtrs[r.level] = iKey(r.ikey)
-			}
-			// commit record to version staging
-			staging.commit(rec)
-		} else {
-			err = errors.SetFile(err, m)
-			if strict || !errors.IsCorrupted(err) {
-				return
-			} else {
-				s.logf("manifest error: %v (skipped)", errors.SetFile(err, m))
-			}
-		}
-		rec.resetCompPtrs()
-		rec.resetAddedTables()
-		rec.resetDeletedTables()
-	}
-
-	switch {
-	case !rec.has(recComparer):
-		return newErrManifestCorrupted(m, "comparer", "missing")
-	case rec.comparer != s.icmp.uName():
-		return newErrManifestCorrupted(m, "comparer", fmt.Sprintf("mismatch: want '%s', got '%s'", s.icmp.uName(), rec.comparer))
-	case !rec.has(recNextFileNum):
-		return newErrManifestCorrupted(m, "next-file-num", "missing")
-	case !rec.has(recJournalNum):
-		return newErrManifestCorrupted(m, "journal-file-num", "missing")
-	case !rec.has(recSeqNum):
-		return newErrManifestCorrupted(m, "seq-num", "missing")
-	}
-
-	s.manifestFile = m
-	s.setVersion(staging.finish())
-	s.setNextFileNum(rec.nextFileNum)
-	s.recordCommited(rec)
-	return nil
-}
-
-// Commit session; need external synchronization.
-func (s *session) commit(r *sessionRecord) (err error) {
-	v := s.version()
-	defer v.release()
-
-	// spawn new version based on current version
-	nv := v.spawn(r)
-
-	if s.manifest == nil {
-		// manifest journal writer not yet created, create one
-		err = s.newManifest(r, nv)
-	} else {
-		err = s.flushManifest(r)
-	}
-
-	// finally, apply new version if no error rise
-	if err == nil {
-		s.setVersion(nv)
-	}
-
-	return
-}
-
-// Pick a compaction based on current state; need external synchronization.
-func (s *session) pickCompaction() *compaction {
-	v := s.version()
-
-	var level int
-	var t0 tFiles
-	if v.cScore >= 1 {
-		level = v.cLevel
-		cptr := s.stCompPtrs[level]
-		tables := v.tables[level]
-		for _, t := range tables {
-			if cptr == nil || s.icmp.Compare(t.imax, cptr) > 0 {
-				t0 = append(t0, t)
-				break
-			}
-		}
-		if len(t0) == 0 {
-			t0 = append(t0, tables[0])
-		}
-	} else {
-		if p := atomic.LoadPointer(&v.cSeek); p != nil {
-			ts := (*tSet)(p)
-			level = ts.level
-			t0 = append(t0, ts.table)
-		} else {
-			v.release()
-			return nil
-		}
-	}
-
-	return newCompaction(s, v, level, t0)
-}
-
-// Create compaction from given level and range; need external synchronization.
-func (s *session) getCompactionRange(level int, umin, umax []byte) *compaction {
-	v := s.version()
-
-	t0 := v.tables[level].getOverlaps(nil, s.icmp, umin, umax, level == 0)
-	if len(t0) == 0 {
-		v.release()
-		return nil
-	}
-
-	// Avoid compacting too much in one shot in case the range is large.
-	// But we cannot do this for level-0 since level-0 files can overlap
-	// and we must not pick one file and drop another older file if the
-	// two files overlap.
-	if level > 0 {
-		limit := uint64(v.s.o.GetCompactionSourceLimit(level))
-		total := uint64(0)
-		for i, t := range t0 {
-			total += t.size
-			if total >= limit {
-				s.logf("table@compaction limiting F·%d -> F·%d", len(t0), i+1)
-				t0 = t0[:i+1]
-				break
-			}
-		}
-	}
-
-	return newCompaction(s, v, level, t0)
-}
-
-func newCompaction(s *session, v *version, level int, t0 tFiles) *compaction {
-	c := &compaction{
-		s:             s,
-		v:             v,
-		level:         level,
-		tables:        [2]tFiles{t0, nil},
-		maxGPOverlaps: uint64(s.o.GetCompactionGPOverlaps(level)),
-		tPtrs:         make([]int, s.o.GetNumLevel()),
-	}
-	c.expand()
-	c.save()
-	return c
-}
-
-// compaction represent a compaction state.
-type compaction struct {
-	s *session
-	v *version
-
-	level         int
-	tables        [2]tFiles
-	maxGPOverlaps uint64
-
-	gp                tFiles
-	gpi               int
-	seenKey           bool
-	gpOverlappedBytes uint64
-	imin, imax        iKey
-	tPtrs             []int
-	released          bool
-
-	snapGPI               int
-	snapSeenKey           bool
-	snapGPOverlappedBytes uint64
-	snapTPtrs             []int
-}
-
-func (c *compaction) save() {
-	c.snapGPI = c.gpi
-	c.snapSeenKey = c.seenKey
-	c.snapGPOverlappedBytes = c.gpOverlappedBytes
-	c.snapTPtrs = append(c.snapTPtrs[:0], c.tPtrs...)
-}
-
-func (c *compaction) restore() {
-	c.gpi = c.snapGPI
-	c.seenKey = c.snapSeenKey
-	c.gpOverlappedBytes = c.snapGPOverlappedBytes
-	c.tPtrs = append(c.tPtrs[:0], c.snapTPtrs...)
-}
-
-func (c *compaction) release() {
-	if !c.released {
-		c.released = true
-		c.v.release()
-	}
-}
-
-// Expand compacted tables; need external synchronization.
-func (c *compaction) expand() {
-	limit := uint64(c.s.o.GetCompactionExpandLimit(c.level))
-	vt0, vt1 := c.v.tables[c.level], c.v.tables[c.level+1]
-
-	t0, t1 := c.tables[0], c.tables[1]
-	imin, imax := t0.getRange(c.s.icmp)
-	// We expand t0 here just incase ukey hop across tables.
-	t0 = vt0.getOverlaps(t0, c.s.icmp, imin.ukey(), imax.ukey(), c.level == 0)
-	if len(t0) != len(c.tables[0]) {
-		imin, imax = t0.getRange(c.s.icmp)
-	}
-	t1 = vt1.getOverlaps(t1, c.s.icmp, imin.ukey(), imax.ukey(), false)
-	// Get entire range covered by compaction.
-	amin, amax := append(t0, t1...).getRange(c.s.icmp)
-
-	// See if we can grow the number of inputs in "level" without
-	// changing the number of "level+1" files we pick up.
-	if len(t1) > 0 {
-		exp0 := vt0.getOverlaps(nil, c.s.icmp, amin.ukey(), amax.ukey(), c.level == 0)
-		if len(exp0) > len(t0) && t1.size()+exp0.size() < limit {
-			xmin, xmax := exp0.getRange(c.s.icmp)
-			exp1 := vt1.getOverlaps(nil, c.s.icmp, xmin.ukey(), xmax.ukey(), false)
-			if len(exp1) == len(t1) {
-				c.s.logf("table@compaction expanding L%d+L%d (F·%d S·%s)+(F·%d S·%s) -> (F·%d S·%s)+(F·%d S·%s)",
-					c.level, c.level+1, len(t0), shortenb(int(t0.size())), len(t1), shortenb(int(t1.size())),
-					len(exp0), shortenb(int(exp0.size())), len(exp1), shortenb(int(exp1.size())))
-				imin, imax = xmin, xmax
-				t0, t1 = exp0, exp1
-				amin, amax = append(t0, t1...).getRange(c.s.icmp)
-			}
-		}
-	}
-
-	// Compute the set of grandparent files that overlap this compaction
-	// (parent == level+1; grandparent == level+2)
-	if c.level+2 < c.s.o.GetNumLevel() {
-		c.gp = c.v.tables[c.level+2].getOverlaps(c.gp, c.s.icmp, amin.ukey(), amax.ukey(), false)
-	}
-
-	c.tables[0], c.tables[1] = t0, t1
-	c.imin, c.imax = imin, imax
-}
-
-// Check whether compaction is trivial.
-func (c *compaction) trivial() bool {
-	return len(c.tables[0]) == 1 && len(c.tables[1]) == 0 && c.gp.size() <= c.maxGPOverlaps
-}
-
-func (c *compaction) baseLevelForKey(ukey []byte) bool {
-	for level, tables := range c.v.tables[c.level+2:] {
-		for c.tPtrs[level] < len(tables) {
-			t := tables[c.tPtrs[level]]
-			if c.s.icmp.uCompare(ukey, t.imax.ukey()) <= 0 {
-				// We've advanced far enough.
-				if c.s.icmp.uCompare(ukey, t.imin.ukey()) >= 0 {
-					// Key falls in this file's range, so definitely not base level.
-					return false
-				}
-				break
-			}
-			c.tPtrs[level]++
-		}
-	}
-	return true
-}
-
-func (c *compaction) shouldStopBefore(ikey iKey) bool {
-	for ; c.gpi < len(c.gp); c.gpi++ {
-		gp := c.gp[c.gpi]
-		if c.s.icmp.Compare(ikey, gp.imax) <= 0 {
-			break
-		}
-		if c.seenKey {
-			c.gpOverlappedBytes += gp.size
-		}
-	}
-	c.seenKey = true
-
-	if c.gpOverlappedBytes > c.maxGPOverlaps {
-		// Too much overlap for current output; start new output.
-		c.gpOverlappedBytes = 0
-		return true
-	}
-	return false
-}
-
-// Creates an iterator.
-func (c *compaction) newIterator() iterator.Iterator {
-	// Creates iterator slice.
-	icap := len(c.tables)
-	if c.level == 0 {
-		// Special case for level-0
-		icap = len(c.tables[0]) + 1
-	}
-	its := make([]iterator.Iterator, 0, icap)
-
-	// Options.
-	ro := &opt.ReadOptions{
-		DontFillCache: true,
-		Strict:        opt.StrictOverride,
-	}
-	strict := c.s.o.GetStrict(opt.StrictCompaction)
-	if strict {
-		ro.Strict |= opt.StrictReader
-	}
-
-	for i, tables := range c.tables {
-		if len(tables) == 0 {
-			continue
-		}
-
-		// Level-0 is not sorted and may overlaps each other.
-		if c.level+i == 0 {
-			for _, t := range tables {
-				its = append(its, c.s.tops.newIterator(t, nil, ro))
-			}
-		} else {
-			it := iterator.NewIndexedIterator(tables.newIndexIterator(c.s.tops, c.s.icmp, nil, ro), strict)
-			its = append(its, it)
-		}
-	}
-
-	return iterator.NewMergedIterator(its, c.s.icmp, strict)
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go

@@ -1,534 +0,0 @@
-// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reservefs.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package storage
-
-import (
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"runtime"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/syndtr/goleveldb/leveldb/util"
-)
-
-var errFileOpen = errors.New("leveldb/storage: file still open")
-
-type fileLock interface {
-	release() error
-}
-
-type fileStorageLock struct {
-	fs *fileStorage
-}
-
-func (lock *fileStorageLock) Release() {
-	fs := lock.fs
-	fs.mu.Lock()
-	defer fs.mu.Unlock()
-	if fs.slock == lock {
-		fs.slock = nil
-	}
-	return
-}
-
-// fileStorage is a file-system backed storage.
-type fileStorage struct {
-	path string
-
-	mu    sync.Mutex
-	flock fileLock
-	slock *fileStorageLock
-	logw  *os.File
-	buf   []byte
-	// Opened file counter; if open < 0 means closed.
-	open int
-	day  int
-}
-
-// OpenFile returns a new filesytem-backed storage implementation with the given
-// path. This also hold a file lock, so any subsequent attempt to open the same
-// path will fail.
-//
-// The storage must be closed after use, by calling Close method.
-func OpenFile(path string) (Storage, error) {
-	if err := os.MkdirAll(path, 0755); err != nil {
-		return nil, err
-	}
-
-	flock, err := newFileLock(filepath.Join(path, "LOCK"))
-	if err != nil {
-		return nil, err
-	}
-
-	defer func() {
-		if err != nil {
-			flock.release()
-		}
-	}()
-
-	rename(filepath.Join(path, "LOG"), filepath.Join(path, "LOG.old"))
-	logw, err := os.OpenFile(filepath.Join(path, "LOG"), os.O_WRONLY|os.O_CREATE, 0644)
-	if err != nil {
-		return nil, err
-	}
-
-	fs := &fileStorage{path: path, flock: flock, logw: logw}
-	runtime.SetFinalizer(fs, (*fileStorage).Close)
-	return fs, nil
-}
-
-func (fs *fileStorage) Lock() (util.Releaser, error) {
-	fs.mu.Lock()
-	defer fs.mu.Unlock()
-	if fs.open < 0 {
-		return nil, ErrClosed
-	}
-	if fs.slock != nil {
-		return nil, ErrLocked
-	}
-	fs.slock = &fileStorageLock{fs: fs}
-	return fs.slock, nil
-}
-
-func itoa(buf []byte, i int, wid int) []byte {
-	var u uint = uint(i)
-	if u == 0 && wid <= 1 {
-		return append(buf, '0')
-	}
-
-	// Assemble decimal in reverse order.
-	var b [32]byte
-	bp := len(b)
-	for ; u > 0 || wid > 0; u /= 10 {
-		bp--
-		wid--
-		b[bp] = byte(u%10) + '0'
-	}
-	return append(buf, b[bp:]...)
-}
-
-func (fs *fileStorage) printDay(t time.Time) {
-	if fs.day == t.Day() {
-		return
-	}
-	fs.day = t.Day()
-	fs.logw.Write([]byte("=============== " + t.Format("Jan 2, 2006 (MST)") + " ===============\n"))
-}
-
-func (fs *fileStorage) doLog(t time.Time, str string) {
-	fs.printDay(t)
-	hour, min, sec := t.Clock()
-	msec := t.Nanosecond() / 1e3
-	// time
-	fs.buf = itoa(fs.buf[:0], hour, 2)
-	fs.buf = append(fs.buf, ':')
-	fs.buf = itoa(fs.buf, min, 2)
-	fs.buf = append(fs.buf, ':')
-	fs.buf = itoa(fs.buf, sec, 2)
-	fs.buf = append(fs.buf, '.')
-	fs.buf = itoa(fs.buf, msec, 6)
-	fs.buf = append(fs.buf, ' ')
-	// write
-	fs.buf = append(fs.buf, []byte(str)...)
-	fs.buf = append(fs.buf, '\n')
-	fs.logw.Write(fs.buf)
-}
-
-func (fs *fileStorage) Log(str string) {
-	t := time.Now()
-	fs.mu.Lock()
-	defer fs.mu.Unlock()
-	if fs.open < 0 {
-		return
-	}
-	fs.doLog(t, str)
-}
-
-func (fs *fileStorage) log(str string) {
-	fs.doLog(time.Now(), str)
-}
-
-func (fs *fileStorage) GetFile(num uint64, t FileType) File {
-	return &file{fs: fs, num: num, t: t}
-}
-
-func (fs *fileStorage) GetFiles(t FileType) (ff []File, err error) {
-	fs.mu.Lock()
-	defer fs.mu.Unlock()
-	if fs.open < 0 {
-		return nil, ErrClosed
-	}
-	dir, err := os.Open(fs.path)
-	if err != nil {
-		return
-	}
-	fnn, err := dir.Readdirnames(0)
-	// Close the dir first before checking for Readdirnames error.
-	if err := dir.Close(); err != nil {
-		fs.log(fmt.Sprintf("close dir: %v", err))
-	}
-	if err != nil {
-		return
-	}
-	f := &file{fs: fs}
-	for _, fn := range fnn {
-		if f.parse(fn) && (f.t&t) != 0 {
-			ff = append(ff, f)
-			f = &file{fs: fs}
-		}
-	}
-	return
-}
-
-func (fs *fileStorage) GetManifest() (f File, err error) {
-	fs.mu.Lock()
-	defer fs.mu.Unlock()
-	if fs.open < 0 {
-		return nil, ErrClosed
-	}
-	dir, err := os.Open(fs.path)
-	if err != nil {
-		return
-	}
-	fnn, err := dir.Readdirnames(0)
-	// Close the dir first before checking for Readdirnames error.
-	if err := dir.Close(); err != nil {
-		fs.log(fmt.Sprintf("close dir: %v", err))
-	}
-	if err != nil {
-		return
-	}
-	// Find latest CURRENT file.
-	var rem []string
-	var pend bool
-	var cerr error
-	for _, fn := range fnn {
-		if strings.HasPrefix(fn, "CURRENT") {
-			pend1 := len(fn) > 7
-			// Make sure it is valid name for a CURRENT file, otherwise skip it.
-			if pend1 {
-				if fn[7] != '.' || len(fn) < 9 {
-					fs.log(fmt.Sprintf("skipping %s: invalid file name", fn))
-					continue
-				}
-				if _, e1 := strconv.ParseUint(fn[8:], 10, 0); e1 != nil {
-					fs.log(fmt.Sprintf("skipping %s: invalid file num: %v", fn, e1))
-					continue
-				}
-			}
-			path := filepath.Join(fs.path, fn)
-			r, e1 := os.OpenFile(path, os.O_RDONLY, 0)
-			if e1 != nil {
-				return nil, e1
-			}
-			b, e1 := ioutil.ReadAll(r)
-			if e1 != nil {
-				r.Close()
-				return nil, e1
-			}
-			f1 := &file{fs: fs}
-			if len(b) < 1 || b[len(b)-1] != '\n' || !f1.parse(string(b[:len(b)-1])) {
-				fs.log(fmt.Sprintf("skipping %s: corrupted or incomplete", fn))
-				if pend1 {
-					rem = append(rem, fn)
-				}
-				if !pend1 || cerr == nil {
-					cerr = fmt.Errorf("leveldb/storage: corrupted or incomplete %s file", fn)
-				}
-			} else if f != nil && f1.Num() < f.Num() {
-				fs.log(fmt.Sprintf("skipping %s: obsolete", fn))
-				if pend1 {
-					rem = append(rem, fn)
-				}
-			} else {
-				f = f1
-				pend = pend1
-			}
-			if err := r.Close(); err != nil {
-				fs.log(fmt.Sprintf("close %s: %v", fn, err))
-			}
-		}
-	}
-	// Don't remove any files if there is no valid CURRENT file.
-	if f == nil {
-		if cerr != nil {
-			err = cerr
-		} else {
-			err = os.ErrNotExist
-		}
-		return
-	}
-	// Rename pending CURRENT file to an effective CURRENT.
-	if pend {
-		path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f.Num())
-		if err := rename(path, filepath.Join(fs.path, "CURRENT")); err != nil {
-			fs.log(fmt.Sprintf("CURRENT.%d -> CURRENT: %v", f.Num(), err))
-		}
-	}
-	// Remove obsolete or incomplete pending CURRENT files.
-	for _, fn := range rem {
-		path := filepath.Join(fs.path, fn)
-		if err := os.Remove(path); err != nil {
-			fs.log(fmt.Sprintf("remove %s: %v", fn, err))
-		}
-	}
-	return
-}
-
-func (fs *fileStorage) SetManifest(f File) (err error) {
-	fs.mu.Lock()
-	defer fs.mu.Unlock()
-	if fs.open < 0 {
-		return ErrClosed
-	}
-	f2, ok := f.(*file)
-	if !ok || f2.t != TypeManifest {
-		return ErrInvalidFile
-	}
-	defer func() {
-		if err != nil {
-			fs.log(fmt.Sprintf("CURRENT: %v", err))
-		}
-	}()
-	path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f2.Num())
-	w, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
-	if err != nil {
-		return err
-	}
-	_, err = fmt.Fprintln(w, f2.name())
-	// Close the file first.
-	if err := w.Close(); err != nil {
-		fs.log(fmt.Sprintf("close CURRENT.%d: %v", f2.num, err))
-	}
-	if err != nil {
-		return err
-	}
-	return rename(path, filepath.Join(fs.path, "CURRENT"))
-}
-
-func (fs *fileStorage) Close() error {
-	fs.mu.Lock()
-	defer fs.mu.Unlock()
-	if fs.open < 0 {
-		return ErrClosed
-	}
-	// Clear the finalizer.
-	runtime.SetFinalizer(fs, nil)
-
-	if fs.open > 0 {
-		fs.log(fmt.Sprintf("refuse to close, %d files still open", fs.open))
-		return fmt.Errorf("leveldb/storage: cannot close, %d files still open", fs.open)
-	}
-	fs.open = -1
-	e1 := fs.logw.Close()
-	err := fs.flock.release()
-	if err == nil {
-		err = e1
-	}
-	return err
-}
-
-type fileWrap struct {
-	*os.File
-	f *file
-}
-
-func (fw fileWrap) Sync() error {
-	if err := fw.File.Sync(); err != nil {
-		return err
-	}
-	if fw.f.Type() == TypeManifest {
-		// Also sync parent directory if file type is manifest.
-		// See: https://code.google.com/p/leveldb/issues/detail?id=190.
-		if err := syncDir(fw.f.fs.path); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (fw fileWrap) Close() error {
-	f := fw.f
-	f.fs.mu.Lock()
-	defer f.fs.mu.Unlock()
-	if !f.open {
-		return ErrClosed
-	}
-	f.open = false
-	f.fs.open--
-	err := fw.File.Close()
-	if err != nil {
-		f.fs.log(fmt.Sprintf("close %s.%d: %v", f.Type(), f.Num(), err))
-	}
-	return err
-}
-
-type file struct {
-	fs   *fileStorage
-	num  uint64
-	t    FileType
-	open bool
-}
-
-func (f *file) Open() (Reader, error) {
-	f.fs.mu.Lock()
-	defer f.fs.mu.Unlock()
-	if f.fs.open < 0 {
-		return nil, ErrClosed
-	}
-	if f.open {
-		return nil, errFileOpen
-	}
-	of, err := os.OpenFile(f.path(), os.O_RDONLY, 0)
-	if err != nil {
-		if f.hasOldName() && os.IsNotExist(err) {
-			of, err = os.OpenFile(f.oldPath(), os.O_RDONLY, 0)
-			if err == nil {
-				goto ok
-			}
-		}
-		return nil, err
-	}
-ok:
-	f.open = true
-	f.fs.open++
-	return fileWrap{of, f}, nil
-}
-
-func (f *file) Create() (Writer, error) {
-	f.fs.mu.Lock()
-	defer f.fs.mu.Unlock()
-	if f.fs.open < 0 {
-		return nil, ErrClosed
-	}
-	if f.open {
-		return nil, errFileOpen
-	}
-	of, err := os.OpenFile(f.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
-	if err != nil {
-		return nil, err
-	}
-	f.open = true
-	f.fs.open++
-	return fileWrap{of, f}, nil
-}
-
-func (f *file) Replace(newfile File) error {
-	f.fs.mu.Lock()
-	defer f.fs.mu.Unlock()
-	if f.fs.open < 0 {
-		return ErrClosed
-	}
-	newfile2, ok := newfile.(*file)
-	if !ok {
-		return ErrInvalidFile
-	}
-	if f.open || newfile2.open {
-		return errFileOpen
-	}
-	return rename(newfile2.path(), f.path())
-}
-
-func (f *file) Type() FileType {
-	return f.t
-}
-
-func (f *file) Num() uint64 {
-	return f.num
-}
-
-func (f *file) Remove() error {
-	f.fs.mu.Lock()
-	defer f.fs.mu.Unlock()
-	if f.fs.open < 0 {
-		return ErrClosed
-	}
-	if f.open {
-		return errFileOpen
-	}
-	err := os.Remove(f.path())
-	if err != nil {
-		f.fs.log(fmt.Sprintf("remove %s.%d: %v", f.Type(), f.Num(), err))
-	}
-	// Also try remove file with old name, just in case.
-	if f.hasOldName() {
-		if e1 := os.Remove(f.oldPath()); !os.IsNotExist(e1) {
-			f.fs.log(fmt.Sprintf("remove %s.%d: %v (old name)", f.Type(), f.Num(), err))
-			err = e1
-		}
-	}
-	return err
-}
-
-func (f *file) hasOldName() bool {
-	return f.t == TypeTable
-}
-
-func (f *file) oldName() string {
-	switch f.t {
-	case TypeTable:
-		return fmt.Sprintf("%06d.sst", f.num)
-	}
-	return f.name()
-}
-
-func (f *file) oldPath() string {
-	return filepath.Join(f.fs.path, f.oldName())
-}
-
-func (f *file) name() string {
-	switch f.t {
-	case TypeManifest:
-		return fmt.Sprintf("MANIFEST-%06d", f.num)
-	case TypeJournal:
-		return fmt.Sprintf("%06d.log", f.num)
-	case TypeTable:
-		return fmt.Sprintf("%06d.ldb", f.num)
-	case TypeTemp:
-		return fmt.Sprintf("%06d.tmp", f.num)
-	default:
-		panic("invalid file type")
-	}
-}
-
-func (f *file) path() string {
-	return filepath.Join(f.fs.path, f.name())
-}
-
-func (f *file) parse(name string) bool {
-	var num uint64
-	var tail string
-	_, err := fmt.Sscanf(name, "%d.%s", &num, &tail)
-	if err == nil {
-		switch tail {
-		case "log":
-			f.t = TypeJournal
-		case "ldb", "sst":
-			f.t = TypeTable
-		case "tmp":
-			f.t = TypeTemp
-		default:
-			return false
-		}
-		f.num = num
-		return true
-	}
-	n, _ := fmt.Sscanf(name, "MANIFEST-%d%s", &num, &tail)
-	if n == 1 {
-		f.t = TypeManifest
-		f.num = num
-		return true
-	}
-
-	return false
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/mem_storage.go

@@ -1,203 +0,0 @@
-// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package storage
-
-import (
-	"bytes"
-	"os"
-	"sync"
-
-	"github.com/syndtr/goleveldb/leveldb/util"
-)
-
-const typeShift = 3
-
-type memStorageLock struct {
-	ms *memStorage
-}
-
-func (lock *memStorageLock) Release() {
-	ms := lock.ms
-	ms.mu.Lock()
-	defer ms.mu.Unlock()
-	if ms.slock == lock {
-		ms.slock = nil
-	}
-	return
-}
-
-// memStorage is a memory-backed storage.
-type memStorage struct {
-	mu       sync.Mutex
-	slock    *memStorageLock
-	files    map[uint64]*memFile
-	manifest *memFilePtr
-}
-
-// NewMemStorage returns a new memory-backed storage implementation.
-func NewMemStorage() Storage {
-	return &memStorage{
-		files: make(map[uint64]*memFile),
-	}
-}
-
-func (ms *memStorage) Lock() (util.Releaser, error) {
-	ms.mu.Lock()
-	defer ms.mu.Unlock()
-	if ms.slock != nil {
-		return nil, ErrLocked
-	}
-	ms.slock = &memStorageLock{ms: ms}
-	return ms.slock, nil
-}
-
-func (*memStorage) Log(str string) {}
-
-func (ms *memStorage) GetFile(num uint64, t FileType) File {
-	return &memFilePtr{ms: ms, num: num, t: t}
-}
-
-func (ms *memStorage) GetFiles(t FileType) ([]File, error) {
-	ms.mu.Lock()
-	var ff []File
-	for x, _ := range ms.files {
-		num, mt := x>>typeShift, FileType(x)&TypeAll
-		if mt&t == 0 {
-			continue
-		}
-		ff = append(ff, &memFilePtr{ms: ms, num: num, t: mt})
-	}
-	ms.mu.Unlock()
-	return ff, nil
-}
-
-func (ms *memStorage) GetManifest() (File, error) {
-	ms.mu.Lock()
-	defer ms.mu.Unlock()
-	if ms.manifest == nil {
-		return nil, os.ErrNotExist
-	}
-	return ms.manifest, nil
-}
-
-func (ms *memStorage) SetManifest(f File) error {
-	fm, ok := f.(*memFilePtr)
-	if !ok || fm.t != TypeManifest {
-		return ErrInvalidFile
-	}
-	ms.mu.Lock()
-	ms.manifest = fm
-	ms.mu.Unlock()
-	return nil
-}
-
-func (*memStorage) Close() error { return nil }
-
-type memReader struct {
-	*bytes.Reader
-	m *memFile
-}
-
-func (mr *memReader) Close() error {
-	return mr.m.Close()
-}
-
-type memFile struct {
-	bytes.Buffer
-	ms   *memStorage
-	open bool
-}
-
-func (*memFile) Sync() error { return nil }
-func (m *memFile) Close() error {
-	m.ms.mu.Lock()
-	m.open = false
-	m.ms.mu.Unlock()
-	return nil
-}
-
-type memFilePtr struct {
-	ms  *memStorage
-	num uint64
-	t   FileType
-}
-
-func (p *memFilePtr) x() uint64 {
-	return p.Num()<<typeShift | uint64(p.Type())
-}
-
-func (p *memFilePtr) Open() (Reader, error) {
-	ms := p.ms
-	ms.mu.Lock()
-	defer ms.mu.Unlock()
-	if m, exist := ms.files[p.x()]; exist {
-		if m.open {
-			return nil, errFileOpen
-		}
-		m.open = true
-		return &memReader{Reader: bytes.NewReader(m.Bytes()), m: m}, nil
-	}
-	return nil, os.ErrNotExist
-}
-
-func (p *memFilePtr) Create() (Writer, error) {
-	ms := p.ms
-	ms.mu.Lock()
-	defer ms.mu.Unlock()
-	m, exist := ms.files[p.x()]
-	if exist {
-		if m.open {
-			return nil, errFileOpen
-		}
-		m.Reset()
-	} else {
-		m = &memFile{ms: ms}
-		ms.files[p.x()] = m
-	}
-	m.open = true
-	return m, nil
-}
-
-func (p *memFilePtr) Replace(newfile File) error {
-	p1, ok := newfile.(*memFilePtr)
-	if !ok {
-		return ErrInvalidFile
-	}
-	ms := p.ms
-	ms.mu.Lock()
-	defer ms.mu.Unlock()
-	m1, exist := ms.files[p1.x()]
-	if !exist {
-		return os.ErrNotExist
-	}
-	m0, exist := ms.files[p.x()]
-	if (exist && m0.open) || m1.open {
-		return errFileOpen
-	}
-	delete(ms.files, p1.x())
-	ms.files[p.x()] = m1
-	return nil
-}
-
-func (p *memFilePtr) Type() FileType {
-	return p.t
-}
-
-func (p *memFilePtr) Num() uint64 {
-	return p.num
-}
-
-func (p *memFilePtr) Remove() error {
-	ms := p.ms
-	ms.mu.Lock()
-	defer ms.mu.Unlock()
-	if _, exist := ms.files[p.x()]; exist {
-		delete(ms.files, p.x())
-		return nil
-	}
-	return os.ErrNotExist
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/storage.go

@@ -1,157 +0,0 @@
-// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-// Package storage provides storage abstraction for LevelDB.
-package storage
-
-import (
-	"errors"
-	"fmt"
-	"io"
-
-	"github.com/syndtr/goleveldb/leveldb/util"
-)
-
-type FileType uint32
-
-const (
-	TypeManifest FileType = 1 << iota
-	TypeJournal
-	TypeTable
-	TypeTemp
-
-	TypeAll = TypeManifest | TypeJournal | TypeTable | TypeTemp
-)
-
-func (t FileType) String() string {
-	switch t {
-	case TypeManifest:
-		return "manifest"
-	case TypeJournal:
-		return "journal"
-	case TypeTable:
-		return "table"
-	case TypeTemp:
-		return "temp"
-	}
-	return fmt.Sprintf("<unknown:%d>", t)
-}
-
-var (
-	ErrInvalidFile = errors.New("leveldb/storage: invalid file for argument")
-	ErrLocked      = errors.New("leveldb/storage: already locked")
-	ErrClosed      = errors.New("leveldb/storage: closed")
-)
-
-// Syncer is the interface that wraps basic Sync method.
-type Syncer interface {
-	// Sync commits the current contents of the file to stable storage.
-	Sync() error
-}
-
-// Reader is the interface that groups the basic Read, Seek, ReadAt and Close
-// methods.
-type Reader interface {
-	io.ReadSeeker
-	io.ReaderAt
-	io.Closer
-}
-
-// Writer is the interface that groups the basic Write, Sync and Close
-// methods.
-type Writer interface {
-	io.WriteCloser
-	Syncer
-}
-
-// File is the file. A file instance must be goroutine-safe.
-type File interface {
-	// Open opens the file for read. Returns os.ErrNotExist error
-	// if the file does not exist.
-	// Returns ErrClosed if the underlying storage is closed.
-	Open() (r Reader, err error)
-
-	// Create creates the file for writting. Truncate the file if
-	// already exist.
-	// Returns ErrClosed if the underlying storage is closed.
-	Create() (w Writer, err error)
-
-	// Replace replaces file with newfile.
-	// Returns ErrClosed if the underlying storage is closed.
-	Replace(newfile File) error
-
-	// Type returns the file type
-	Type() FileType
-
-	// Num returns the file number.
-	Num() uint64
-
-	// Remove removes the file.
-	// Returns ErrClosed if the underlying storage is closed.
-	Remove() error
-}
-
-// Storage is the storage. A storage instance must be goroutine-safe.
-type Storage interface {
-	// Lock locks the storage. Any subsequent attempt to call Lock will fail
-	// until the last lock released.
-	// After use the caller should call the Release method.
-	Lock() (l util.Releaser, err error)
-
-	// Log logs a string. This is used for logging. An implementation
-	// may write to a file, stdout or simply do nothing.
-	Log(str string)
-
-	// GetFile returns a file for the given number and type. GetFile will never
-	// returns nil, even if the underlying storage is closed.
-	GetFile(num uint64, t FileType) File
-
-	// GetFiles returns a slice of files that match the given file types.
-	// The file types may be OR'ed together.
-	GetFiles(t FileType) ([]File, error)
-
-	// GetManifest returns a manifest file. Returns os.ErrNotExist if manifest
-	// file does not exist.
-	GetManifest() (File, error)
-
-	// SetManifest sets the given file as manifest file. The given file should
-	// be a manifest file type or error will be returned.
-	SetManifest(f File) error
-
-	// Close closes the storage. It is valid to call Close multiple times.
-	// Other methods should not be called after the storage has been closed.
-	Close() error
-}
-
-// FileInfo wraps basic file info.
-type FileInfo struct {
-	Type FileType
-	Num  uint64
-}
-
-func (fi FileInfo) String() string {
-	switch fi.Type {
-	case TypeManifest:
-		return fmt.Sprintf("MANIFEST-%06d", fi.Num)
-	case TypeJournal:
-		return fmt.Sprintf("%06d.log", fi.Num)
-	case TypeTable:
-		return fmt.Sprintf("%06d.ldb", fi.Num)
-	case TypeTemp:
-		return fmt.Sprintf("%06d.tmp", fi.Num)
-	default:
-		return fmt.Sprintf("%#x-%d", fi.Type, fi.Num)
-	}
-}
-
-// NewFileInfo creates new FileInfo from the given File. It will returns nil
-// if File is nil.
-func NewFileInfo(f File) *FileInfo {
-	if f == nil {
-		return nil
-	}
-	return &FileInfo{f.Type(), f.Num()}
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage_test.go

@@ -1,539 +0,0 @@
-// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENE file.
-
-package leveldb
-
-import (
-	"errors"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"math/rand"
-	"os"
-	"path/filepath"
-	"sync"
-	"testing"
-
-	"github.com/syndtr/goleveldb/leveldb/storage"
-	"github.com/syndtr/goleveldb/leveldb/util"
-)
-
-const typeShift = 4
-
-var (
-	tsErrInvalidFile = errors.New("leveldb.testStorage: invalid file for argument")
-	tsErrFileOpen    = errors.New("leveldb.testStorage: file still open")
-)
-
-var (
-	tsFSEnv   = os.Getenv("GOLEVELDB_USEFS")
-	tsTempdir = os.Getenv("GOLEVELDB_TEMPDIR")
-	tsKeepFS  = tsFSEnv == "2"
-	tsFS      = tsKeepFS || tsFSEnv == "" || tsFSEnv == "1"
-	tsMU      = &sync.Mutex{}
-	tsNum     = 0
-)
-
-type tsOp uint
-
-const (
-	tsOpOpen tsOp = iota
-	tsOpCreate
-	tsOpRead
-	tsOpReadAt
-	tsOpWrite
-	tsOpSync
-
-	tsOpNum
-)
-
-type tsLock struct {
-	ts *testStorage
-	r  util.Releaser
-}
-
-func (l tsLock) Release() {
-	l.r.Release()
-	l.ts.t.Log("I: storage lock released")
-}
-
-type tsReader struct {
-	tf tsFile
-	storage.Reader
-}
-
-func (tr tsReader) Read(b []byte) (n int, err error) {
-	ts := tr.tf.ts
-	ts.countRead(tr.tf.Type())
-	if tr.tf.shouldErrLocked(tsOpRead) {
-		return 0, errors.New("leveldb.testStorage: emulated read error")
-	}
-	n, err = tr.Reader.Read(b)
-	if err != nil && err != io.EOF {
-		ts.t.Errorf("E: read error, num=%d type=%v n=%d: %v", tr.tf.Num(), tr.tf.Type(), n, err)
-	}
-	return
-}
-
-func (tr tsReader) ReadAt(b []byte, off int64) (n int, err error) {
-	ts := tr.tf.ts
-	ts.countRead(tr.tf.Type())
-	if tr.tf.shouldErrLocked(tsOpReadAt) {
-		return 0, errors.New("leveldb.testStorage: emulated readAt error")
-	}
-	n, err = tr.Reader.ReadAt(b, off)
-	if err != nil && err != io.EOF {
-		ts.t.Errorf("E: readAt error, num=%d type=%v off=%d n=%d: %v", tr.tf.Num(), tr.tf.Type(), off, n, err)
-	}
-	return
-}
-
-func (tr tsReader) Close() (err error) {
-	err = tr.Reader.Close()
-	tr.tf.close("reader", err)
-	return
-}
-
-type tsWriter struct {
-	tf tsFile
-	storage.Writer
-}
-
-func (tw tsWriter) Write(b []byte) (n int, err error) {
-	if tw.tf.shouldErrLocked(tsOpWrite) {
-		return 0, errors.New("leveldb.testStorage: emulated write error")
-	}
-	n, err = tw.Writer.Write(b)
-	if err != nil {
-		tw.tf.ts.t.Errorf("E: write error, num=%d type=%v n=%d: %v", tw.tf.Num(), tw.tf.Type(), n, err)
-	}
-	return
-}
-
-func (tw tsWriter) Sync() (err error) {
-	ts := tw.tf.ts
-	ts.mu.Lock()
-	for ts.emuDelaySync&tw.tf.Type() != 0 {
-		ts.cond.Wait()
-	}
-	ts.mu.Unlock()
-	if tw.tf.shouldErrLocked(tsOpSync) {
-		return errors.New("leveldb.testStorage: emulated sync error")
-	}
-	err = tw.Writer.Sync()
-	if err != nil {
-		tw.tf.ts.t.Errorf("E: sync error, num=%d type=%v: %v", tw.tf.Num(), tw.tf.Type(), err)
-	}
-	return
-}
-
-func (tw tsWriter) Close() (err error) {
-	err = tw.Writer.Close()
-	tw.tf.close("writer", err)
-	return
-}
-
-type tsFile struct {
-	ts *testStorage
-	storage.File
-}
-
-func (tf tsFile) x() uint64 {
-	return tf.Num()<<typeShift | uint64(tf.Type())
-}
-
-func (tf tsFile) shouldErr(op tsOp) bool {
-	return tf.ts.shouldErr(tf, op)
-}
-
-func (tf tsFile) shouldErrLocked(op tsOp) bool {
-	tf.ts.mu.Lock()
-	defer tf.ts.mu.Unlock()
-	return tf.shouldErr(op)
-}
-
-func (tf tsFile) checkOpen(m string) error {
-	ts := tf.ts
-	if writer, ok := ts.opens[tf.x()]; ok {
-		if writer {
-			ts.t.Errorf("E: cannot %s file, num=%d type=%v: a writer still open", m, tf.Num(), tf.Type())
-		} else {
-			ts.t.Errorf("E: cannot %s file, num=%d type=%v: a reader still open", m, tf.Num(), tf.Type())
-		}
-		return tsErrFileOpen
-	}
-	return nil
-}
-
-func (tf tsFile) close(m string, err error) {
-	ts := tf.ts
-	ts.mu.Lock()
-	defer ts.mu.Unlock()
-	if _, ok := ts.opens[tf.x()]; !ok {
-		ts.t.Errorf("E: %s: redudant file closing, num=%d type=%v", m, tf.Num(), tf.Type())
-	} else if err == nil {
-		ts.t.Logf("I: %s: file closed, num=%d type=%v", m, tf.Num(), tf.Type())
-	}
-	delete(ts.opens, tf.x())
-	if err != nil {
-		ts.t.Errorf("E: %s: cannot close file, num=%d type=%v: %v", m, tf.Num(), tf.Type(), err)
-	}
-}
-
-func (tf tsFile) Open() (r storage.Reader, err error) {
-	ts := tf.ts
-	ts.mu.Lock()
-	defer ts.mu.Unlock()
-	err = tf.checkOpen("open")
-	if err != nil {
-		return
-	}
-	if tf.shouldErr(tsOpOpen) {
-		err = errors.New("leveldb.testStorage: emulated open error")
-		return
-	}
-	r, err = tf.File.Open()
-	if err != nil {
-		if ts.ignoreOpenErr&tf.Type() != 0 {
-			ts.t.Logf("I: cannot open file, num=%d type=%v: %v (ignored)", tf.Num(), tf.Type(), err)
-		} else {
-			ts.t.Errorf("E: cannot open file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
-		}
-	} else {
-		ts.t.Logf("I: file opened, num=%d type=%v", tf.Num(), tf.Type())
-		ts.opens[tf.x()] = false
-		r = tsReader{tf, r}
-	}
-	return
-}
-
-func (tf tsFile) Create() (w storage.Writer, err error) {
-	ts := tf.ts
-	ts.mu.Lock()
-	defer ts.mu.Unlock()
-	err = tf.checkOpen("create")
-	if err != nil {
-		return
-	}
-	if tf.shouldErr(tsOpCreate) {
-		err = errors.New("leveldb.testStorage: emulated create error")
-		return
-	}
-	w, err = tf.File.Create()
-	if err != nil {
-		ts.t.Errorf("E: cannot create file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
-	} else {
-		ts.t.Logf("I: file created, num=%d type=%v", tf.Num(), tf.Type())
-		ts.opens[tf.x()] = true
-		w = tsWriter{tf, w}
-	}
-	return
-}
-
-func (tf tsFile) Replace(newfile storage.File) (err error) {
-	ts := tf.ts
-	ts.mu.Lock()
-	defer ts.mu.Unlock()
-	err = tf.checkOpen("replace")
-	if err != nil {
-		return
-	}
-	err = tf.File.Replace(newfile.(tsFile).File)
-	if err != nil {
-		ts.t.Errorf("E: cannot replace file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
-	} else {
-		ts.t.Logf("I: file replace, num=%d type=%v", tf.Num(), tf.Type())
-	}
-	return
-}
-
-func (tf tsFile) Remove() (err error) {
-	ts := tf.ts
-	ts.mu.Lock()
-	defer ts.mu.Unlock()
-	err = tf.checkOpen("remove")
-	if err != nil {
-		return
-	}
-	err = tf.File.Remove()
-	if err != nil {
-		ts.t.Errorf("E: cannot remove file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
-	} else {
-		ts.t.Logf("I: file removed, num=%d type=%v", tf.Num(), tf.Type())
-	}
-	return
-}
-
-type testStorage struct {
-	t *testing.T
-	storage.Storage
-	closeFn func() error
-
-	mu   sync.Mutex
-	cond sync.Cond
-	// Open files, true=writer, false=reader
-	opens         map[uint64]bool
-	emuDelaySync  storage.FileType
-	ignoreOpenErr storage.FileType
-	readCnt       uint64
-	readCntEn     storage.FileType
-
-	emuErr         [tsOpNum]storage.FileType
-	emuErrOnce     [tsOpNum]storage.FileType
-	emuRandErr     [tsOpNum]storage.FileType
-	emuRandErrProb int
-	emuErrOnceMap  map[uint64]uint
-	emuRandRand    *rand.Rand
-}
-
-func (ts *testStorage) shouldErr(tf tsFile, op tsOp) bool {
-	if ts.emuErr[op]&tf.Type() != 0 {
-		return true
-	} else if ts.emuRandErr[op]&tf.Type() != 0 || ts.emuErrOnce[op]&tf.Type() != 0 {
-		sop := uint(1) << op
-		eop := ts.emuErrOnceMap[tf.x()]
-		if eop&sop == 0 && (ts.emuRandRand.Int()%ts.emuRandErrProb == 0 || ts.emuErrOnce[op]&tf.Type() != 0) {
-			ts.emuErrOnceMap[tf.x()] = eop | sop
-			ts.t.Logf("I: emulated error: file=%d type=%v op=%v", tf.Num(), tf.Type(), op)
-			return true
-		}
-	}
-	return false
-}
-
-func (ts *testStorage) SetEmuErr(t storage.FileType, ops ...tsOp) {
-	ts.mu.Lock()
-	for _, op := range ops {
-		ts.emuErr[op] = t
-	}
-	ts.mu.Unlock()
-}
-
-func (ts *testStorage) SetEmuErrOnce(t storage.FileType, ops ...tsOp) {
-	ts.mu.Lock()
-	for _, op := range ops {
-		ts.emuErrOnce[op] = t
-	}
-	ts.mu.Unlock()
-}
-
-func (ts *testStorage) SetEmuRandErr(t storage.FileType, ops ...tsOp) {
-	ts.mu.Lock()
-	for _, op := range ops {
-		ts.emuRandErr[op] = t
-	}
-	ts.mu.Unlock()
-}
-
-func (ts *testStorage) SetEmuRandErrProb(prob int) {
-	ts.mu.Lock()
-	ts.emuRandErrProb = prob
-	ts.mu.Unlock()
-}
-
-func (ts *testStorage) DelaySync(t storage.FileType) {
-	ts.mu.Lock()
-	ts.emuDelaySync |= t
-	ts.cond.Broadcast()
-	ts.mu.Unlock()
-}
-
-func (ts *testStorage) ReleaseSync(t storage.FileType) {
-	ts.mu.Lock()
-	ts.emuDelaySync &= ^t
-	ts.cond.Broadcast()
-	ts.mu.Unlock()
-}
-
-func (ts *testStorage) ReadCounter() uint64 {
-	ts.mu.Lock()
-	defer ts.mu.Unlock()
-	return ts.readCnt
-}
-
-func (ts *testStorage) ResetReadCounter() {
-	ts.mu.Lock()
-	ts.readCnt = 0
-	ts.mu.Unlock()
-}
-
-func (ts *testStorage) SetReadCounter(t storage.FileType) {
-	ts.mu.Lock()
-	ts.readCntEn = t
-	ts.mu.Unlock()
-}
-
-func (ts *testStorage) countRead(t storage.FileType) {
-	ts.mu.Lock()
-	if ts.readCntEn&t != 0 {
-		ts.readCnt++
-	}
-	ts.mu.Unlock()
-}
-
-func (ts *testStorage) SetIgnoreOpenErr(t storage.FileType) {
-	ts.ignoreOpenErr = t
-}
-
-func (ts *testStorage) Lock() (r util.Releaser, err error) {
-	r, err = ts.Storage.Lock()
-	if err != nil {
-		ts.t.Logf("W: storage locking failed: %v", err)
-	} else {
-		ts.t.Log("I: storage locked")
-		r = tsLock{ts, r}
-	}
-	return
-}
-
-func (ts *testStorage) Log(str string) {
-	ts.t.Log("L: " + str)
-	ts.Storage.Log(str)
-}
-
-func (ts *testStorage) GetFile(num uint64, t storage.FileType) storage.File {
-	return tsFile{ts, ts.Storage.GetFile(num, t)}
-}
-
-func (ts *testStorage) GetFiles(t storage.FileType) (ff []storage.File, err error) {
-	ff0, err := ts.Storage.GetFiles(t)
-	if err != nil {
-		ts.t.Errorf("E: get files failed: %v", err)
-		return
-	}
-	ff = make([]storage.File, len(ff0))
-	for i, f := range ff0 {
-		ff[i] = tsFile{ts, f}
-	}
-	ts.t.Logf("I: get files, type=0x%x count=%d", int(t), len(ff))
-	return
-}
-
-func (ts *testStorage) GetManifest() (f storage.File, err error) {
-	f0, err := ts.Storage.GetManifest()
-	if err != nil {
-		if !os.IsNotExist(err) {
-			ts.t.Errorf("E: get manifest failed: %v", err)
-		}
-		return
-	}
-	f = tsFile{ts, f0}
-	ts.t.Logf("I: get manifest, num=%d", f.Num())
-	return
-}
-
-func (ts *testStorage) SetManifest(f storage.File) error {
-	tf, ok := f.(tsFile)
-	if !ok {
-		ts.t.Error("E: set manifest failed: type assertion failed")
-		return tsErrInvalidFile
-	} else if tf.Type() != storage.TypeManifest {
-		ts.t.Errorf("E: set manifest failed: invalid file type: %s", tf.Type())
-		return tsErrInvalidFile
-	}
-	err := ts.Storage.SetManifest(tf.File)
-	if err != nil {
-		ts.t.Errorf("E: set manifest failed: %v", err)
-	} else {
-		ts.t.Logf("I: set manifest, num=%d", tf.Num())
-	}
-	return err
-}
-
-func (ts *testStorage) Close() error {
-	ts.CloseCheck()
-	err := ts.Storage.Close()
-	if err != nil {
-		ts.t.Errorf("E: closing storage failed: %v", err)
-	} else {
-		ts.t.Log("I: storage closed")
-	}
-	if ts.closeFn != nil {
-		if err := ts.closeFn(); err != nil {
-			ts.t.Errorf("E: close function: %v", err)
-		}
-	}
-	return err
-}
-
-func (ts *testStorage) CloseCheck() {
-	ts.mu.Lock()
-	if len(ts.opens) == 0 {
-		ts.t.Log("I: all files are closed")
-	} else {
-		ts.t.Errorf("E: %d files still open", len(ts.opens))
-		for x, writer := range ts.opens {
-			num, tt := x>>typeShift, storage.FileType(x)&storage.TypeAll
-			ts.t.Errorf("E: * num=%d type=%v writer=%v", num, tt, writer)
-		}
-	}
-	ts.mu.Unlock()
-}
-
-func newTestStorage(t *testing.T) *testStorage {
-	var stor storage.Storage
-	var closeFn func() error
-	if tsFS {
-		for {
-			tsMU.Lock()
-			num := tsNum
-			tsNum++
-			tsMU.Unlock()
-			tempdir := tsTempdir
-			if tempdir == "" {
-				tempdir = os.TempDir()
-			}
-			path := filepath.Join(tempdir, fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num))
-			if _, err := os.Stat(path); err != nil {
-				stor, err = storage.OpenFile(path)
-				if err != nil {
-					t.Fatalf("F: cannot create storage: %v", err)
-				}
-				t.Logf("I: storage created: %s", path)
-				closeFn = func() error {
-					for _, name := range []string{"LOG.old", "LOG"} {
-						f, err := os.Open(filepath.Join(path, name))
-						if err != nil {
-							continue
-						}
-						if log, err := ioutil.ReadAll(f); err != nil {
-							t.Logf("---------------------- %s ----------------------", name)
-							t.Logf("cannot read log: %v", err)
-							t.Logf("---------------------- %s ----------------------", name)
-						} else if len(log) > 0 {
-							t.Logf("---------------------- %s ----------------------\n%s", name, string(log))
-							t.Logf("---------------------- %s ----------------------", name)
-						}
-						f.Close()
-					}
-					if t.Failed() {
-						t.Logf("testing failed, test DB preserved at %s", path)
-						return nil
-					}
-					if tsKeepFS {
-						return nil
-					}
-					return os.RemoveAll(path)
-				}
-
-				break
-			}
-		}
-	} else {
-		stor = storage.NewMemStorage()
-	}
-	ts := &testStorage{
-		t:              t,
-		Storage:        stor,
-		closeFn:        closeFn,
-		opens:          make(map[uint64]bool),
-		emuErrOnceMap:  make(map[uint64]uint),
-		emuRandErrProb: 0x999,
-		emuRandRand:    rand.New(rand.NewSource(0xfacedead)),
-	}
-	ts.cond.L = &ts.mu
-	return ts
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/kvtest.go

@@ -1,187 +0,0 @@
-// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package testutil
-
-import (
-	"fmt"
-	"math/rand"
-
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/syndtr/goleveldb/leveldb/errors"
-	"github.com/syndtr/goleveldb/leveldb/util"
-)
-
-func KeyValueTesting(rnd *rand.Rand, kv KeyValue, p DB, setup func(KeyValue) DB, teardown func(DB)) {
-	if rnd == nil {
-		rnd = NewRand()
-	}
-
-	if p == nil {
-		BeforeEach(func() {
-			p = setup(kv)
-		})
-		if teardown != nil {
-			AfterEach(func() {
-				teardown(p)
-			})
-		}
-	}
-
-	It("Should find all keys with Find", func() {
-		if db, ok := p.(Find); ok {
-			ShuffledIndex(nil, kv.Len(), 1, func(i int) {
-				key_, key, value := kv.IndexInexact(i)
-
-				// Using exact key.
-				rkey, rvalue, err := db.TestFind(key)
-				Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key)
-				Expect(rkey).Should(Equal(key), "Key")
-				Expect(rvalue).Should(Equal(value), "Value for key %q", key)
-
-				// Using inexact key.
-				rkey, rvalue, err = db.TestFind(key_)
-				Expect(err).ShouldNot(HaveOccurred(), "Error for key %q (%q)", key_, key)
-				Expect(rkey).Should(Equal(key))
-				Expect(rvalue).Should(Equal(value), "Value for key %q (%q)", key_, key)
-			})
-		}
-	})
-
-	It("Should return error if the key is not present", func() {
-		if db, ok := p.(Find); ok {
-			var key []byte
-			if kv.Len() > 0 {
-				key_, _ := kv.Index(kv.Len() - 1)
-				key = BytesAfter(key_)
-			}
-			rkey, _, err := db.TestFind(key)
-			Expect(err).Should(HaveOccurred(), "Find for key %q yield key %q", key, rkey)
-			Expect(err).Should(Equal(errors.ErrNotFound))
-		}
-	})
-
-	It("Should only find exact key with Get", func() {
-		if db, ok := p.(Get); ok {
-			ShuffledIndex(nil, kv.Len(), 1, func(i int) {
-				key_, key, value := kv.IndexInexact(i)
-
-				// Using exact key.
-				rvalue, err := db.TestGet(key)
-				Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key)
-				Expect(rvalue).Should(Equal(value), "Value for key %q", key)
-
-				// Using inexact key.
-				if len(key_) > 0 {
-					_, err = db.TestGet(key_)
-					Expect(err).Should(HaveOccurred(), "Error for key %q", key_)
-					Expect(err).Should(Equal(errors.ErrNotFound))
-				}
-			})
-		}
-	})
-
-	It("Should only find present key with Has", func() {
-		if db, ok := p.(Has); ok {
-			ShuffledIndex(nil, kv.Len(), 1, func(i int) {
-				key_, key, _ := kv.IndexInexact(i)
-
-				// Using exact key.
-				ret, err := db.TestHas(key)
-				Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key)
-				Expect(ret).Should(BeTrue(), "False for key %q", key)
-
-				// Using inexact key.
-				if len(key_) > 0 {
-					ret, err = db.TestHas(key_)
-					Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key_)
-					Expect(ret).ShouldNot(BeTrue(), "True for key %q", key)
-				}
-			})
-		}
-	})
-
-	TestIter := func(r *util.Range, _kv KeyValue) {
-		if db, ok := p.(NewIterator); ok {
-			iter := db.TestNewIterator(r)
-			Expect(iter.Error()).ShouldNot(HaveOccurred())
-
-			t := IteratorTesting{
-				KeyValue: _kv,
-				Iter:     iter,
-			}
-
-			DoIteratorTesting(&t)
-			iter.Release()
-		}
-	}
-
-	It("Should iterates and seeks correctly", func(done Done) {
-		TestIter(nil, kv.Clone())
-		done <- true
-	}, 3.0)
-
-	RandomIndex(rnd, kv.Len(), Min(kv.Len(), 50), func(i int) {
-		type slice struct {
-			r            *util.Range
-			start, limit int
-		}
-
-		key_, _, _ := kv.IndexInexact(i)
-		for _, x := range []slice{
-			{&util.Range{Start: key_, Limit: nil}, i, kv.Len()},
-			{&util.Range{Start: nil, Limit: key_}, 0, i},
-		} {
-			It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", x.start, x.limit), func(done Done) {
-				TestIter(x.r, kv.Slice(x.start, x.limit))
-				done <- true
-			}, 3.0)
-		}
-	})
-
-	RandomRange(rnd, kv.Len(), Min(kv.Len(), 50), func(start, limit int) {
-		It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", start, limit), func(done Done) {
-			r := kv.Range(start, limit)
-			TestIter(&r, kv.Slice(start, limit))
-			done <- true
-		}, 3.0)
-	})
-}
-
-func AllKeyValueTesting(rnd *rand.Rand, body, setup func(KeyValue) DB, teardown func(DB)) {
-	Test := func(kv *KeyValue) func() {
-		return func() {
-			var p DB
-			if setup != nil {
-				Defer("setup", func() {
-					p = setup(*kv)
-				})
-			}
-			if teardown != nil {
-				Defer("teardown", func() {
-					teardown(p)
-				})
-			}
-			if body != nil {
-				p = body(*kv)
-			}
-			KeyValueTesting(rnd, *kv, p, func(KeyValue) DB {
-				return p
-			}, nil)
-		}
-	}
-
-	Describe("with no key/value (empty)", Test(&KeyValue{}))
-	Describe("with empty key", Test(KeyValue_EmptyKey()))
-	Describe("with empty value", Test(KeyValue_EmptyValue()))
-	Describe("with one key/value", Test(KeyValue_OneKeyValue()))
-	Describe("with big value", Test(KeyValue_BigValue()))
-	Describe("with special key", Test(KeyValue_SpecialKey()))
-	Describe("with multiple key/value", Test(KeyValue_MultipleKeyValue()))
-	Describe("with generated key/value", Test(KeyValue_Generate(nil, 120, 1, 50, 10, 120)))
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/storage.go

@@ -1,586 +0,0 @@
-// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package testutil
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"runtime"
-	"strings"
-	"sync"
-
-	. "github.com/onsi/gomega"
-
-	"github.com/syndtr/goleveldb/leveldb/storage"
-	"github.com/syndtr/goleveldb/leveldb/util"
-)
-
-var (
-	storageMu     sync.Mutex
-	storageUseFS  bool = true
-	storageKeepFS bool = false
-	storageNum    int
-)
-
-type StorageMode int
-
-const (
-	ModeOpen StorageMode = 1 << iota
-	ModeCreate
-	ModeRemove
-	ModeRead
-	ModeWrite
-	ModeSync
-	ModeClose
-)
-
-const (
-	modeOpen = iota
-	modeCreate
-	modeRemove
-	modeRead
-	modeWrite
-	modeSync
-	modeClose
-
-	modeCount
-)
-
-const (
-	typeManifest = iota
-	typeJournal
-	typeTable
-	typeTemp
-
-	typeCount
-)
-
-const flattenCount = modeCount * typeCount
-
-func flattenType(m StorageMode, t storage.FileType) int {
-	var x int
-	switch m {
-	case ModeOpen:
-		x = modeOpen
-	case ModeCreate:
-		x = modeCreate
-	case ModeRemove:
-		x = modeRemove
-	case ModeRead:
-		x = modeRead
-	case ModeWrite:
-		x = modeWrite
-	case ModeSync:
-		x = modeSync
-	case ModeClose:
-		x = modeClose
-	default:
-		panic("invalid storage mode")
-	}
-	x *= typeCount
-	switch t {
-	case storage.TypeManifest:
-		return x + typeManifest
-	case storage.TypeJournal:
-		return x + typeJournal
-	case storage.TypeTable:
-		return x + typeTable
-	case storage.TypeTemp:
-		return x + typeTemp
-	default:
-		panic("invalid file type")
-	}
-}
-
-func listFlattenType(m StorageMode, t storage.FileType) []int {
-	ret := make([]int, 0, flattenCount)
-	add := func(x int) {
-		x *= typeCount
-		switch {
-		case t&storage.TypeManifest != 0:
-			ret = append(ret, x+typeManifest)
-		case t&storage.TypeJournal != 0:
-			ret = append(ret, x+typeJournal)
-		case t&storage.TypeTable != 0:
-			ret = append(ret, x+typeTable)
-		case t&storage.TypeTemp != 0:
-			ret = append(ret, x+typeTemp)
-		}
-	}
-	switch {
-	case m&ModeOpen != 0:
-		add(modeOpen)
-	case m&ModeCreate != 0:
-		add(modeCreate)
-	case m&ModeRemove != 0:
-		add(modeRemove)
-	case m&ModeRead != 0:
-		add(modeRead)
-	case m&ModeWrite != 0:
-		add(modeWrite)
-	case m&ModeSync != 0:
-		add(modeSync)
-	case m&ModeClose != 0:
-		add(modeClose)
-	}
-	return ret
-}
-
-func packFile(num uint64, t storage.FileType) uint64 {
-	if num>>(64-typeCount) != 0 {
-		panic("overflow")
-	}
-	return num<<typeCount | uint64(t)
-}
-
-func unpackFile(x uint64) (uint64, storage.FileType) {
-	return x >> typeCount, storage.FileType(x) & storage.TypeAll
-}
-
-type emulatedError struct {
-	err error
-}
-
-func (err emulatedError) Error() string {
-	return fmt.Sprintf("emulated storage error: %v", err.err)
-}
-
-type storageLock struct {
-	s *Storage
-	r util.Releaser
-}
-
-func (l storageLock) Release() {
-	l.r.Release()
-	l.s.logI("storage lock released")
-}
-
-type reader struct {
-	f *file
-	storage.Reader
-}
-
-func (r *reader) Read(p []byte) (n int, err error) {
-	err = r.f.s.emulateError(ModeRead, r.f.Type())
-	if err == nil {
-		r.f.s.stall(ModeRead, r.f.Type())
-		n, err = r.Reader.Read(p)
-	}
-	r.f.s.count(ModeRead, r.f.Type(), n)
-	if err != nil && err != io.EOF {
-		r.f.s.logI("read error, num=%d type=%v n=%d err=%v", r.f.Num(), r.f.Type(), n, err)
-	}
-	return
-}
-
-func (r *reader) ReadAt(p []byte, off int64) (n int, err error) {
-	err = r.f.s.emulateError(ModeRead, r.f.Type())
-	if err == nil {
-		r.f.s.stall(ModeRead, r.f.Type())
-		n, err = r.Reader.ReadAt(p, off)
-	}
-	r.f.s.count(ModeRead, r.f.Type(), n)
-	if err != nil && err != io.EOF {
-		r.f.s.logI("readAt error, num=%d type=%v offset=%d n=%d err=%v", r.f.Num(), r.f.Type(), off, n, err)
-	}
-	return
-}
-
-func (r *reader) Close() (err error) {
-	return r.f.doClose(r.Reader)
-}
-
-type writer struct {
-	f *file
-	storage.Writer
-}
-
-func (w *writer) Write(p []byte) (n int, err error) {
-	err = w.f.s.emulateError(ModeWrite, w.f.Type())
-	if err == nil {
-		w.f.s.stall(ModeWrite, w.f.Type())
-		n, err = w.Writer.Write(p)
-	}
-	w.f.s.count(ModeWrite, w.f.Type(), n)
-	if err != nil && err != io.EOF {
-		w.f.s.logI("write error, num=%d type=%v n=%d err=%v", w.f.Num(), w.f.Type(), n, err)
-	}
-	return
-}
-
-func (w *writer) Sync() (err error) {
-	err = w.f.s.emulateError(ModeSync, w.f.Type())
-	if err == nil {
-		w.f.s.stall(ModeSync, w.f.Type())
-		err = w.Writer.Sync()
-	}
-	w.f.s.count(ModeSync, w.f.Type(), 0)
-	if err != nil {
-		w.f.s.logI("sync error, num=%d type=%v err=%v", w.f.Num(), w.f.Type(), err)
-	}
-	return
-}
-
-func (w *writer) Close() (err error) {
-	return w.f.doClose(w.Writer)
-}
-
-type file struct {
-	s *Storage
-	storage.File
-}
-
-func (f *file) pack() uint64 {
-	return packFile(f.Num(), f.Type())
-}
-
-func (f *file) assertOpen() {
-	ExpectWithOffset(2, f.s.opens).NotTo(HaveKey(f.pack()), "File open, num=%d type=%v writer=%v", f.Num(), f.Type(), f.s.opens[f.pack()])
-}
-
-func (f *file) doClose(closer io.Closer) (err error) {
-	err = f.s.emulateError(ModeClose, f.Type())
-	if err == nil {
-		f.s.stall(ModeClose, f.Type())
-	}
-	f.s.mu.Lock()
-	defer f.s.mu.Unlock()
-	if err == nil {
-		ExpectWithOffset(2, f.s.opens).To(HaveKey(f.pack()), "File closed, num=%d type=%v", f.Num(), f.Type())
-		err = closer.Close()
-	}
-	f.s.countNB(ModeClose, f.Type(), 0)
-	writer := f.s.opens[f.pack()]
-	if err != nil {
-		f.s.logISkip(1, "file close failed, num=%d type=%v writer=%v err=%v", f.Num(), f.Type(), writer, err)
-	} else {
-		f.s.logISkip(1, "file closed, num=%d type=%v writer=%v", f.Num(), f.Type(), writer)
-		delete(f.s.opens, f.pack())
-	}
-	return
-}
-
-func (f *file) Open() (r storage.Reader, err error) {
-	err = f.s.emulateError(ModeOpen, f.Type())
-	if err == nil {
-		f.s.stall(ModeOpen, f.Type())
-	}
-	f.s.mu.Lock()
-	defer f.s.mu.Unlock()
-	if err == nil {
-		f.assertOpen()
-		f.s.countNB(ModeOpen, f.Type(), 0)
-		r, err = f.File.Open()
-	}
-	if err != nil {
-		f.s.logI("file open failed, num=%d type=%v err=%v", f.Num(), f.Type(), err)
-	} else {
-		f.s.logI("file opened, num=%d type=%v", f.Num(), f.Type())
-		f.s.opens[f.pack()] = false
-		r = &reader{f, r}
-	}
-	return
-}
-
-func (f *file) Create() (w storage.Writer, err error) {
-	err = f.s.emulateError(ModeCreate, f.Type())
-	if err == nil {
-		f.s.stall(ModeCreate, f.Type())
-	}
-	f.s.mu.Lock()
-	defer f.s.mu.Unlock()
-	if err == nil {
-		f.assertOpen()
-		f.s.countNB(ModeCreate, f.Type(), 0)
-		w, err = f.File.Create()
-	}
-	if err != nil {
-		f.s.logI("file create failed, num=%d type=%v err=%v", f.Num(), f.Type(), err)
-	} else {
-		f.s.logI("file created, num=%d type=%v", f.Num(), f.Type())
-		f.s.opens[f.pack()] = true
-		w = &writer{f, w}
-	}
-	return
-}
-
-func (f *file) Remove() (err error) {
-	err = f.s.emulateError(ModeRemove, f.Type())
-	if err == nil {
-		f.s.stall(ModeRemove, f.Type())
-	}
-	f.s.mu.Lock()
-	defer f.s.mu.Unlock()
-	if err == nil {
-		f.assertOpen()
-		f.s.countNB(ModeRemove, f.Type(), 0)
-		err = f.File.Remove()
-	}
-	if err != nil {
-		f.s.logI("file remove failed, num=%d type=%v err=%v", f.Num(), f.Type(), err)
-	} else {
-		f.s.logI("file removed, num=%d type=%v", f.Num(), f.Type())
-	}
-	return
-}
-
-type Storage struct {
-	storage.Storage
-	closeFn func() error
-
-	lmu sync.Mutex
-	lb  bytes.Buffer
-
-	mu sync.Mutex
-	// Open files, true=writer, false=reader
-	opens         map[uint64]bool
-	counters      [flattenCount]int
-	bytesCounter  [flattenCount]int64
-	emulatedError [flattenCount]error
-	stallCond     sync.Cond
-	stalled       [flattenCount]bool
-}
-
-func (s *Storage) log(skip int, str string) {
-	s.lmu.Lock()
-	defer s.lmu.Unlock()
-	_, file, line, ok := runtime.Caller(skip + 2)
-	if ok {
-		// Truncate file name at last file name separator.
-		if index := strings.LastIndex(file, "/"); index >= 0 {
-			file = file[index+1:]
-		} else if index = strings.LastIndex(file, "\\"); index >= 0 {
-			file = file[index+1:]
-		}
-	} else {
-		file = "???"
-		line = 1
-	}
-	fmt.Fprintf(&s.lb, "%s:%d: ", file, line)
-	lines := strings.Split(str, "\n")
-	if l := len(lines); l > 1 && lines[l-1] == "" {
-		lines = lines[:l-1]
-	}
-	for i, line := range lines {
-		if i > 0 {
-			s.lb.WriteString("\n\t")
-		}
-		s.lb.WriteString(line)
-	}
-	s.lb.WriteByte('\n')
-}
-
-func (s *Storage) logISkip(skip int, format string, args ...interface{}) {
-	pc, _, _, ok := runtime.Caller(skip + 1)
-	if ok {
-		if f := runtime.FuncForPC(pc); f != nil {
-			fname := f.Name()
-			if index := strings.LastIndex(fname, "."); index >= 0 {
-				fname = fname[index+1:]
-			}
-			format = fname + ": " + format
-		}
-	}
-	s.log(skip+1, fmt.Sprintf(format, args...))
-}
-
-func (s *Storage) logI(format string, args ...interface{}) {
-	s.logISkip(1, format, args...)
-}
-
-func (s *Storage) Log(str string) {
-	s.log(1, "Log: "+str)
-	s.Storage.Log(str)
-}
-
-func (s *Storage) Lock() (r util.Releaser, err error) {
-	r, err = s.Storage.Lock()
-	if err != nil {
-		s.logI("storage locking failed, err=%v", err)
-	} else {
-		s.logI("storage locked")
-		r = storageLock{s, r}
-	}
-	return
-}
-
-func (s *Storage) GetFile(num uint64, t storage.FileType) storage.File {
-	return &file{s, s.Storage.GetFile(num, t)}
-}
-
-func (s *Storage) GetFiles(t storage.FileType) (files []storage.File, err error) {
-	rfiles, err := s.Storage.GetFiles(t)
-	if err != nil {
-		s.logI("get files failed, err=%v", err)
-		return
-	}
-	files = make([]storage.File, len(rfiles))
-	for i, f := range rfiles {
-		files[i] = &file{s, f}
-	}
-	s.logI("get files, type=0x%x count=%d", int(t), len(files))
-	return
-}
-
-func (s *Storage) GetManifest() (f storage.File, err error) {
-	manifest, err := s.Storage.GetManifest()
-	if err != nil {
-		if !os.IsNotExist(err) {
-			s.logI("get manifest failed, err=%v", err)
-		}
-		return
-	}
-	s.logI("get manifest, num=%d", manifest.Num())
-	return &file{s, manifest}, nil
-}
-
-func (s *Storage) SetManifest(f storage.File) error {
-	f_, ok := f.(*file)
-	ExpectWithOffset(1, ok).To(BeTrue())
-	ExpectWithOffset(1, f_.Type()).To(Equal(storage.TypeManifest))
-	err := s.Storage.SetManifest(f_.File)
-	if err != nil {
-		s.logI("set manifest failed, err=%v", err)
-	} else {
-		s.logI("set manifest, num=%d", f_.Num())
-	}
-	return err
-}
-
-func (s *Storage) openFiles() string {
-	out := "Open files:"
-	for x, writer := range s.opens {
-		num, t := unpackFile(x)
-		out += fmt.Sprintf("\n · num=%d type=%v writer=%v", num, t, writer)
-	}
-	return out
-}
-
-func (s *Storage) Close() error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	ExpectWithOffset(1, s.opens).To(BeEmpty(), s.openFiles())
-	err := s.Storage.Close()
-	if err != nil {
-		s.logI("storage closing failed, err=%v", err)
-	} else {
-		s.logI("storage closed")
-	}
-	if s.closeFn != nil {
-		if err1 := s.closeFn(); err1 != nil {
-			s.logI("close func error, err=%v", err1)
-		}
-	}
-	return err
-}
-
-func (s *Storage) countNB(m StorageMode, t storage.FileType, n int) {
-	s.counters[flattenType(m, t)]++
-	s.bytesCounter[flattenType(m, t)] += int64(n)
-}
-
-func (s *Storage) count(m StorageMode, t storage.FileType, n int) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	s.countNB(m, t, n)
-}
-
-func (s *Storage) ResetCounter(m StorageMode, t storage.FileType) {
-	for _, x := range listFlattenType(m, t) {
-		s.counters[x] = 0
-		s.bytesCounter[x] = 0
-	}
-}
-
-func (s *Storage) Counter(m StorageMode, t storage.FileType) (count int, bytes int64) {
-	for _, x := range listFlattenType(m, t) {
-		count += s.counters[x]
-		bytes += s.bytesCounter[x]
-	}
-	return
-}
-
-func (s *Storage) emulateError(m StorageMode, t storage.FileType) error {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	err := s.emulatedError[flattenType(m, t)]
-	if err != nil {
-		return emulatedError{err}
-	}
-	return nil
-}
-
-func (s *Storage) EmulateError(m StorageMode, t storage.FileType, err error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	for _, x := range listFlattenType(m, t) {
-		s.emulatedError[x] = err
-	}
-}
-
-func (s *Storage) stall(m StorageMode, t storage.FileType) {
-	x := flattenType(m, t)
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	for s.stalled[x] {
-		s.stallCond.Wait()
-	}
-}
-
-func (s *Storage) Stall(m StorageMode, t storage.FileType) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	for _, x := range listFlattenType(m, t) {
-		s.stalled[x] = true
-	}
-}
-
-func (s *Storage) Release(m StorageMode, t storage.FileType) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
-	for _, x := range listFlattenType(m, t) {
-		s.stalled[x] = false
-	}
-	s.stallCond.Broadcast()
-}
-
-func NewStorage() *Storage {
-	var stor storage.Storage
-	var closeFn func() error
-	if storageUseFS {
-		for {
-			storageMu.Lock()
-			num := storageNum
-			storageNum++
-			storageMu.Unlock()
-			path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num))
-			if _, err := os.Stat(path); os.IsNotExist(err) {
-				stor, err = storage.OpenFile(path)
-				ExpectWithOffset(1, err).NotTo(HaveOccurred(), "creating storage at %s", path)
-				closeFn = func() error {
-					if storageKeepFS {
-						return nil
-					}
-					return os.RemoveAll(path)
-				}
-				break
-			}
-		}
-	} else {
-		stor = storage.NewMemStorage()
-	}
-	s := &Storage{
-		Storage: stor,
-		closeFn: closeFn,
-		opens:   make(map[uint64]bool),
-	}
-	s.stallCond.L = &s.mu
-	return s
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/version.go

@@ -1,436 +0,0 @@
-// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package leveldb
-
-import (
-	"sync/atomic"
-	"unsafe"
-
-	"github.com/syndtr/goleveldb/leveldb/iterator"
-	"github.com/syndtr/goleveldb/leveldb/opt"
-	"github.com/syndtr/goleveldb/leveldb/util"
-)
-
-type tSet struct {
-	level int
-	table *tFile
-}
-
-type version struct {
-	s *session
-
-	tables []tFiles
-
-	// Level that should be compacted next and its compaction score.
-	// Score < 1 means compaction is not strictly needed. These fields
-	// are initialized by computeCompaction()
-	cLevel int
-	cScore float64
-
-	cSeek unsafe.Pointer
-
-	ref int
-	// Succeeding version.
-	next *version
-}
-
-func newVersion(s *session) *version {
-	return &version{s: s, tables: make([]tFiles, s.o.GetNumLevel())}
-}
-
-func (v *version) releaseNB() {
-	v.ref--
-	if v.ref > 0 {
-		return
-	}
-	if v.ref < 0 {
-		panic("negative version ref")
-	}
-
-	tables := make(map[uint64]bool)
-	for _, tt := range v.next.tables {
-		for _, t := range tt {
-			num := t.file.Num()
-			tables[num] = true
-		}
-	}
-
-	for _, tt := range v.tables {
-		for _, t := range tt {
-			num := t.file.Num()
-			if _, ok := tables[num]; !ok {
-				v.s.tops.remove(t)
-			}
-		}
-	}
-
-	v.next.releaseNB()
-	v.next = nil
-}
-
-func (v *version) release() {
-	v.s.vmu.Lock()
-	v.releaseNB()
-	v.s.vmu.Unlock()
-}
-
-func (v *version) walkOverlapping(ikey iKey, f func(level int, t *tFile) bool, lf func(level int) bool) {
-	ukey := ikey.ukey()
-
-	// Walk tables level-by-level.
-	for level, tables := range v.tables {
-		if len(tables) == 0 {
-			continue
-		}
-
-		if level == 0 {
-			// Level-0 files may overlap each other. Find all files that
-			// overlap ukey.
-			for _, t := range tables {
-				if t.overlaps(v.s.icmp, ukey, ukey) {
-					if !f(level, t) {
-						return
-					}
-				}
-			}
-		} else {
-			if i := tables.searchMax(v.s.icmp, ikey); i < len(tables) {
-				t := tables[i]
-				if v.s.icmp.uCompare(ukey, t.imin.ukey()) >= 0 {
-					if !f(level, t) {
-						return
-					}
-				}
-			}
-		}
-
-		if lf != nil && !lf(level) {
-			return
-		}
-	}
-}
-
-func (v *version) get(ikey iKey, ro *opt.ReadOptions, noValue bool) (value []byte, tcomp bool, err error) {
-	ukey := ikey.ukey()
-
-	var (
-		tset  *tSet
-		tseek bool
-
-		// Level-0.
-		zfound bool
-		zseq   uint64
-		zkt    kType
-		zval   []byte
-	)
-
-	err = ErrNotFound
-
-	// Since entries never hope across level, finding key/value
-	// in smaller level make later levels irrelevant.
-	v.walkOverlapping(ikey, func(level int, t *tFile) bool {
-		if !tseek {
-			if tset == nil {
-				tset = &tSet{level, t}
-			} else if tset.table.consumeSeek() <= 0 {
-				tseek = true
-				tcomp = atomic.CompareAndSwapPointer(&v.cSeek, nil, unsafe.Pointer(tset))
-			}
-		}
-
-		var (
-			fikey, fval []byte
-			ferr        error
-		)
-		if noValue {
-			fikey, ferr = v.s.tops.findKey(t, ikey, ro)
-		} else {
-			fikey, fval, ferr = v.s.tops.find(t, ikey, ro)
-		}
-		switch ferr {
-		case nil:
-		case ErrNotFound:
-			return true
-		default:
-			err = ferr
-			return false
-		}
-
-		if fukey, fseq, fkt, fkerr := parseIkey(fikey); fkerr == nil {
-			if v.s.icmp.uCompare(ukey, fukey) == 0 {
-				if level == 0 {
-					if fseq >= zseq {
-						zfound = true
-						zseq = fseq
-						zkt = fkt
-						zval = fval
-					}
-				} else {
-					switch fkt {
-					case ktVal:
-						value = fval
-						err = nil
-					case ktDel:
-					default:
-						panic("leveldb: invalid iKey type")
-					}
-					return false
-				}
-			}
-		} else {
-			err = fkerr
-			return false
-		}
-
-		return true
-	}, func(level int) bool {
-		if zfound {
-			switch zkt {
-			case ktVal:
-				value = zval
-				err = nil
-			case ktDel:
-			default:
-				panic("leveldb: invalid iKey type")
-			}
-			return false
-		}
-
-		return true
-	})
-
-	return
-}
-
-func (v *version) getIterators(slice *util.Range, ro *opt.ReadOptions) (its []iterator.Iterator) {
-	// Merge all level zero files together since they may overlap
-	for _, t := range v.tables[0] {
-		it := v.s.tops.newIterator(t, slice, ro)
-		its = append(its, it)
-	}
-
-	strict := opt.GetStrict(v.s.o.Options, ro, opt.StrictReader)
-	for _, tables := range v.tables[1:] {
-		if len(tables) == 0 {
-			continue
-		}
-
-		it := iterator.NewIndexedIterator(tables.newIndexIterator(v.s.tops, v.s.icmp, slice, ro), strict)
-		its = append(its, it)
-	}
-
-	return
-}
-
-func (v *version) newStaging() *versionStaging {
-	return &versionStaging{base: v, tables: make([]tablesScratch, v.s.o.GetNumLevel())}
-}
-
-// Spawn a new version based on this version.
-func (v *version) spawn(r *sessionRecord) *version {
-	staging := v.newStaging()
-	staging.commit(r)
-	return staging.finish()
-}
-
-func (v *version) fillRecord(r *sessionRecord) {
-	for level, ts := range v.tables {
-		for _, t := range ts {
-			r.addTableFile(level, t)
-		}
-	}
-}
-
-func (v *version) tLen(level int) int {
-	return len(v.tables[level])
-}
-
-func (v *version) offsetOf(ikey iKey) (n uint64, err error) {
-	for level, tables := range v.tables {
-		for _, t := range tables {
-			if v.s.icmp.Compare(t.imax, ikey) <= 0 {
-				// Entire file is before "ikey", so just add the file size
-				n += t.size
-			} else if v.s.icmp.Compare(t.imin, ikey) > 0 {
-				// Entire file is after "ikey", so ignore
-				if level > 0 {
-					// Files other than level 0 are sorted by meta->min, so
-					// no further files in this level will contain data for
-					// "ikey".
-					break
-				}
-			} else {
-				// "ikey" falls in the range for this table. Add the
-				// approximate offset of "ikey" within the table.
-				var nn uint64
-				nn, err = v.s.tops.offsetOf(t, ikey)
-				if err != nil {
-					return 0, err
-				}
-				n += nn
-			}
-		}
-	}
-
-	return
-}
-
-func (v *version) pickLevel(umin, umax []byte) (level int) {
-	if !v.tables[0].overlaps(v.s.icmp, umin, umax, true) {
-		var overlaps tFiles
-		maxLevel := v.s.o.GetMaxMemCompationLevel()
-		for ; level < maxLevel; level++ {
-			if v.tables[level+1].overlaps(v.s.icmp, umin, umax, false) {
-				break
-			}
-			overlaps = v.tables[level+2].getOverlaps(overlaps, v.s.icmp, umin, umax, false)
-			if overlaps.size() > uint64(v.s.o.GetCompactionGPOverlaps(level)) {
-				break
-			}
-		}
-	}
-
-	return
-}
-
-func (v *version) computeCompaction() {
-	// Precomputed best level for next compaction
-	var bestLevel int = -1
-	var bestScore float64 = -1
-
-	for level, tables := range v.tables {
-		var score float64
-		if level == 0 {
-			// We treat level-0 specially by bounding the number of files
-			// instead of number of bytes for two reasons:
-			//
-			// (1) With larger write-buffer sizes, it is nice not to do too
-			// many level-0 compactions.
-			//
-			// (2) The files in level-0 are merged on every read and
-			// therefore we wish to avoid too many files when the individual
-			// file size is small (perhaps because of a small write-buffer
-			// setting, or very high compression ratios, or lots of
-			// overwrites/deletions).
-			score = float64(len(tables)) / float64(v.s.o.GetCompactionL0Trigger())
-		} else {
-			score = float64(tables.size()) / float64(v.s.o.GetCompactionTotalSize(level))
-		}
-
-		if score > bestScore {
-			bestLevel = level
-			bestScore = score
-		}
-	}
-
-	v.cLevel = bestLevel
-	v.cScore = bestScore
-}
-
-func (v *version) needCompaction() bool {
-	return v.cScore >= 1 || atomic.LoadPointer(&v.cSeek) != nil
-}
-
-type tablesScratch struct {
-	added   map[uint64]atRecord
-	deleted map[uint64]struct{}
-}
-
-type versionStaging struct {
-	base   *version
-	tables []tablesScratch
-}
-
-func (p *versionStaging) commit(r *sessionRecord) {
-	// Deleted tables.
-	for _, r := range r.deletedTables {
-		tm := &(p.tables[r.level])
-
-		if len(p.base.tables[r.level]) > 0 {
-			if tm.deleted == nil {
-				tm.deleted = make(map[uint64]struct{})
-			}
-			tm.deleted[r.num] = struct{}{}
-		}
-
-		if tm.added != nil {
-			delete(tm.added, r.num)
-		}
-	}
-
-	// New tables.
-	for _, r := range r.addedTables {
-		tm := &(p.tables[r.level])
-
-		if tm.added == nil {
-			tm.added = make(map[uint64]atRecord)
-		}
-		tm.added[r.num] = r
-
-		if tm.deleted != nil {
-			delete(tm.deleted, r.num)
-		}
-	}
-}
-
-func (p *versionStaging) finish() *version {
-	// Build new version.
-	nv := newVersion(p.base.s)
-	for level, tm := range p.tables {
-		btables := p.base.tables[level]
-
-		n := len(btables) + len(tm.added) - len(tm.deleted)
-		if n < 0 {
-			n = 0
-		}
-		nt := make(tFiles, 0, n)
-
-		// Base tables.
-		for _, t := range btables {
-			if _, ok := tm.deleted[t.file.Num()]; ok {
-				continue
-			}
-			if _, ok := tm.added[t.file.Num()]; ok {
-				continue
-			}
-			nt = append(nt, t)
-		}
-
-		// New tables.
-		for _, r := range tm.added {
-			nt = append(nt, p.base.s.tableFileFromRecord(r))
-		}
-
-		// Sort tables.
-		if level == 0 {
-			nt.sortByNum()
-		} else {
-			nt.sortByKey(p.base.s.icmp)
-		}
-		nv.tables[level] = nt
-	}
-
-	// Compute compaction score for new version.
-	nv.computeCompaction()
-
-	return nv
-}
-
-type versionReleaser struct {
-	v    *version
-	once bool
-}
-
-func (vr *versionReleaser) Release() {
-	v := vr.v
-	v.s.vmu.Lock()
-	if !vr.once {
-		v.releaseNB()
-		vr.once = true
-	}
-	v.s.vmu.Unlock()
-}

Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/encode.go

@@ -1,258 +0,0 @@
-// Copyright 2011 The Snappy-Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package snappy
-
-import (
-	"encoding/binary"
-	"io"
-)
-
-// We limit how far copy back-references can go, the same as the C++ code.
-const maxOffset = 1 << 15
-
-// emitLiteral writes a literal chunk and returns the number of bytes written.
-func emitLiteral(dst, lit []byte) int {
-	i, n := 0, uint(len(lit)-1)
-	switch {
-	case n < 60:
-		dst[0] = uint8(n)<<2 | tagLiteral
-		i = 1
-	case n < 1<<8:
-		dst[0] = 60<<2 | tagLiteral
-		dst[1] = uint8(n)
-		i = 2
-	case n < 1<<16:
-		dst[0] = 61<<2 | tagLiteral
-		dst[1] = uint8(n)
-		dst[2] = uint8(n >> 8)
-		i = 3
-	case n < 1<<24:
-		dst[0] = 62<<2 | tagLiteral
-		dst[1] = uint8(n)
-		dst[2] = uint8(n >> 8)
-		dst[3] = uint8(n >> 16)
-		i = 4
-	case int64(n) < 1<<32:
-		dst[0] = 63<<2 | tagLiteral
-		dst[1] = uint8(n)
-		dst[2] = uint8(n >> 8)
-		dst[3] = uint8(n >> 16)
-		dst[4] = uint8(n >> 24)
-		i = 5
-	default:
-		panic("snappy: source buffer is too long")
-	}
-	if copy(dst[i:], lit) != len(lit) {
-		panic("snappy: destination buffer is too short")
-	}
-	return i + len(lit)
-}
-
-// emitCopy writes a copy chunk and returns the number of bytes written.
-func emitCopy(dst []byte, offset, length int) int {
-	i := 0
-	for length > 0 {
-		x := length - 4
-		if 0 <= x && x < 1<<3 && offset < 1<<11 {
-			dst[i+0] = uint8(offset>>8)&0x07<<5 | uint8(x)<<2 | tagCopy1
-			dst[i+1] = uint8(offset)
-			i += 2
-			break
-		}
-
-		x = length
-		if x > 1<<6 {
-			x = 1 << 6
-		}
-		dst[i+0] = uint8(x-1)<<2 | tagCopy2
-		dst[i+1] = uint8(offset)
-		dst[i+2] = uint8(offset >> 8)
-		i += 3
-		length -= x
-	}
-	return i
-}
-
-// Encode returns the encoded form of src. The returned slice may be a sub-
-// slice of dst if dst was large enough to hold the entire encoded block.
-// Otherwise, a newly allocated slice will be returned.
-// It is valid to pass a nil dst.
-func Encode(dst, src []byte) ([]byte, error) {
-	if n := MaxEncodedLen(len(src)); len(dst) < n {
-		dst = make([]byte, n)
-	}
-
-	// The block starts with the varint-encoded length of the decompressed bytes.
-	d := binary.PutUvarint(dst, uint64(len(src)))
-
-	// Return early if src is short.
-	if len(src) <= 4 {
-		if len(src) != 0 {
-			d += emitLiteral(dst[d:], src)
-		}
-		return dst[:d], nil
-	}
-
-	// Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive.
-	const maxTableSize = 1 << 14
-	shift, tableSize := uint(32-8), 1<<8
-	for tableSize < maxTableSize && tableSize < len(src) {
-		shift--
-		tableSize *= 2
-	}
-	var table [maxTableSize]int
-
-	// Iterate over the source bytes.
-	var (
-		s   int // The iterator position.
-		t   int // The last position with the same hash as s.
-		lit int // The start position of any pending literal bytes.
-	)
-	for s+3 < len(src) {
-		// Update the hash table.
-		b0, b1, b2, b3 := src[s], src[s+1], src[s+2], src[s+3]
-		h := uint32(b0) | uint32(b1)<<8 | uint32(b2)<<16 | uint32(b3)<<24
-		p := &table[(h*0x1e35a7bd)>>shift]
-		// We need to to store values in [-1, inf) in table. To save
-		// some initialization time, (re)use the table's zero value
-		// and shift the values against this zero: add 1 on writes,
-		// subtract 1 on reads.
-		t, *p = *p-1, s+1
-		// If t is invalid or src[s:s+4] differs from src[t:t+4], accumulate a literal byte.
-		if t < 0 || s-t >= maxOffset || b0 != src[t] || b1 != src[t+1] || b2 != src[t+2] || b3 != src[t+3] {
-			s++
-			continue
-		}
-		// Otherwise, we have a match. First, emit any pending literal bytes.
-		if lit != s {
-			d += emitLiteral(dst[d:], src[lit:s])
-		}
-		// Extend the match to be as long as possible.
-		s0 := s
-		s, t = s+4, t+4
-		for s < len(src) && src[s] == src[t] {
-			s++
-			t++
-		}
-		// Emit the copied bytes.
-		d += emitCopy(dst[d:], s-t, s-s0)
-		lit = s
-	}
-
-	// Emit any final pending literal bytes and return.
-	if lit != len(src) {
-		d += emitLiteral(dst[d:], src[lit:])
-	}
-	return dst[:d], nil
-}
-
-// MaxEncodedLen returns the maximum length of a snappy block, given its
-// uncompressed length.
-func MaxEncodedLen(srcLen int) int {
-	// Compressed data can be defined as:
-	//    compressed := item* literal*
-	//    item       := literal* copy
-	//
-	// The trailing literal sequence has a space blowup of at most 62/60
-	// since a literal of length 60 needs one tag byte + one extra byte
-	// for length information.
-	//
-	// Item blowup is trickier to measure. Suppose the "copy" op copies
-	// 4 bytes of data. Because of a special check in the encoding code,
-	// we produce a 4-byte copy only if the offset is < 65536. Therefore
-	// the copy op takes 3 bytes to encode, and this type of item leads
-	// to at most the 62/60 blowup for representing literals.
-	//
-	// Suppose the "copy" op copies 5 bytes of data. If the offset is big
-	// enough, it will take 5 bytes to encode the copy op. Therefore the
-	// worst case here is a one-byte literal followed by a five-byte copy.
-	// That is, 6 bytes of input turn into 7 bytes of "compressed" data.
-	//
-	// This last factor dominates the blowup, so the final estimate is:
-	return 32 + srcLen + srcLen/6
-}
-
-// NewWriter returns a new Writer that compresses to w, using the framing
-// format described at
-// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
-func NewWriter(w io.Writer) *Writer {
-	return &Writer{
-		w:   w,
-		enc: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)),
-	}
-}
-
-// Writer is an io.Writer than can write Snappy-compressed bytes.
-type Writer struct {
-	w           io.Writer
-	err         error
-	enc         []byte
-	buf         [checksumSize + chunkHeaderSize]byte
-	wroteHeader bool
-}
-
-// Reset discards the writer's state and switches the Snappy writer to write to
-// w. This permits reusing a Writer rather than allocating a new one.
-func (w *Writer) Reset(writer io.Writer) {
-	w.w = writer
-	w.err = nil
-	w.wroteHeader = false
-}
-
-// Write satisfies the io.Writer interface.
-func (w *Writer) Write(p []byte) (n int, errRet error) {
-	if w.err != nil {
-		return 0, w.err
-	}
-	if !w.wroteHeader {
-		copy(w.enc, magicChunk)
-		if _, err := w.w.Write(w.enc[:len(magicChunk)]); err != nil {
-			w.err = err
-			return n, err
-		}
-		w.wroteHeader = true
-	}
-	for len(p) > 0 {
-		var uncompressed []byte
-		if len(p) > maxUncompressedChunkLen {
-			uncompressed, p = p[:maxUncompressedChunkLen], p[maxUncompressedChunkLen:]
-		} else {
-			uncompressed, p = p, nil
-		}
-		checksum := crc(uncompressed)
-
-		// Compress the buffer, discarding the result if the improvement
-		// isn't at least 12.5%.
-		chunkType := uint8(chunkTypeCompressedData)
-		chunkBody, err := Encode(w.enc, uncompressed)
-		if err != nil {
-			w.err = err
-			return n, err
-		}
-		if len(chunkBody) >= len(uncompressed)-len(uncompressed)/8 {
-			chunkType, chunkBody = chunkTypeUncompressedData, uncompressed
-		}
-
-		chunkLen := 4 + len(chunkBody)
-		w.buf[0] = chunkType
-		w.buf[1] = uint8(chunkLen >> 0)
-		w.buf[2] = uint8(chunkLen >> 8)
-		w.buf[3] = uint8(chunkLen >> 16)
-		w.buf[4] = uint8(checksum >> 0)
-		w.buf[5] = uint8(checksum >> 8)
-		w.buf[6] = uint8(checksum >> 16)
-		w.buf[7] = uint8(checksum >> 24)
-		if _, err = w.w.Write(w.buf[:]); err != nil {
-			w.err = err
-			return n, err
-		}
-		if _, err = w.w.Write(chunkBody); err != nil {
-			w.err = err
-			return n, err
-		}
-		n += len(uncompressed)
-	}
-	return n, nil
-}

Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy_test.go

@@ -1,364 +0,0 @@
-// Copyright 2011 The Snappy-Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package snappy
-
-import (
-	"bytes"
-	"flag"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"math/rand"
-	"net/http"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-)
-
-var (
-	download = flag.Bool("download", false, "If true, download any missing files before running benchmarks")
-	testdata = flag.String("testdata", "testdata", "Directory containing the test data")
-)
-
-func roundtrip(b, ebuf, dbuf []byte) error {
-	e, err := Encode(ebuf, b)
-	if err != nil {
-		return fmt.Errorf("encoding error: %v", err)
-	}
-	d, err := Decode(dbuf, e)
-	if err != nil {
-		return fmt.Errorf("decoding error: %v", err)
-	}
-	if !bytes.Equal(b, d) {
-		return fmt.Errorf("roundtrip mismatch:\n\twant %v\n\tgot  %v", b, d)
-	}
-	return nil
-}
-
-func TestEmpty(t *testing.T) {
-	if err := roundtrip(nil, nil, nil); err != nil {
-		t.Fatal(err)
-	}
-}
-
-func TestSmallCopy(t *testing.T) {
-	for _, ebuf := range [][]byte{nil, make([]byte, 20), make([]byte, 64)} {
-		for _, dbuf := range [][]byte{nil, make([]byte, 20), make([]byte, 64)} {
-			for i := 0; i < 32; i++ {
-				s := "aaaa" + strings.Repeat("b", i) + "aaaabbbb"
-				if err := roundtrip([]byte(s), ebuf, dbuf); err != nil {
-					t.Errorf("len(ebuf)=%d, len(dbuf)=%d, i=%d: %v", len(ebuf), len(dbuf), i, err)
-				}
-			}
-		}
-	}
-}
-
-func TestSmallRand(t *testing.T) {
-	rng := rand.New(rand.NewSource(27354294))
-	for n := 1; n < 20000; n += 23 {
-		b := make([]byte, n)
-		for i := range b {
-			b[i] = uint8(rng.Uint32())
-		}
-		if err := roundtrip(b, nil, nil); err != nil {
-			t.Fatal(err)
-		}
-	}
-}
-
-func TestSmallRegular(t *testing.T) {
-	for n := 1; n < 20000; n += 23 {
-		b := make([]byte, n)
-		for i := range b {
-			b[i] = uint8(i%10 + 'a')
-		}
-		if err := roundtrip(b, nil, nil); err != nil {
-			t.Fatal(err)
-		}
-	}
-}
-
-func cmp(a, b []byte) error {
-	if len(a) != len(b) {
-		return fmt.Errorf("got %d bytes, want %d", len(a), len(b))
-	}
-	for i := range a {
-		if a[i] != b[i] {
-			return fmt.Errorf("byte #%d: got 0x%02x, want 0x%02x", i, a[i], b[i])
-		}
-	}
-	return nil
-}
-
-func TestFramingFormat(t *testing.T) {
-	// src is comprised of alternating 1e5-sized sequences of random
-	// (incompressible) bytes and repeated (compressible) bytes. 1e5 was chosen
-	// because it is larger than maxUncompressedChunkLen (64k).
-	src := make([]byte, 1e6)
-	rng := rand.New(rand.NewSource(1))
-	for i := 0; i < 10; i++ {
-		if i%2 == 0 {
-			for j := 0; j < 1e5; j++ {
-				src[1e5*i+j] = uint8(rng.Intn(256))
-			}
-		} else {
-			for j := 0; j < 1e5; j++ {
-				src[1e5*i+j] = uint8(i)
-			}
-		}
-	}
-
-	buf := new(bytes.Buffer)
-	if _, err := NewWriter(buf).Write(src); err != nil {
-		t.Fatalf("Write: encoding: %v", err)
-	}
-	dst, err := ioutil.ReadAll(NewReader(buf))
-	if err != nil {
-		t.Fatalf("ReadAll: decoding: %v", err)
-	}
-	if err := cmp(dst, src); err != nil {
-		t.Fatal(err)
-	}
-}
-
-func TestReaderReset(t *testing.T) {
-	gold := bytes.Repeat([]byte("All that is gold does not glitter,\n"), 10000)
-	buf := new(bytes.Buffer)
-	if _, err := NewWriter(buf).Write(gold); err != nil {
-		t.Fatalf("Write: %v", err)
-	}
-	encoded, invalid, partial := buf.String(), "invalid", "partial"
-	r := NewReader(nil)
-	for i, s := range []string{encoded, invalid, partial, encoded, partial, invalid, encoded, encoded} {
-		if s == partial {
-			r.Reset(strings.NewReader(encoded))
-			if _, err := r.Read(make([]byte, 101)); err != nil {
-				t.Errorf("#%d: %v", i, err)
-				continue
-			}
-			continue
-		}
-		r.Reset(strings.NewReader(s))
-		got, err := ioutil.ReadAll(r)
-		switch s {
-		case encoded:
-			if err != nil {
-				t.Errorf("#%d: %v", i, err)
-				continue
-			}
-			if err := cmp(got, gold); err != nil {
-				t.Errorf("#%d: %v", i, err)
-				continue
-			}
-		case invalid:
-			if err == nil {
-				t.Errorf("#%d: got nil error, want non-nil", i)
-				continue
-			}
-		}
-	}
-}
-
-func TestWriterReset(t *testing.T) {
-	gold := bytes.Repeat([]byte("Not all those who wander are lost;\n"), 10000)
-	var gots, wants [][]byte
-	const n = 20
-	w, failed := NewWriter(nil), false
-	for i := 0; i <= n; i++ {
-		buf := new(bytes.Buffer)
-		w.Reset(buf)
-		want := gold[:len(gold)*i/n]
-		if _, err := w.Write(want); err != nil {
-			t.Errorf("#%d: Write: %v", i, err)
-			failed = true
-			continue
-		}
-		got, err := ioutil.ReadAll(NewReader(buf))
-		if err != nil {
-			t.Errorf("#%d: ReadAll: %v", i, err)
-			failed = true
-			continue
-		}
-		gots = append(gots, got)
-		wants = append(wants, want)
-	}
-	if failed {
-		return
-	}
-	for i := range gots {
-		if err := cmp(gots[i], wants[i]); err != nil {
-			t.Errorf("#%d: %v", i, err)
-		}
-	}
-}
-
-func benchDecode(b *testing.B, src []byte) {
-	encoded, err := Encode(nil, src)
-	if err != nil {
-		b.Fatal(err)
-	}
-	// Bandwidth is in amount of uncompressed data.
-	b.SetBytes(int64(len(src)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		Decode(src, encoded)
-	}
-}
-
-func benchEncode(b *testing.B, src []byte) {
-	// Bandwidth is in amount of uncompressed data.
-	b.SetBytes(int64(len(src)))
-	dst := make([]byte, MaxEncodedLen(len(src)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		Encode(dst, src)
-	}
-}
-
-func readFile(b testing.TB, filename string) []byte {
-	src, err := ioutil.ReadFile(filename)
-	if err != nil {
-		b.Fatalf("failed reading %s: %s", filename, err)
-	}
-	if len(src) == 0 {
-		b.Fatalf("%s has zero length", filename)
-	}
-	return src
-}
-
-// expand returns a slice of length n containing repeated copies of src.
-func expand(src []byte, n int) []byte {
-	dst := make([]byte, n)
-	for x := dst; len(x) > 0; {
-		i := copy(x, src)
-		x = x[i:]
-	}
-	return dst
-}
-
-func benchWords(b *testing.B, n int, decode bool) {
-	// Note: the file is OS-language dependent so the resulting values are not
-	// directly comparable for non-US-English OS installations.
-	data := expand(readFile(b, "/usr/share/dict/words"), n)
-	if decode {
-		benchDecode(b, data)
-	} else {
-		benchEncode(b, data)
-	}
-}
-
-func BenchmarkWordsDecode1e3(b *testing.B) { benchWords(b, 1e3, true) }
-func BenchmarkWordsDecode1e4(b *testing.B) { benchWords(b, 1e4, true) }
-func BenchmarkWordsDecode1e5(b *testing.B) { benchWords(b, 1e5, true) }
-func BenchmarkWordsDecode1e6(b *testing.B) { benchWords(b, 1e6, true) }
-func BenchmarkWordsEncode1e3(b *testing.B) { benchWords(b, 1e3, false) }
-func BenchmarkWordsEncode1e4(b *testing.B) { benchWords(b, 1e4, false) }
-func BenchmarkWordsEncode1e5(b *testing.B) { benchWords(b, 1e5, false) }
-func BenchmarkWordsEncode1e6(b *testing.B) { benchWords(b, 1e6, false) }
-
-// testFiles' values are copied directly from
-// https://raw.githubusercontent.com/google/snappy/master/snappy_unittest.cc
-// The label field is unused in snappy-go.
-var testFiles = []struct {
-	label    string
-	filename string
-}{
-	{"html", "html"},
-	{"urls", "urls.10K"},
-	{"jpg", "fireworks.jpeg"},
-	{"jpg_200", "fireworks.jpeg"},
-	{"pdf", "paper-100k.pdf"},
-	{"html4", "html_x_4"},
-	{"txt1", "alice29.txt"},
-	{"txt2", "asyoulik.txt"},
-	{"txt3", "lcet10.txt"},
-	{"txt4", "plrabn12.txt"},
-	{"pb", "geo.protodata"},
-	{"gaviota", "kppkn.gtb"},
-}
-
-// The test data files are present at this canonical URL.
-const baseURL = "https://raw.githubusercontent.com/google/snappy/master/testdata/"
-
-func downloadTestdata(basename string) (errRet error) {
-	filename := filepath.Join(*testdata, basename)
-	if stat, err := os.Stat(filename); err == nil && stat.Size() != 0 {
-		return nil
-	}
-
-	if !*download {
-		return fmt.Errorf("test data not found; skipping benchmark without the -download flag")
-	}
-	// Download the official snappy C++ implementation reference test data
-	// files for benchmarking.
-	if err := os.Mkdir(*testdata, 0777); err != nil && !os.IsExist(err) {
-		return fmt.Errorf("failed to create testdata: %s", err)
-	}
-
-	f, err := os.Create(filename)
-	if err != nil {
-		return fmt.Errorf("failed to create %s: %s", filename, err)
-	}
-	defer f.Close()
-	defer func() {
-		if errRet != nil {
-			os.Remove(filename)
-		}
-	}()
-	url := baseURL + basename
-	resp, err := http.Get(url)
-	if err != nil {
-		return fmt.Errorf("failed to download %s: %s", url, err)
-	}
-	defer resp.Body.Close()
-	if s := resp.StatusCode; s != http.StatusOK {
-		return fmt.Errorf("downloading %s: HTTP status code %d (%s)", url, s, http.StatusText(s))
-	}
-	_, err = io.Copy(f, resp.Body)
-	if err != nil {
-		return fmt.Errorf("failed to download %s to %s: %s", url, filename, err)
-	}
-	return nil
-}
-
-func benchFile(b *testing.B, n int, decode bool) {
-	if err := downloadTestdata(testFiles[n].filename); err != nil {
-		b.Fatalf("failed to download testdata: %s", err)
-	}
-	data := readFile(b, filepath.Join(*testdata, testFiles[n].filename))
-	if decode {
-		benchDecode(b, data)
-	} else {
-		benchEncode(b, data)
-	}
-}
-
-// Naming convention is kept similar to what snappy's C++ implementation uses.
-func Benchmark_UFlat0(b *testing.B)  { benchFile(b, 0, true) }
-func Benchmark_UFlat1(b *testing.B)  { benchFile(b, 1, true) }
-func Benchmark_UFlat2(b *testing.B)  { benchFile(b, 2, true) }
-func Benchmark_UFlat3(b *testing.B)  { benchFile(b, 3, true) }
-func Benchmark_UFlat4(b *testing.B)  { benchFile(b, 4, true) }
-func Benchmark_UFlat5(b *testing.B)  { benchFile(b, 5, true) }
-func Benchmark_UFlat6(b *testing.B)  { benchFile(b, 6, true) }
-func Benchmark_UFlat7(b *testing.B)  { benchFile(b, 7, true) }
-func Benchmark_UFlat8(b *testing.B)  { benchFile(b, 8, true) }
-func Benchmark_UFlat9(b *testing.B)  { benchFile(b, 9, true) }
-func Benchmark_UFlat10(b *testing.B) { benchFile(b, 10, true) }
-func Benchmark_UFlat11(b *testing.B) { benchFile(b, 11, true) }
-func Benchmark_ZFlat0(b *testing.B)  { benchFile(b, 0, false) }
-func Benchmark_ZFlat1(b *testing.B)  { benchFile(b, 1, false) }
-func Benchmark_ZFlat2(b *testing.B)  { benchFile(b, 2, false) }
-func Benchmark_ZFlat3(b *testing.B)  { benchFile(b, 3, false) }
-func Benchmark_ZFlat4(b *testing.B)  { benchFile(b, 4, false) }
-func Benchmark_ZFlat5(b *testing.B)  { benchFile(b, 5, false) }
-func Benchmark_ZFlat6(b *testing.B)  { benchFile(b, 6, false) }
-func Benchmark_ZFlat7(b *testing.B)  { benchFile(b, 7, false) }
-func Benchmark_ZFlat8(b *testing.B)  { benchFile(b, 8, false) }
-func Benchmark_ZFlat9(b *testing.B)  { benchFile(b, 9, false) }
-func Benchmark_ZFlat10(b *testing.B) { benchFile(b, 10, false) }
-func Benchmark_ZFlat11(b *testing.B) { benchFile(b, 11, false) }

Godeps/_workspace/src/github.com/thejerf/suture/.travis.yml

@@ -1,7 +0,0 @@
-language: go
-go:
-  - 1.1
-  - 1.2
-  - 1.3
-  - 1.4
-  - tip

ISSUE_TEMPLATE.md

@@ -0,0 +1,28 @@
+Do not report security issues in this bug tracker. Instead, contact
+security@syncthing.net directly - see https://syncthing.net/security.html
+for more information.
+
+If your issue is a support request ("How do I get my devices to connect?"
+or similar), please use the support forum at https://forum.syncthing.net/
+where a large number of helpful people hang out. This issue tracker is for
+reporting bugs or feature requests directly to the developers.
+
+If your issue is a bug report, replace this boilerplate with a description
+of the problem, being sure to include at least:
+
+ - what happened,
+ - what you expected to happen instead, and
+ - any steps to reproduce the problem.
+
+Also fill out the version information below and add log output or
+screenshots as appropriate.
+
+If your issue is a feature request, simply replace this template text in
+its entirety.
+
+### Version Information
+
+Syncthing Version: v0.x.y
+OS Version: Windows 7 / Ubuntu 14.04 / ...
+Browser Version: (if applicable, for GUI issues)
+

NICKS

@@ -1,55 +1,114 @@
 # This file maps email addresses used in commits to nicks used the changelog.
+# It is auto generated from the AUTHORS file by script/authors.go.
 
-AudriusButkevicius	<audrius.butkevicius@gmail.com>
-Cathryne	<cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
-KayoticSully	<kayoticsully@gmail.com>
-Moter8		<moter8@gmail.com>
-Nutomic		<me@nutomic.com>
-Rewt0r		<rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
-Vilbrekin	<vilbrekin@gmail.com>
-Zillode		<zillode@zillode.be>
-alex2108	<register-github@alex-graf.de>
-andrew-d	<andrew@du.nham.ca>
-asdil12		<dominik@heidler.eu>
-bencurthoys	<ben@bencurthoys.com>
-bigbear2nd	<bigbear2nd@gmail.com>
-brendanlong	<self@brendanlong.com>
-bsidhom		<bsidhom@gmail.com>
-calmh		<jakob@nym.se>
-cdata		<chris@scriptolo.gy>
-ceh		<emil@hessman.se>
-cqcallaw	<enlightened.despot@gmail.com>
-dzarda		<dzardacz@gmail.com>
-facastagnini	<federico.castagnini@gmail.com>
-filoozoom	<philippe@schommers.be>
-frioux		<frew@afoolishmanifesto.com> <frioux@gmail.com>
-gillisig	<gilli@vx.is>
-jarlebring	<jarlebring@gmail.com>
-jedie		<github.com@jensdiemer.de> <git@jensdiemer.de>
-jpjp		<jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
-kamadak		<kamada@nanohz.org>
-kilburn		<kilburn@la3.org>
-kozec		<kozec@kozec.com>
-krozycki	<rozycki.karol@gmail.com>
-marcindziadus	<dziadus.marcin@gmail.com>
+acogdev <jake@acogdev.com>
+alessandro.g89 <alessandro.g89@gmail.com>
+alex2108 <register-github@alex-graf.de>
+andersonvom <andersonvom@gmail.com>
+andrew-d <andrew@du.nham.ca>
+asdil12 <dominik@heidler.eu>
+AudriusButkevicius <audrius.butkevicius@gmail.com>
+aviau <alexandre@alexandreviau.net>
+aviau <aviau@debian.org>
+bencurthoys <ben@bencurthoys.com>
+bigbear2nd <bigbear2nd@gmail.com>
+brbecker <brbecker@gmail.com>
+brendanlong <self@brendanlong.com>
+brgmnn <dan.arne.bergmann@gmail.com>
+brgmnn <brgmnn@users.noreply.github.com>
+bsidhom <bsidhom@gmail.com>
+buinsky <vix_booja@tut.by>
+burkemw3 <mburke@amplify.com>
+burkemw3 <burkemw3@gmail.com>
+calmh <jakob@nym.se>
+canton7 <antony.male@gmail.com>
+Cathryne <cathryne.linenweaver@gmail.com>
+Cathryne <Cathryne@users.noreply.github.com>
+cdata <chris@scriptolo.gy>
+cdhowie <me@chrishowie.com>
+ceh <emil@hessman.se>
+cqcallaw <enlightened.despot@gmail.com>
+dinosore <dinosore@dbrsoftware.co.uk>
+dva <denisva@gmail.com>
+dzarda <dzardacz@gmail.com>
+eipiminus1 <eipiminusone+github@gmail.com>
+eipiminus1 <eipiminus1@users.noreply.github.com>
+facastagnini <federico.castagnini@gmail.com>
+filoozoom <philippe@schommers.be>
+frioux <frew@afoolishmanifesto.com>
+frioux <frioux@gmail.com>
+fti7 <frank@isemann.name>
+gillisig <gilli@vx.is>
+hadogenes <szafar@linux.pl>
+ironmig <kma1660@gmail.com>
+jarlebring <jarlebring@gmail.com>
+jedie <github.com@jensdiemer.de>
+jedie <git@jensdiemer.de>
+jgke <jgke@jgke.fi>
+jpjp <jamespatterson@operamail.com>
+jpjp <jpjp@users.noreply.github.com>
+kamadak <kamada@nanohz.org>
+KayoticSully <kayoticsully@gmail.com>
+kc1212 <kc04bc@gmx.com>
+kc1212 <kc1212@users.noreply.github.com>
+kilburn <kilburn@la3.org>
+kluppy <kluppy@going2blue.com>
+kozec <kozec@kozec.com>
+kralo <max.schulze@online.de>
+kralo <kralo@users.noreply.github.com>
+krozycki <rozycki.karol@gmail.com>
+letiemble <laurent.etiemble@gmail.com>
+letiemble <laurent.etiemble@monobjc.net>
+lkwg82 <lkwg82@gmx.de>
+LordLandon <lordlandon@gmail.com>
+majedev <majed.alhajry@gmail.com>
+marcindziadus <dziadus.marcin@gmail.com>
 marclaporte <marc@marclaporte.com>
-moshen		<moshen.colin@gmail.com>
-mvdan		<mvdan@mvdan.cc>
-pascalj		<github@pascalj.com> <mail@pascal-jungblut.com>
-peterhoeg	<peter@speartail.com>
-philips		<brandon@ifup.org>
-piobpl		<piotrb10@gmail.com>
-pluby		<phill.luby@newredo.com>
-pyfisch		<pyfisch@gmail.com>
-ralder		<ralder@yandex.ru>
-rumpelsepp	<stefan@sevenbyte.org>
-qbit		<qbit@deftly.net>
-sciurius	<jvromans@squirrel.nl>
-seehuhn		<voss@seehuhn.de>
-snnd		<dw@risu.io>
-timabell	<tim@timwise.co.uk>
-tnn2		<tnn@nygren.pp.se>
-tojrobinson	<tully@tojr.org>
-uok		<ueomkail@gmail.com> <uok@users.noreply.github.com>
-veeti		<veeti.paananen@rojekti.fi>
-zukoo		<fxgsell@gmail.com>
+marclaporte <marc@laporte.name>
+mateon1 <matin1111@wp.pl>
+mogwa1 <devriesb@gmail.com>
+moshen <moshen.colin@gmail.com>
+Moter8 <moter8@gmail.com>
+mvdan <mvdan@mvdan.cc>
+norgeous <daniel@harte.me>
+norgeous <daniel@danielharte.co.uk>
+norgeous <norgeous@users.noreply.github.com>
+nrm21 <natemorrison@gmail.com>
+Nutomic <me@nutomic.com>
+pascalj <github@pascalj.com>
+pascalj <mail@pascal-jungblut.com>
+peterhoeg <peter@speartail.com>
+philips <brandon@ifup.org>
+piobpl <piotrb10@gmail.com>
+plouj <ploujj@gmail.com>
+pluby <phill.luby@newredo.com>
+pyfisch <pyfisch@gmail.com>
+qbit <qbit@deftly.net>
+ralder <ralder@yandex.ru>
+Rewt0r <rewt0r@gmx.com>
+Rewt0r <Rewt0r@users.noreply.github.com>
+rumpelsepp <stefan@sevenbyte.org>
+rumpelsepp <rumpelsepp@sevenbyte.org>
+scienmind <scintertech@cryptolab.net>
+sciurius <jvromans@squirrel.nl>
+seehuhn <voss@seehuhn.de>
+simplypeachy <aD@simplypeachy.co.uk>
+simplypeachy <simplypeachy@users.noreply.github.com>
+snnd <dw@risu.io>
+Stefan-Code <stefan.github@gmail.com>
+Stefan-Code <Stefan.github@gmail.com>
+timabell <tim@timwise.co.uk>
+tnn2 <tnn@nygren.pp.se>
+tojrobinson <tully@tojr.org>
+tpng <benny.tpng@gmail.com>
+tylerbrazier <tyler@tylerbrazier.com>
+uok <ueomkail@gmail.com>
+uok <uok@users.noreply.github.com>
+veeti <veeti.paananen@rojekti.fi>
+Vilbrekin <vilbrekin@gmail.com>
+wkennington <william@wkennington.com>
+WSGCSysadmin <e.meitner@willystreet.coop>
+wweich <wweich@users.noreply.github.com>
+wweich <wweich@gmx.de>
+Zillode <zillode@zillode.be>
+zukoo <fxgsell@gmail.com>

PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,32 @@
+### Purpose
+
+Describe the purpose of this change. If there is an existing issue that is
+resolved by this pull request, ensure that the commit subject is on the form
+`Some short description (fixes #1234)` where 1234 is the issue number.
+
+### Testing
+
+Describe what testing has been done, and how the reviewer can test the change
+if new tests are not included.
+
+### Screenshots
+
+If this is a GUI change, include screenshots of the change. If not, please
+feel free to just delete this section.
+
+### Documentation
+
+If this is a user visible change (including API and protocol changes), add a link here
+to the corresponding pull request on https://github.com/syncthing/docs or describe
+the documentation changes necessary.
+
+### Authorship
+
+Every author of a code contribution (Go, Javascript, HTML, CSS etc, with the
+possible exception of minor typo corrections and similar) is recorded in the
+AUTHORS and NICKS files and the in-GUI credits. If this is your first
+contribution, a maintainer will add you properly before accepting the
+contribution. You need not do so yourself or worry about the fact that the
+"authors" automated test fails. However, if your name (such as you want it
+presented in the credits) is not visible on your Github profile or in your
+commit messages, please assist by providing it here.

README.md

@@ -1,59 +1,68 @@
-syncthing
-=========
+# Syncthing
 
-[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing.svg?style=flat-square)](http://build.syncthing.net/job/syncthing/lastBuild/)
-[![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/syncthing)
-[![MPLv2 License](http://img.shields.io/badge/license-MPLv2-blue.svg?style=flat-square)](https://www.mozilla.org/MPL/2.0/)
+[![Latest Build (Official)](https://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing.svg?style=flat-square&label=unix%20build)](http://build.syncthing.net/job/syncthing/lastBuild/)
+[![API Documentation](https://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/syncthing)
+[![MPLv2 License](https://img.shields.io/badge/license-MPLv2-blue.svg?style=flat-square)](https://www.mozilla.org/MPL/2.0/)
 
-This is the `syncthing` project which pursues the following goals:
+This is the Syncthing project which pursues the following goals:
 
  1. Define a protocol for synchronization of a folder between a number of
     collaborating devices. This protocol should be well defined, unambiguous,
     easily understood, free to use, efficient, secure and language neutral.
-    This is called the [Block Exchange
-    Protocol](https://github.com/syncthing/specs/blob/master/BEPv1.md).
+    This is called the [Block Exchange Protocol][1].
 
  2. Provide the reference implementation to demonstrate the usability of
     said protocol. This is the `syncthing` utility. We hope that
     alternative, compatible implementations of the protocol will arise.
 
 The two are evolving together; the protocol is not to be considered
-stable until syncthing 1.0 is released, at which point it is locked down
+stable until Syncthing 1.0 is released, at which point it is locked down
 for incompatible changes.
 
-Getting Started
----------------
+## Getting Started
 
-Take a look at the [getting started
-guide](https://github.com/syncthing/syncthing/wiki/Getting-Started).
+Take a look at the [getting started guide][2].
 
-There are a few examples for keeping syncthing running in the background
-on your system in [the etc directory](https://github.com/syncthing/syncthing/blob/master/etc).
+There are a few examples for keeping Syncthing running in the background
+on your system in [the etc directory][3]. There are also several [GUI
+implementations][11] for Windows, Mac and Linux.
 
-There is an IRC channel, `#syncthing` on Freenode, for talking directly
-to developers and users.
+## Getting in Touch
 
-Building
---------
+The first and best point of contact is the [Forum][8]. There is also an IRC
+channel, `#syncthing` on [freenode][4] (with a [web client][9]), for talking
+directly to developers and users. If you've found something that is clearly a
+bug, feel free to report it in the [GitHub issue tracker][10].
 
-Building Syncthing from source is easy, and there's a
-[guide](https://github.com/syncthing/syncthing/wiki/Building).
+## Building
+
+Building Syncthing from source is easy, and there's a [guide][5]
 that describes it for both Unix and Windows systems.
 
-Signed Releases
----------------
+## Signed Releases
 
-As of v0.10.15 and onwards, git tags and release binaries are GPG signed
-with the key D26E6ED000654A3E (see https://syncthing.net/security.html).
-For release binaries, MD5 and SHA1 checksums are calculated and signed,
-available in the md5sum.txt.asc and sha1sum.txt.asc files.
+As of v0.10.15 and onwards release binaries are GPG signed with the key
+D26E6ED000654A3E, available from https://syncthing.net/security.html and
+most key servers.
 
-Documentation
-=============
+There is also a built in automatic upgrade mechanism (disabled in some
+distribution channels) which uses a compiled in ECDSA signature. Mac OS
+X binaries are also properly code signed.
 
-The [syncthing
-documentation](https://github.com/syncthing/syncthing/wiki/) is on the
-Github wiki.
+## Documentation
 
-All code is licensed under the
-[MPLv2 License](https://github.com/syncthing/syncthing/blob/master/LICENSE).
+Please see the [Syncthing documentation site][6].
+
+All code is licensed under the [MPLv2 License][7].
+
+[1]: http://docs.syncthing.net/specs/bep-v1.html
+[2]: http://docs.syncthing.net/intro/getting-started.html
+[3]: https://github.com/syncthing/syncthing/blob/master/etc
+[4]: http://www.freenode.net/irc_servers.shtml
+[5]: http://docs.syncthing.net/dev/building.html
+[6]: http://docs.syncthing.net/
+[7]: https://github.com/syncthing/syncthing/blob/master/LICENSE
+[8]: https://forum.syncthing.net/
+[9]: https://kiwiirc.com/client/irc.freenode.net/#syncthing
+[10]: https://github.com/syncthing/syncthing/issues
+[11]: http://docs.syncthing.net/users/contrib.html#gui-wrappers

authors.go

@@ -1,58 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-// +build ignore
-
-// Generates the list of contributors in gui/index.html based on contents of
-// AUTHORS.
-
-package main
-
-import (
-	"io/ioutil"
-	"log"
-	"os"
-	"regexp"
-	"sort"
-	"strings"
-)
-
-func main() {
-	bs := readAll("AUTHORS")
-	lines := strings.Split(string(bs), "\n")
-	nameRe := regexp.MustCompile(`(.+?)\s+<`)
-	authors := make([]string, 0, len(lines))
-	for _, line := range lines {
-		if m := nameRe.FindStringSubmatch(line); len(m) == 2 {
-			authors = append(authors, "          <li class=\"auto-generated\">"+m[1]+"</li>")
-		}
-	}
-	sort.Strings(authors)
-	replacement := strings.Join(authors, "\n")
-
-	authorsRe := regexp.MustCompile(`(?s)id="contributor-list">.*?</ul>`)
-	bs = readAll("gui/index.html")
-	bs = authorsRe.ReplaceAll(bs, []byte("id=\"contributor-list\">\n"+replacement+"\n        </ul>"))
-
-	if err := ioutil.WriteFile("gui/index.html", bs, 0644); err != nil {
-		log.Fatal(err)
-	}
-}
-
-func readAll(path string) []byte {
-	fd, err := os.Open(path)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer fd.Close()
-
-	bs, err := ioutil.ReadAll(fd)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	return bs
-}

build.sh

@@ -4,79 +4,93 @@ IFS=$'\n\t'
 
 STTRACE=${STTRACE:-}
 
+script() {
+	name="$1"
+	shift
+	go run "script/$name.go" "$@"
+}
+
+build() {
+	go run build.go "$@"
+}
+
 case "${1:-default}" in
 	default)
-		go run build.go
+		build
 		;;
 
 	clean)
-		go run build.go "$1"
-		;;
-
-	test)
-		ulimit -t 60 &>/dev/null || true
-		ulimit -d 512000 &>/dev/null || true
-		ulimit -m 512000 &>/dev/null || true
-
-		go run build.go "$1"
+		build "$@"
 		;;
 
 	tar)
-		go run build.go "$1"
-		;;
-
-	deps)
-		go run build.go "$1"
+		build "$@"
 		;;
 
 	assets)
-		go run build.go "$1"
+		build "$@"
 		;;
 
 	xdr)
-		go run build.go "$1"
+		build "$@"
 		;;
 
 	translate)
-		go run build.go "$1"
+		build "$@"
 		;;
 
-	transifex)
-		go run build.go "$1"
-		;;
-
-	noupgrade)
-		go run build.go -no-upgrade tar
-		;;
-
-	all)
-		go run build.go -goos darwin -goarch amd64 tar
-		go run build.go -goos darwin -goarch 386 tar
-
-		go run build.go -goos dragonfly -goarch 386 tar
-		go run build.go -goos dragonfly -goarch amd64 tar
-
-		go run build.go -goos freebsd -goarch 386 tar
-		go run build.go -goos freebsd -goarch amd64 tar
-
-		go run build.go -goos linux -goarch 386 tar
-		go run build.go -goos linux -goarch amd64 tar
-		go run build.go -goos linux -goarch arm tar
-
-		go run build.go -goos netbsd -goarch 386 tar
-		go run build.go -goos netbsd -goarch amd64 tar
-
-		go run build.go -goos openbsd -goarch 386 tar
-		go run build.go -goos openbsd -goarch amd64 tar
-
-		go run build.go -goos solaris -goarch amd64 tar
-
-		go run build.go -goos windows -goarch 386 zip
-		go run build.go -goos windows -goarch amd64 zip
+	deb)
+		build "$@"
 		;;
 
 	setup)
-		echo "Don't worry, just build."
+		build "$@"
+		;;
+
+	test)
+		ulimit -t 600 &>/dev/null || true
+		ulimit -d 512000 &>/dev/null || true
+		ulimit -m 512000 &>/dev/null || true
+		LOGGER_DISCARD=1 build test
+		;;
+
+	bench)
+		LOGGER_DISCARD=1 build bench | script benchfilter
+		;;
+
+	prerelease)
+		go run script/authors.go
+		build transifex
+		git add -A gui/default/assets/ lib/auto/
+		pushd man ; ./refresh.sh ; popd
+		git add -A man
+		;;
+
+	noupgrade)
+		build -no-upgrade tar
+		;;
+
+	all)
+		platforms=(
+			darwin-amd64 dragonfly-amd64 freebsd-amd64 linux-amd64 netbsd-amd64 openbsd-amd64 solaris-amd64 windows-amd64
+			freebsd-386 linux-386 netbsd-386 openbsd-386 windows-386
+			linux-arm linux-arm64 linux-ppc64 linux-ppc64le
+		)
+
+		for plat in "${platforms[@]}"; do
+			echo Building "$plat"
+
+			goos="${plat%-*}"
+			goarch="${plat#*-}"
+			dist="tar"
+
+			if [[ $goos == "windows" ]]; then
+				dist="zip"
+			fi
+
+			build -goos "$goos" -goarch "$goarch" "$dist"
+			echo
+		done
 		;;
 
 	test-cov)
@@ -84,19 +98,16 @@ case "${1:-default}" in
 		ulimit -d 512000 &>/dev/null || true
 		ulimit -m 512000 &>/dev/null || true
 
-		go get github.com/axw/gocov/gocov
-		go get github.com/AlekSi/gocov-xml
-
 		echo "mode: set" > coverage.out
 		fail=0
 
 		# For every package in the repo
-		for dir in $(go list ./...) ; do
+		for dir in $(go list ./lib/... ./cmd/...) ; do
 			# run the tests
-			godep go test -coverprofile=profile.out $dir
+			GOPATH="$(pwd)/Godeps/_workspace:$GOPATH" go test -race -coverprofile=profile.out $dir
 			if [ -f profile.out ] ; then
 				# and if there was test output, append it to coverage.out
-				grep -v "mode: set" profile.out >> coverage.out
+				grep -v "mode: " profile.out >> coverage.out
 				rm profile.out
 			fi
 		done
@@ -111,23 +122,35 @@ case "${1:-default}" in
 		fi
 		;;
 
+	test-xunit)
+		ulimit -t 600 &>/dev/null || true
+		ulimit -d 512000 &>/dev/null || true
+		ulimit -m 512000 &>/dev/null || true
+
+		(GOPATH="$(pwd)/Godeps/_workspace:$GOPATH" go test -v -race ./lib/... ./cmd/... || true) > tests.out
+		go2xunit -output tests.xml -fail < tests.out
+		;;
+
 	docker-all)
+		img=${DOCKERIMG:-syncthing/build:latest}
 		docker run --rm -h syncthing-builder -u $(id -u) -t \
 			-v $(pwd):/go/src/github.com/syncthing/syncthing \
 			-w /go/src/github.com/syncthing/syncthing \
 			-e "STTRACE=$STTRACE" \
-			syncthing/build:latest \
+			"$img" \
 			sh -c './build.sh clean \
-				&& ./build.sh all \
-				&& STTRACE=all ./build.sh test-cov'
+				&& ./build.sh test-cov \
+				&& ./build.sh bench \
+				&& ./build.sh all'
 		;;
 
 	docker-test)
+		img=${DOCKERIMG:-syncthing/build:latest}
 		docker run --rm -h syncthing-builder -u $(id -u) -t \
 			-v $(pwd):/go/src/github.com/syncthing/syncthing \
 			-w /go/src/github.com/syncthing/syncthing \
 			-e "STTRACE=$STTRACE" \
-			syncthing/build:latest \
+			"$img" \
 			sh -euxc './build.sh clean \
 				&& go run build.go -race \
 				&& export GOPATH=$(pwd)/Godeps/_workspace:$GOPATH \
@@ -137,21 +160,23 @@ case "${1:-default}" in
 		;;
 
 	docker-lint)
+		img=${DOCKERIMG:-syncthing/build:latest}
 		docker run --rm -h syncthing-builder -u $(id -u) -t \
 			-v $(pwd):/go/src/github.com/syncthing/syncthing \
 			-w /go/src/github.com/syncthing/syncthing \
 			-e "STTRACE=$STTRACE" \
-			syncthing/build:latest \
+			"$img" \
 			sh -euxc 'go run build.go lint'
 		;;
 
 
 	docker-vet)
+		img=${DOCKERIMG:-syncthing/build:latest}
 		docker run --rm -h syncthing-builder -u $(id -u) -t \
 			-v $(pwd):/go/src/github.com/syncthing/syncthing \
 			-w /go/src/github.com/syncthing/syncthing \
 			-e "STTRACE=$STTRACE" \
-			syncthing/build:latest \
+			"$img" \
 			sh -euxc 'go run build.go vet'
 		;;
 

check-contrib.sh

@@ -1,51 +0,0 @@
-#!/bin/bash
-
-missing-authors() {
-	for email in $(git log --format=%ae HEAD | sort | uniq) ; do
-		grep -q "$email" AUTHORS || echo $email
-	done
-}
-
-no-docs-typos() {
-	# Commits that are known to not change code
-	grep -v 63bd0136fb40a91efaa279cb4b4159d82e8e6904 |\
-	grep -v 4e2feb6fbc791bb8a2daf0ab8efb10775d66343e |\
-	grep -v f2459ef3319b2f060dbcdacd0c35a1788a94b8bd |\
-	grep -v b61f418bf2d1f7d5a9d7088a20a2a448e5e66801 |\
-	grep -v f0621207e3953711f9ab86d99724f1d0faac45b1 |\
-	grep -v f1120d7aa936c0658429edef0037792520b46334 |\
-	grep -v a9339d0627fff439879d157c75077f02c9fac61b |\
-	grep -v 254c63763a3ad42fd82259f1767db526cff94a14 |\
-	grep -v 4b76ec40c07078beaa2c5e250ed7d9bd6276a718 |\
-	grep -v ffc39dfbcb34eacc3ea12327a02b6e7741a2c207 |\
-	grep -v 32a76901a91ff0f663db6f0830e0aedec946e4d0
-}
-
-print-missing-authors() {
-	for email in $(missing-authors) ; do
-		git log --author="$email" --format="%H %ae %s" | no-docs-typos
-	done
-}
-
-print-missing-copyright() {
-	find . -name \*.go | xargs egrep -L 'Copyright \(C\)|automatically generated' | grep -v Godeps | grep -v internal/auto/
-}
-
-authors=$(print-missing-authors)
-if [[ ! -z $authors ]] ; then
-	echo '***'
-	echo Author emails not in AUTHORS:
-	echo $authors
-	echo '***'
-	exit 1
-fi
-
-copy=$(print-missing-copyright)
-if [[ ! -z $copy ]] ; then
-	echo ***
-	echo Files missing copyright notice:
-	echo $copy
-	echo ***
-	exit 1
-fi
-

cmd/discosrv/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2014-2015 The Discosrv Authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+- The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

cmd/discosrv/README.md

@@ -0,0 +1,40 @@
+discosrv
+========
+
+[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/discosrv.svg?style=flat-square)](http://build.syncthing.net/job/discosrv/lastBuild/)
+
+This is the global discovery server for the `syncthing` project.
+
+To get it, run `go get github.com/syncthing/discosrv` or download the
+[latest build](http://build.syncthing.net/job/discosrv/lastSuccessfulBuild/artifact/)
+from the build server.
+
+Usage
+-----
+
+The discovery server supports `ql` and `postgres` backends.
+Specify the backend via `-db-backend` and the database DSN via `-db-dsn`.
+
+By default it will use in-memory `ql` backend. If you wish to persist the
+information on disk between restarts in `ql`, specify a file DSN:
+
+```bash
+$ discosrv -db-dsn="file:///var/run/discosrv.db"
+```
+
+For `postgres`, you will need to create a database and a user with permissions
+to create tables in it, then start the discosrv as follows:
+
+```bash
+$ export DISCOSRV_DB_DSN="postgres://user:password@localhost/databasename"
+$ discosrv -db-backend="postgres"
+```
+
+You can pass the DSN as command line option, but the value what you pass in will
+be visible in most process managers, potentially exposing the database password
+to other users.
+
+In all cases, the appropriate tables and indexes will be created at first
+startup. If it doesn't exit with an error, you're fine.
+
+See `discosrv -help` for other options.

cmd/discosrv/clean.go

@@ -0,0 +1,75 @@
+// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"database/sql"
+	"log"
+	"time"
+)
+
+type cleansrv struct {
+	intv time.Duration
+	db   *sql.DB
+	prep map[string]*sql.Stmt
+}
+
+func (s *cleansrv) Serve() {
+	for {
+		time.Sleep(next(s.intv))
+
+		err := s.cleanOldEntries()
+		if err != nil {
+			log.Println("Clean:", err)
+		}
+	}
+}
+
+func (s *cleansrv) Stop() {
+	panic("stop unimplemented")
+}
+
+func (s *cleansrv) cleanOldEntries() (err error) {
+	var tx *sql.Tx
+	tx, err = s.db.Begin()
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err == nil {
+			err = tx.Commit()
+		} else {
+			tx.Rollback()
+		}
+	}()
+
+	res, err := tx.Stmt(s.prep["cleanAddress"]).Exec()
+	if err != nil {
+		return err
+	}
+	if rows, _ := res.RowsAffected(); rows > 0 {
+		log.Printf("Clean: %d old addresses", rows)
+	}
+
+	res, err = tx.Stmt(s.prep["cleanDevice"]).Exec()
+	if err != nil {
+		return err
+	}
+	if rows, _ := res.RowsAffected(); rows > 0 {
+		log.Printf("Clean: %d old devices", rows)
+	}
+
+	var devs, addrs int
+	row := tx.Stmt(s.prep["countDevice"]).QueryRow()
+	if err = row.Scan(&devs); err != nil {
+		return err
+	}
+	row = tx.Stmt(s.prep["countAddress"]).QueryRow()
+	if err = row.Scan(&addrs); err != nil {
+		return err
+	}
+
+	log.Printf("Database: %d devices, %d addresses", devs, addrs)
+	return nil
+}

cmd/discosrv/db.go

@@ -0,0 +1,32 @@
+// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+type setupFunc func(db *sql.DB) error
+type compileFunc func(db *sql.DB) (map[string]*sql.Stmt, error)
+
+var (
+	setupFuncs   = make(map[string]setupFunc)
+	compileFuncs = make(map[string]compileFunc)
+)
+
+func register(name string, setup setupFunc, compile compileFunc) {
+	setupFuncs[name] = setup
+	compileFuncs[name] = compile
+}
+
+func setup(backend string, db *sql.DB) (map[string]*sql.Stmt, error) {
+	setup, ok := setupFuncs[backend]
+	if !ok {
+		return nil, fmt.Errorf("Unsupported backend")
+	}
+	if err := setup(db); err != nil {
+		return nil, err
+	}
+	return compileFuncs[backend](db)
+}

cmd/discosrv/main.go

@@ -0,0 +1,141 @@
+// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"crypto/tls"
+	"database/sql"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"runtime"
+	"strconv"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/thejerf/suture"
+)
+
+const (
+	minNegCache  = 60        // seconds
+	maxNegCache  = 3600      // seconds
+	maxDeviceAge = 7 * 86400 // one week, in seconds
+)
+
+var (
+	Version    string
+	BuildStamp string
+	BuildUser  string
+	BuildHost  string
+
+	BuildDate   time.Time
+	LongVersion string
+)
+
+func init() {
+	stamp, _ := strconv.Atoi(BuildStamp)
+	BuildDate = time.Unix(int64(stamp), 0)
+
+	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
+	LongVersion = fmt.Sprintf(`discosrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
+}
+
+var (
+	lruSize     = 10240
+	limitAvg    = 5
+	limitBurst  = 20
+	globalStats stats
+	statsFile   string
+	backend     = "ql"
+	dsn         = getEnvDefault("DISCOSRV_DB_DSN", "memory://discosrv")
+	certFile    = "cert.pem"
+	keyFile     = "key.pem"
+	debug       = false
+	useHTTP     = false
+)
+
+func main() {
+	const (
+		cleanIntv = 1 * time.Hour
+		statsIntv = 5 * time.Minute
+	)
+
+	var listen string
+
+	log.SetOutput(os.Stdout)
+	log.SetFlags(0)
+
+	flag.StringVar(&listen, "listen", ":8443", "Listen address")
+	flag.IntVar(&lruSize, "limit-cache", lruSize, "Limiter cache entries")
+	flag.IntVar(&limitAvg, "limit-avg", limitAvg, "Allowed average package rate, per 10 s")
+	flag.IntVar(&limitBurst, "limit-burst", limitBurst, "Allowed burst size, packets")
+	flag.StringVar(&statsFile, "stats-file", statsFile, "File to write periodic operation stats to")
+	flag.StringVar(&backend, "db-backend", backend, "Database backend to use")
+	flag.StringVar(&dsn, "db-dsn", dsn, "Database DSN")
+	flag.StringVar(&certFile, "cert", certFile, "Certificate file")
+	flag.StringVar(&keyFile, "key", keyFile, "Key file")
+	flag.BoolVar(&debug, "debug", debug, "Debug")
+	flag.BoolVar(&useHTTP, "http", useHTTP, "Listen on HTTP (behind an HTTPS proxy)")
+	flag.Parse()
+
+	log.Println(LongVersion)
+
+	var cert tls.Certificate
+	var err error
+	if !useHTTP {
+		cert, err = tls.LoadX509KeyPair(certFile, keyFile)
+		if err != nil {
+			log.Fatalln("Failed to load X509 key pair:", err)
+		}
+
+		devID := protocol.NewDeviceID(cert.Certificate[0])
+		log.Println("Server device ID is", devID)
+	}
+
+	db, err := sql.Open(backend, dsn)
+	if err != nil {
+		log.Fatalln("sql.Open:", err)
+	}
+	prep, err := setup(backend, db)
+	if err != nil {
+		log.Fatalln("Setup:", err)
+	}
+
+	main := suture.NewSimple("main")
+
+	main.Add(&querysrv{
+		addr: listen,
+		cert: cert,
+		db:   db,
+		prep: prep,
+	})
+
+	main.Add(&cleansrv{
+		intv: cleanIntv,
+		db:   db,
+		prep: prep,
+	})
+
+	main.Add(&statssrv{
+		intv: statsIntv,
+		file: statsFile,
+		db:   db,
+	})
+
+	globalStats.Reset()
+	main.Serve()
+}
+
+func getEnvDefault(key, def string) string {
+	if val := os.Getenv(key); val != "" {
+		return val
+	}
+	return def
+}
+
+func next(intv time.Duration) time.Duration {
+	t0 := time.Now()
+	t1 := t0.Add(intv).Truncate(intv)
+	return t1.Sub(t0)
+}

cmd/discosrv/psql.go

@@ -0,0 +1,97 @@
+// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"database/sql"
+	"fmt"
+
+	_ "github.com/lib/pq"
+)
+
+func init() {
+	register("postgres", postgresSetup, postgresCompile)
+}
+
+func postgresSetup(db *sql.DB) error {
+	var err error
+
+	db.SetMaxIdleConns(4)
+	db.SetMaxOpenConns(8)
+
+	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS Devices (
+		DeviceID CHAR(63) NOT NULL PRIMARY KEY,
+		Seen TIMESTAMP NOT NULL
+	)`)
+	if err != nil {
+		return err
+	}
+
+	row := db.QueryRow(`SELECT 'DevicesDeviceIDIndex'::regclass`)
+	if err := row.Scan(nil); err != nil {
+		_, err = db.Exec(`CREATE INDEX DevicesDeviceIDIndex ON Devices (DeviceID)`)
+	}
+	if err != nil {
+		return err
+	}
+
+	row = db.QueryRow(`SELECT 'DevicesSeenIndex'::regclass`)
+	if err := row.Scan(nil); err != nil {
+		_, err = db.Exec(`CREATE INDEX DevicesSeenIndex ON Devices (Seen)`)
+	}
+	if err != nil {
+		return err
+	}
+
+	_, err = db.Exec(`CREATE TABLE IF NOT EXISTS Addresses (
+		DeviceID CHAR(63) NOT NULL,
+		Seen TIMESTAMP NOT NULL,
+		Address VARCHAR(256) NOT NULL
+	)`)
+	if err != nil {
+		return err
+	}
+
+	row = db.QueryRow(`SELECT 'AddressesDeviceIDSeenIndex'::regclass`)
+	if err := row.Scan(nil); err != nil {
+		_, err = db.Exec(`CREATE INDEX AddressesDeviceIDSeenIndex ON Addresses (DeviceID, Seen)`)
+	}
+	if err != nil {
+		return err
+	}
+
+	row = db.QueryRow(`SELECT 'AddressesDeviceIDAddressIndex'::regclass`)
+	if err := row.Scan(nil); err != nil {
+		_, err = db.Exec(`CREATE INDEX AddressesDeviceIDAddressIndex ON Addresses (DeviceID, Address)`)
+	}
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func postgresCompile(db *sql.DB) (map[string]*sql.Stmt, error) {
+	stmts := map[string]string{
+		"cleanAddress":  "DELETE FROM Addresses WHERE Seen < now() - '2 hour'::INTERVAL",
+		"cleanDevice":   fmt.Sprintf("DELETE FROM Devices WHERE Seen < now() - '%d hour'::INTERVAL", maxDeviceAge/3600),
+		"countAddress":  "SELECT count(*) FROM Addresses",
+		"countDevice":   "SELECT count(*) FROM Devices",
+		"insertAddress": "INSERT INTO Addresses (DeviceID, Seen, Address) VALUES ($1, now(), $2)",
+		"insertDevice":  "INSERT INTO Devices (DeviceID, Seen) VALUES ($1, now())",
+		"selectAddress": "SELECT Address FROM Addresses WHERE DeviceID=$1 AND Seen > now() - '1 hour'::INTERVAL ORDER BY random() LIMIT 16",
+		"selectDevice":  "SELECT Seen FROM Devices WHERE DeviceID=$1",
+		"updateAddress": "UPDATE Addresses SET Seen=now() WHERE DeviceID=$1 AND Address=$2",
+		"updateDevice":  "UPDATE Devices SET Seen=now() WHERE DeviceID=$1",
+	}
+
+	res := make(map[string]*sql.Stmt, len(stmts))
+	for key, stmt := range stmts {
+		prep, err := db.Prepare(stmt)
+		if err != nil {
+			return nil, err
+		}
+		res[key] = prep
+	}
+	return res, nil
+}

cmd/discosrv/ql.go

@@ -0,0 +1,81 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"database/sql"
+	"fmt"
+	"log"
+
+	"github.com/cznic/ql"
+)
+
+func init() {
+	ql.RegisterDriver()
+	register("ql", qlSetup, qlCompile)
+}
+
+func qlSetup(db *sql.DB) (err error) {
+	tx, err := db.Begin()
+	if err != nil {
+		return
+	}
+
+	defer func() {
+		if err == nil {
+			err = tx.Commit()
+		} else {
+			tx.Rollback()
+		}
+	}()
+
+	_, err = tx.Exec(`CREATE TABLE IF NOT EXISTS Devices (
+		DeviceID STRING NOT NULL,
+		Seen TIME NOT NULL
+	)`)
+	if err != nil {
+		return
+	}
+
+	if _, err = tx.Exec(`CREATE INDEX IF NOT EXISTS DevicesDeviceIDIndex ON Devices (DeviceID)`); err != nil {
+		return
+	}
+
+	_, err = tx.Exec(`CREATE TABLE IF NOT EXISTS Addresses (
+		DeviceID STRING NOT NULL,
+		Seen TIME NOT NULL,
+		Address STRING NOT NULL,
+	)`)
+	if err != nil {
+		return
+	}
+
+	_, err = tx.Exec(`CREATE INDEX IF NOT EXISTS AddressesDeviceIDAddressIndex ON Addresses (DeviceID, Address)`)
+	return
+}
+
+func qlCompile(db *sql.DB) (map[string]*sql.Stmt, error) {
+	stmts := map[string]string{
+		"cleanAddress":  `DELETE FROM Addresses WHERE Seen < now() - duration("2h")`,
+		"cleanDevice":   fmt.Sprintf(`DELETE FROM Devices WHERE Seen < now() - duration("%dh")`, maxDeviceAge/3600),
+		"countAddress":  "SELECT count(*) FROM Addresses",
+		"countDevice":   "SELECT count(*) FROM Devices",
+		"insertAddress": "INSERT INTO Addresses (DeviceID, Seen, Address) VALUES ($1, now(), $2)",
+		"insertDevice":  "INSERT INTO Devices (DeviceID, Seen) VALUES ($1, now())",
+		"selectAddress": `SELECT Address from Addresses WHERE DeviceID==$1 AND Seen > now() - duration("1h") LIMIT 16`,
+		"selectDevice":  "SELECT Seen FROM Devices WHERE DeviceID==$1",
+		"updateAddress": "UPDATE Addresses Seen=now() WHERE DeviceID==$1 AND Address==$2",
+		"updateDevice":  "UPDATE Devices Seen=now() WHERE DeviceID==$1",
+	}
+
+	res := make(map[string]*sql.Stmt, len(stmts))
+	for key, stmt := range stmts {
+		prep, err := db.Prepare(stmt)
+		if err != nil {
+			log.Println("Failed to compile", stmt)
+			return nil, err
+		}
+		res[key] = prep
+	}
+	return res, nil
+}

cmd/discosrv/querysrv.go

@@ -0,0 +1,478 @@
+// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"bytes"
+	"crypto/tls"
+	"database/sql"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"log"
+	"math/rand"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"sync"
+	"time"
+
+	"github.com/golang/groupcache/lru"
+	"github.com/juju/ratelimit"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"golang.org/x/net/context"
+)
+
+type querysrv struct {
+	addr     string
+	db       *sql.DB
+	prep     map[string]*sql.Stmt
+	limiter  *safeCache
+	cert     tls.Certificate
+	listener net.Listener
+}
+
+type announcement struct {
+	Seen      time.Time `json:"seen"`
+	Addresses []string  `json:"addresses"`
+}
+
+type safeCache struct {
+	*lru.Cache
+	mut sync.Mutex
+}
+
+func (s *safeCache) Get(key string) (val interface{}, ok bool) {
+	s.mut.Lock()
+	val, ok = s.Cache.Get(key)
+	s.mut.Unlock()
+	return
+}
+
+func (s *safeCache) Add(key string, val interface{}) {
+	s.mut.Lock()
+	s.Cache.Add(key, val)
+	s.mut.Unlock()
+}
+
+type requestID int64
+
+func (i requestID) String() string {
+	return fmt.Sprintf("%016x", int64(i))
+}
+
+func negCacheFor(lastSeen time.Time) int {
+	since := time.Since(lastSeen).Seconds()
+	if since >= maxDeviceAge {
+		return maxNegCache
+	}
+	if since < 0 {
+		// That's weird
+		return minNegCache
+	}
+
+	// Return a value linearly scaled from minNegCache (at zero seconds ago)
+	// to maxNegCache (at maxDeviceAge seconds ago).
+	r := since / maxDeviceAge
+	return int(minNegCache + r*(maxNegCache-minNegCache))
+}
+
+func (s *querysrv) Serve() {
+	s.limiter = &safeCache{
+		Cache: lru.New(lruSize),
+	}
+
+	if useHTTP {
+		listener, err := net.Listen("tcp", s.addr)
+		if err != nil {
+			log.Println("Listen:", err)
+			return
+		}
+		s.listener = listener
+	} else {
+		tlsCfg := &tls.Config{
+			Certificates:           []tls.Certificate{s.cert},
+			ClientAuth:             tls.RequestClientCert,
+			SessionTicketsDisabled: true,
+			MinVersion:             tls.VersionTLS12,
+			CipherSuites: []uint16{
+				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+			},
+		}
+
+		tlsListener, err := tls.Listen("tcp", s.addr, tlsCfg)
+		if err != nil {
+			log.Println("Listen:", err)
+			return
+		}
+		s.listener = tlsListener
+	}
+
+	http.HandleFunc("/v2/", s.handler)
+	http.HandleFunc("/ping", handlePing)
+
+	srv := &http.Server{
+		ReadTimeout:    5 * time.Second,
+		WriteTimeout:   5 * time.Second,
+		MaxHeaderBytes: 1 << 10,
+	}
+
+	if err := srv.Serve(s.listener); err != nil {
+		log.Println("Serve:", err)
+	}
+}
+
+var topCtx = context.Background()
+
+func (s *querysrv) handler(w http.ResponseWriter, req *http.Request) {
+	reqID := requestID(rand.Int63())
+	ctx := context.WithValue(topCtx, "id", reqID)
+
+	if debug {
+		log.Println(reqID, req.Method, req.URL)
+	}
+
+	t0 := time.Now()
+	defer func() {
+		diff := time.Since(t0)
+		var comment string
+		if diff > time.Second {
+			comment = "(very slow request)"
+		} else if diff > 100*time.Millisecond {
+			comment = "(slow request)"
+		}
+		if comment != "" || debug {
+			log.Println(reqID, req.Method, req.URL, "completed in", diff, comment)
+		}
+	}()
+
+	var remoteIP net.IP
+	if useHTTP {
+		remoteIP = net.ParseIP(req.Header.Get("X-Forwarded-For"))
+	} else {
+		addr, err := net.ResolveTCPAddr("tcp", req.RemoteAddr)
+		if err != nil {
+			log.Println("remoteAddr:", err)
+			http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+			return
+		}
+		remoteIP = addr.IP
+	}
+
+	if s.limit(remoteIP) {
+		if debug {
+			log.Println(remoteIP, "is limited")
+		}
+		w.Header().Set("Retry-After", "60")
+		http.Error(w, "Too Many Requests", 429)
+		return
+	}
+
+	switch req.Method {
+	case "GET":
+		s.handleGET(ctx, w, req)
+	case "POST":
+		s.handlePOST(ctx, remoteIP, w, req)
+	default:
+		globalStats.Error()
+		http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func (s *querysrv) handleGET(ctx context.Context, w http.ResponseWriter, req *http.Request) {
+	reqID := ctx.Value("id").(requestID)
+
+	deviceID, err := protocol.DeviceIDFromString(req.URL.Query().Get("device"))
+	if err != nil {
+		if debug {
+			log.Println(reqID, "bad device param")
+		}
+		globalStats.Error()
+		http.Error(w, "Bad Request", http.StatusBadRequest)
+		return
+	}
+
+	var ann announcement
+
+	ann.Seen, err = s.getDeviceSeen(deviceID)
+	negCache := strconv.Itoa(negCacheFor(ann.Seen))
+	w.Header().Set("Retry-After", negCache)
+	w.Header().Set("Cache-Control", "public, max-age="+negCache)
+
+	if err != nil {
+		// The device is not in the database.
+		globalStats.Query()
+		http.Error(w, "Not Found", http.StatusNotFound)
+		return
+	}
+
+	t0 := time.Now()
+	ann.Addresses, err = s.getAddresses(ctx, deviceID)
+	if err != nil {
+		log.Println(reqID, "getAddresses:", err)
+		globalStats.Error()
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+	if debug {
+		log.Println(reqID, "getAddresses in", time.Since(t0))
+	}
+
+	globalStats.Query()
+
+	if len(ann.Addresses) == 0 {
+		http.Error(w, "Not Found", http.StatusNotFound)
+		return
+	}
+
+	globalStats.Answer()
+
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(ann)
+}
+
+func (s *querysrv) handlePOST(ctx context.Context, remoteIP net.IP, w http.ResponseWriter, req *http.Request) {
+	reqID := ctx.Value("id").(requestID)
+
+	rawCert := certificateBytes(req)
+	if rawCert == nil {
+		if debug {
+			log.Println(reqID, "no certificates")
+		}
+		globalStats.Error()
+		http.Error(w, "Forbidden", http.StatusForbidden)
+		return
+	}
+
+	var ann announcement
+	if err := json.NewDecoder(req.Body).Decode(&ann); err != nil {
+		if debug {
+			log.Println(reqID, "decode:", err)
+		}
+		globalStats.Error()
+		http.Error(w, "Bad Request", http.StatusBadRequest)
+		return
+	}
+
+	deviceID := protocol.NewDeviceID(rawCert)
+
+	// handleAnnounce returns *two* errors. The first indicates a problem with
+	// something the client posted to us. We should return a 400 Bad Request
+	// and not worry about it. The second indicates that the request was fine,
+	// but something internal messed up. We should log it and respond with a
+	// more apologetic 500 Internal Server Error.
+	userErr, internalErr := s.handleAnnounce(ctx, remoteIP, deviceID, ann.Addresses)
+	if userErr != nil {
+		if debug {
+			log.Println(reqID, "handleAnnounce:", userErr)
+		}
+		globalStats.Error()
+		http.Error(w, "Bad Request", http.StatusBadRequest)
+		return
+	}
+	if internalErr != nil {
+		log.Println(reqID, "handleAnnounce:", internalErr)
+		globalStats.Error()
+		http.Error(w, "Internal Server Error", http.StatusInternalServerError)
+		return
+	}
+
+	globalStats.Announce()
+
+	// TODO: Slowly increase this for stable clients
+	w.Header().Set("Reannounce-After", "1800")
+
+	// We could return the lookup result here, but it's kind of unnecessarily
+	// expensive to go query the database again so we let the client decide to
+	// do a lookup if they really care.
+	w.WriteHeader(http.StatusNoContent)
+}
+
+func (s *querysrv) Stop() {
+	s.listener.Close()
+}
+
+func (s *querysrv) handleAnnounce(ctx context.Context, remote net.IP, deviceID protocol.DeviceID, addresses []string) (userErr, internalErr error) {
+	reqID := ctx.Value("id").(requestID)
+
+	tx, err := s.db.Begin()
+	if err != nil {
+		internalErr = err
+		return
+	}
+
+	defer func() {
+		// Since we return from a bunch of different places, we handle
+		// rollback in the defer.
+		if internalErr != nil || userErr != nil {
+			tx.Rollback()
+		}
+	}()
+
+	for _, annAddr := range addresses {
+		uri, err := url.Parse(annAddr)
+		if err != nil {
+			userErr = err
+			return
+		}
+
+		host, port, err := net.SplitHostPort(uri.Host)
+		if err != nil {
+			userErr = err
+			return
+		}
+
+		ip := net.ParseIP(host)
+		if host == "" || ip.IsUnspecified() {
+			host = remote.String()
+		}
+
+		uri.Host = net.JoinHostPort(host, port)
+
+		if err := s.updateAddress(ctx, tx, deviceID, uri.String()); err != nil {
+			internalErr = err
+			return
+		}
+	}
+
+	if err := s.updateDevice(ctx, tx, deviceID); err != nil {
+		internalErr = err
+		return
+	}
+
+	t0 := time.Now()
+	internalErr = tx.Commit()
+	if debug {
+		log.Println(reqID, "commit in", time.Since(t0))
+	}
+	return
+}
+
+func (s *querysrv) limit(remote net.IP) bool {
+	key := remote.String()
+
+	bkt, ok := s.limiter.Get(key)
+	if ok {
+		bkt := bkt.(*ratelimit.Bucket)
+		if bkt.TakeAvailable(1) != 1 {
+			// Rate limit exceeded; ignore packet
+			return true
+		}
+	} else {
+		// One packet per ten seconds average rate, burst ten packets
+		s.limiter.Add(key, ratelimit.NewBucket(10*time.Second/time.Duration(limitAvg), int64(limitBurst)))
+	}
+
+	return false
+}
+
+func (s *querysrv) updateDevice(ctx context.Context, tx *sql.Tx, device protocol.DeviceID) error {
+	reqID := ctx.Value("id").(requestID)
+	t0 := time.Now()
+	res, err := tx.Stmt(s.prep["updateDevice"]).Exec(device.String())
+	if err != nil {
+		return err
+	}
+	if debug {
+		log.Println(reqID, "updateDevice in", time.Since(t0))
+	}
+
+	if rows, _ := res.RowsAffected(); rows == 0 {
+		t0 = time.Now()
+		_, err := tx.Stmt(s.prep["insertDevice"]).Exec(device.String())
+		if err != nil {
+			return err
+		}
+		if debug {
+			log.Println(reqID, "insertDevice in", time.Since(t0))
+		}
+	}
+
+	return nil
+}
+
+func (s *querysrv) updateAddress(ctx context.Context, tx *sql.Tx, device protocol.DeviceID, uri string) error {
+	res, err := tx.Stmt(s.prep["updateAddress"]).Exec(device.String(), uri)
+	if err != nil {
+		return err
+	}
+
+	if rows, _ := res.RowsAffected(); rows == 0 {
+		_, err := tx.Stmt(s.prep["insertAddress"]).Exec(device.String(), uri)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (s *querysrv) getAddresses(ctx context.Context, device protocol.DeviceID) ([]string, error) {
+	rows, err := s.prep["selectAddress"].Query(device.String())
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var res []string
+	for rows.Next() {
+		var addr string
+
+		err := rows.Scan(&addr)
+		if err != nil {
+			log.Println("Scan:", err)
+			continue
+		}
+		res = append(res, addr)
+	}
+
+	return res, nil
+}
+
+func (s *querysrv) getDeviceSeen(device protocol.DeviceID) (time.Time, error) {
+	row := s.prep["selectDevice"].QueryRow(device.String())
+	var seen time.Time
+	if err := row.Scan(&seen); err != nil {
+		return time.Time{}, err
+	}
+	return seen, nil
+}
+
+func handlePing(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(204)
+}
+
+func certificateBytes(req *http.Request) []byte {
+	if req.TLS != nil && len(req.TLS.PeerCertificates) > 0 {
+		return req.TLS.PeerCertificates[0].Raw
+	}
+
+	if hdr := req.Header.Get("X-SSL-Cert"); hdr != "" {
+		bs := []byte(hdr)
+		// The certificate is in PEM format but with spaces for newlines. We
+		// need to reinstate the newlines for the PEM decoder. But we need to
+		// leave the spaces in the BEGIN and END lines - the first and last
+		// space - alone.
+		firstSpace := bytes.Index(bs, []byte(" "))
+		lastSpace := bytes.LastIndex(bs, []byte(" "))
+		for i := firstSpace + 1; i < lastSpace; i++ {
+			if bs[i] == ' ' {
+				bs[i] = '\n'
+			}
+		}
+		block, _ := pem.Decode(bs)
+		if block == nil {
+			// Decoding failed
+			return nil
+		}
+		return block.Bytes
+	}
+
+	return nil
+}

cmd/discosrv/stats.go

@@ -0,0 +1,141 @@
+// Copyright (C) 2014-2015 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"bytes"
+	"database/sql"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"sync/atomic"
+	"time"
+)
+
+type stats struct {
+	// Incremented atomically
+	announces int64
+	queries   int64
+	answers   int64
+	errors    int64
+}
+
+func (s *stats) Announce() {
+	atomic.AddInt64(&s.announces, 1)
+}
+
+func (s *stats) Query() {
+	atomic.AddInt64(&s.queries, 1)
+}
+
+func (s *stats) Answer() {
+	atomic.AddInt64(&s.answers, 1)
+}
+
+func (s *stats) Error() {
+	atomic.AddInt64(&s.errors, 1)
+}
+
+// Reset returns a copy of the current stats and resets the counters to
+// zero.
+func (s *stats) Reset() stats {
+	// Create a copy of the stats using atomic reads
+	copy := stats{
+		announces: atomic.LoadInt64(&s.announces),
+		queries:   atomic.LoadInt64(&s.queries),
+		answers:   atomic.LoadInt64(&s.answers),
+		errors:    atomic.LoadInt64(&s.errors),
+	}
+
+	// Reset the stats by subtracting the values that we copied
+	atomic.AddInt64(&s.announces, -copy.announces)
+	atomic.AddInt64(&s.queries, -copy.queries)
+	atomic.AddInt64(&s.answers, -copy.answers)
+	atomic.AddInt64(&s.errors, -copy.errors)
+
+	return copy
+}
+
+type statssrv struct {
+	intv time.Duration
+	file string
+	db   *sql.DB
+}
+
+func (s *statssrv) Serve() {
+	lastReset := time.Now()
+	for {
+		time.Sleep(next(s.intv))
+
+		stats := globalStats.Reset()
+		d := time.Since(lastReset).Seconds()
+		lastReset = time.Now()
+
+		log.Printf("Stats: %.02f announces/s, %.02f queries/s, %.02f answers/s, %.02f errors/s",
+			float64(stats.announces)/d, float64(stats.queries)/d, float64(stats.answers)/d, float64(stats.errors)/d)
+
+		if s.file != "" {
+			s.writeToFile(stats, d)
+		}
+	}
+}
+
+func (s *statssrv) Stop() {
+	panic("stop unimplemented")
+}
+
+func (s *statssrv) writeToFile(stats stats, secs float64) {
+	newLine := []byte("\n")
+
+	var addrs int
+	row := s.db.QueryRow("SELECT COUNT(*) FROM Addresses")
+	if err := row.Scan(&addrs); err != nil {
+		log.Println("stats query:", err)
+		return
+	}
+
+	fd, err := os.OpenFile(s.file, os.O_RDWR|os.O_CREATE, 0666)
+	if err != nil {
+		log.Println("stats file:", err)
+		return
+	}
+	defer func() {
+		err = fd.Close()
+		if err != nil {
+			log.Println("stats file:", err)
+		}
+	}()
+
+	bs, err := ioutil.ReadAll(fd)
+	if err != nil {
+		log.Println("stats file:", err)
+		return
+	}
+	lines := bytes.Split(bytes.TrimSpace(bs), newLine)
+	if len(lines) > 12 {
+		lines = lines[len(lines)-12:]
+	}
+
+	latest := fmt.Sprintf("%v: %6d addresses, %8.02f announces/s, %8.02f queries/s, %8.02f answers/s, %8.02f errors/s\n",
+		time.Now().UTC().Format(time.RFC3339), addrs,
+		float64(stats.announces)/secs, float64(stats.queries)/secs, float64(stats.answers)/secs, float64(stats.errors)/secs)
+	lines = append(lines, []byte(latest))
+
+	_, err = fd.Seek(0, 0)
+	if err != nil {
+		log.Println("stats file:", err)
+		return
+	}
+	err = fd.Truncate(0)
+	if err != nil {
+		log.Println("stats file:", err)
+		return
+	}
+
+	_, err = fd.Write(bytes.Join(lines, newLine))
+	if err != nil {
+		log.Println("stats file:", err)
+		return
+	}
+}

cmd/relaysrv/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 The Syncthing Project
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

cmd/relaysrv/README.md

@@ -0,0 +1,141 @@
+relaysrv
+========
+
+[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/relaysrv.svg?style=flat-square)](http://build.syncthing.net/job/relaysrv/lastBuild/)
+
+This is the relay server for the `syncthing` project.
+
+To get it, run `go get github.com/syncthing/relaysrv` or download the
+[latest build](http://build.syncthing.net/job/relaysrv/lastSuccessfulBuild/artifact/)
+from the build server.
+
+:exclamation:Warnings:exclamation: - Read or regret
+-----
+
+By default, all relay servers will join the default public relay pool, which means that the relay server will be availble for public use, and **will consume your bandwidth** helping others to connect.
+
+If you wish to disable this behaviour, please specify `-pools=""` argument.
+
+Please note that `relaysrv` is only usable by `syncthing` **version v0.12 and onwards**.
+
+To run `relaysrv` you need to have port 22067 available to the internet, which means you might need to allow it through your firewall if you **have a public IP, or setup a port-forwarding** (22067 to 22067) if you are behind a router.
+
+Furthermore, **by default relaysrv will also expose a /status HTTP endpoint on port 22070**, which is used by the pool servers to peek at metrics of the relaysrv, such as what are the current transfer rates, how many clients are connected, etc, etc. If you wish this information to be available, similarlly you might want to allow it through your firewall, or port-forward it (22070 to 22070) on your NAT device.
+
+This is **not mandatory** for the relaysrv to function, and is used only to gather metrics and present them in the overview page of the pool server, displaying stats about the specific relay.
+
+At the point of writing the endpoint output looks as follows:
+
+```
+{
+    "bytesProxied": 0,
+    "goArch": "amd64",
+    "goMaxProcs": 1,
+    "goNumRoutine": 13,
+    "goOS": "linux",
+    "goVersion": "go1.6",
+    "kbps10s1m5m15m30m60m": [
+        0,
+        0,
+        0,
+        0,
+        0,
+        0
+    ],
+    "numActiveSessions": 0,
+    "numConnections": 0,
+    "numPendingSessionKeys": 2,
+    "numProxies": 0,
+    "options": {
+        "global-rate": 0,
+        "message-timeout": 60,
+        "network-timeout": 120,
+        "per-session-rate": 0,
+        "ping-interval": 60,
+        "pools": [
+            "https://relays.syncthing.net/endpoint"
+        ],
+        "provided-by": ""
+    },
+    "startTime": "2016-03-06T12:53:07.090847749-05:00",
+    "uptimeSeconds": 17
+}
+```
+
+If you wish to disable the /status endpoint, provide `-status-srv=""` as one of the arguments when starting the relaysrv.
+
+Running for public use
+----
+Make sure you have a public IP with port 22067 open, or make sure you have port-forwarding (22067 to 22067) if you are behind a router.
+
+Run the `relaysrv` with no arguments (or `-debug` if you want more output), and that should be enough for the server to join the public relay pool.
+You should see a message saying:
+```
+2015/09/21 22:45:46 pool.go:60: Joined https://relays.syncthing.net/endpoint rejoining in 48m0s
+```
+
+See `relaysrv -help` for other options, such as rate limits, timeout intervals, etc.
+
+Running for private use
+-----
+
+Once you've started the `relaysrv`, it will generate a key pair and print an URI:
+```bash
+relay://:22067/?id=EZQOIDM-6DDD4ZI-DJ65NSM-4OQWRAT-EIKSMJO-OZ552BO-WQZEGYY-STS5RQM&pingInterval=1m0s&networkTimeout=2m0s&sessionLimitBps=0&globalLimitBps=0&statusAddr=:22070
+```
+
+This URI contains partial address of the relay server, as well as it's options which in the future may be taken into account when choosing the best suitable relay out of multiple available.
+
+Because `-listen` option was not used, the `relaysrv` does not know it's external IP, therefore you should replace the host part of the URI with your public IP address on which the `relaysrv` will be available:
+
+```bash
+relay://123.123.123.123:22067/?id=EZQOIDM-6DDD4ZI-DJ65NSM-4OQWRAT-EIKSMJO-OZ552BO-WQZEGYY-STS5RQM&pingInterval=1m0s&networkTimeout=2m0s&sessionLimitBps=0&globalLimitBps=0&statusAddr=:22070
+```
+
+If you do not care about certificate pinning (improved security) or do not care about passing verbose settings to the clients, you can shorten the URL to just the host part:
+
+```bash
+relay://123.123.123.123:22067
+```
+
+This URI can then be used in `syncthing` as one of the relay servers.
+
+See `relaysrv -help` for other options, such as rate limits, timeout intervals, etc.
+
+Other items available in this repo
+----
+##### testutil
+A test utility which can be used to test connectivity of a relay server.
+You need to generate two x509 key pairs (key.pem and cert.pem), one for the client, another one for the server, in separate directories.
+Afterwards, start the client:
+```bash
+./testutil -relay="relay://uri.of.relay" -keys=certs/client/ -join
+```
+
+This prints out the client ID:
+```
+2015/09/21 23:00:52 main.go:42: ID: BG2C5ZA-W7XPFDO-LH222Z6-65F3HJX-ADFTGRT-3SBFIGM-KV26O2Q-E5RMRQ2
+```
+
+In the other terminal run the following:
+
+```bash
+ ./testutil -relay="relay://uri.of.relay" -keys=certs/server/ -connect=BG2C5ZA-W7XPFDO-LH222Z6-65F3HJX-ADFTGRT-3SBFIGM-KV26O2Q-E5RMRQ2
+```
+
+Which should then give you an interactive prompt, where you can type things in one terminal, and they get relayed to the other terminal.
+
+Relay related libraries used by this repo
+----
+##### Relay protocol definition.
+
+[Available here](https://github.com/syncthing/syncthing/tree/master/lib/relay/protocol)
+
+
+##### Relay client
+
+Only used by the testutil.
+
+[Available here](https://github.com/syncthing/syncthing/tree/master/lib/relay/client)
+
+

cmd/relaysrv/etc/linux-systemd/syncthing-relaysrv.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Syncthing relay server
+After=network.target
+
+[Service]
+User=syncthing-relaysrv
+Group=syncthing-relaysrv
+ExecStart=/usr/bin/relaysrv
+WorkingDirectory=/var/lib/syncthing-relaysrv
+
+PrivateTmp=true
+ProtectSystem=full
+ProtectHome=true
+NoNewPrivileges=true
+
+[Install]
+WantedBy=multi-user.target

cmd/relaysrv/listener.go

@@ -0,0 +1,345 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors.
+
+package main
+
+import (
+	"crypto/tls"
+	"encoding/hex"
+	"log"
+	"net"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/tlsutil"
+
+	"github.com/syncthing/syncthing/lib/relay/protocol"
+)
+
+var (
+	outboxesMut    = sync.RWMutex{}
+	outboxes       = make(map[syncthingprotocol.DeviceID]chan interface{})
+	numConnections int64
+)
+
+func listener(addr string, config *tls.Config) {
+	tcpListener, err := net.Listen("tcp", addr)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	listener := tlsutil.DowngradingListener{
+		Listener: tcpListener,
+	}
+
+	for {
+		conn, isTLS, err := listener.AcceptNoWrapTLS()
+		if err != nil {
+			if debug {
+				log.Println("Listener failed to accept connection from", conn.RemoteAddr(), ". Possibly a TCP Ping.")
+			}
+			continue
+		}
+
+		setTCPOptions(conn)
+
+		if debug {
+			log.Println("Listener accepted connection from", conn.RemoteAddr(), "tls", isTLS)
+		}
+
+		if isTLS {
+			go protocolConnectionHandler(conn, config)
+		} else {
+			go sessionConnectionHandler(conn)
+		}
+
+	}
+}
+
+func protocolConnectionHandler(tcpConn net.Conn, config *tls.Config) {
+	conn := tls.Server(tcpConn, config)
+	err := conn.Handshake()
+	if err != nil {
+		if debug {
+			log.Println("Protocol connection TLS handshake:", conn.RemoteAddr(), err)
+		}
+		conn.Close()
+		return
+	}
+
+	state := conn.ConnectionState()
+	if (!state.NegotiatedProtocolIsMutual || state.NegotiatedProtocol != protocol.ProtocolName) && debug {
+		log.Println("Protocol negotiation error")
+	}
+
+	certs := state.PeerCertificates
+	if len(certs) != 1 {
+		if debug {
+			log.Println("Certificate list error")
+		}
+		conn.Close()
+		return
+	}
+
+	id := syncthingprotocol.NewDeviceID(certs[0].Raw)
+
+	messages := make(chan interface{})
+	errors := make(chan error, 1)
+	outbox := make(chan interface{})
+
+	// Read messages from the connection and send them on the messages
+	// channel. When there is an error, send it on the error channel and
+	// return. Applies also when the connection gets closed, so the pattern
+	// below is to close the connection on error, then wait for the error
+	// signal from messageReader to exit.
+	go messageReader(conn, messages, errors)
+
+	pingTicker := time.NewTicker(pingInterval)
+	timeoutTicker := time.NewTimer(networkTimeout)
+	joined := false
+
+	for {
+		select {
+		case message := <-messages:
+			timeoutTicker.Reset(networkTimeout)
+			if debug {
+				log.Printf("Message %T from %s", message, id)
+			}
+
+			switch msg := message.(type) {
+			case protocol.JoinRelayRequest:
+				if atomic.LoadInt32(&overLimit) > 0 {
+					protocol.WriteMessage(conn, protocol.RelayFull{})
+					if debug {
+						log.Println("Refusing join request from", id, "due to being over limits")
+					}
+					conn.Close()
+					limitCheckTimer.Reset(time.Second)
+					continue
+				}
+
+				outboxesMut.RLock()
+				_, ok := outboxes[id]
+				outboxesMut.RUnlock()
+				if ok {
+					protocol.WriteMessage(conn, protocol.ResponseAlreadyConnected)
+					if debug {
+						log.Println("Already have a peer with the same ID", id, conn.RemoteAddr())
+					}
+					conn.Close()
+					continue
+				}
+
+				outboxesMut.Lock()
+				outboxes[id] = outbox
+				outboxesMut.Unlock()
+				joined = true
+
+				protocol.WriteMessage(conn, protocol.ResponseSuccess)
+
+			case protocol.ConnectRequest:
+				requestedPeer := syncthingprotocol.DeviceIDFromBytes(msg.ID)
+				outboxesMut.RLock()
+				peerOutbox, ok := outboxes[requestedPeer]
+				outboxesMut.RUnlock()
+				if !ok {
+					if debug {
+						log.Println(id, "is looking for", requestedPeer, "which does not exist")
+					}
+					protocol.WriteMessage(conn, protocol.ResponseNotFound)
+					conn.Close()
+					continue
+				}
+				// requestedPeer is the server, id is the client
+				ses := newSession(requestedPeer, id, sessionLimiter, globalLimiter)
+
+				go ses.Serve()
+
+				clientInvitation := ses.GetClientInvitationMessage()
+				serverInvitation := ses.GetServerInvitationMessage()
+
+				if err := protocol.WriteMessage(conn, clientInvitation); err != nil {
+					if debug {
+						log.Printf("Error sending invitation from %s to client: %s", id, err)
+					}
+					conn.Close()
+					continue
+				}
+
+				peerOutbox <- serverInvitation
+
+				if debug {
+					log.Println("Sent invitation from", id, "to", requestedPeer)
+				}
+				conn.Close()
+
+			case protocol.Ping:
+				if err := protocol.WriteMessage(conn, protocol.Pong{}); err != nil {
+					if debug {
+						log.Println("Error writing pong:", err)
+					}
+					conn.Close()
+					continue
+				}
+
+			case protocol.Pong:
+				// Nothing
+
+			default:
+				if debug {
+					log.Printf("Unknown message %s: %T", id, message)
+				}
+				protocol.WriteMessage(conn, protocol.ResponseUnexpectedMessage)
+				conn.Close()
+			}
+
+		case err := <-errors:
+			if debug {
+				log.Printf("Closing connection %s: %s", id, err)
+			}
+			close(outbox)
+
+			// Potentially closing a second time.
+			conn.Close()
+
+			if joined {
+				// Only delete the outbox if the client is joined, as it might be
+				// a lookup request coming from the same client.
+				outboxesMut.Lock()
+				delete(outboxes, id)
+				outboxesMut.Unlock()
+				// Also, kill all sessions related to this node, as it probably
+				// went offline. This is for the other end to realize the client
+				// is no longer there faster. This also helps resolve
+				// 'already connected' errors when one of the sides is
+				// restarting, and connecting to the other peer before the other
+				// peer even realised that the node has gone away.
+				dropSessions(id)
+			}
+			return
+
+		case <-pingTicker.C:
+			if !joined {
+				if debug {
+					log.Println(id, "didn't join within", pingInterval)
+				}
+				conn.Close()
+				continue
+			}
+
+			if err := protocol.WriteMessage(conn, protocol.Ping{}); err != nil {
+				if debug {
+					log.Println(id, err)
+				}
+				conn.Close()
+			}
+
+			if atomic.LoadInt32(&overLimit) > 0 && !hasSessions(id) {
+				if debug {
+					log.Println("Dropping", id, "as it has no sessions and we are over our limits")
+				}
+				protocol.WriteMessage(conn, protocol.RelayFull{})
+				conn.Close()
+
+				limitCheckTimer.Reset(time.Second)
+			}
+
+		case <-timeoutTicker.C:
+			// We should receive a error from the reader loop, which will cause
+			// us to quit this loop.
+			if debug {
+				log.Printf("%s timed out", id)
+			}
+			conn.Close()
+
+		case msg := <-outbox:
+			if msg == nil {
+				conn.Close()
+				return
+			}
+			if debug {
+				log.Printf("Sending message %T to %s", msg, id)
+			}
+			if err := protocol.WriteMessage(conn, msg); err != nil {
+				if debug {
+					log.Println(id, err)
+				}
+				conn.Close()
+			}
+		}
+	}
+}
+
+func sessionConnectionHandler(conn net.Conn) {
+	if err := conn.SetDeadline(time.Now().Add(messageTimeout)); err != nil {
+		if debug {
+			log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr())
+		}
+		return
+	}
+
+	message, err := protocol.ReadMessage(conn)
+	if err != nil {
+		return
+	}
+
+	switch msg := message.(type) {
+	case protocol.JoinSessionRequest:
+		ses := findSession(string(msg.Key))
+		if debug {
+			log.Println(conn.RemoteAddr(), "session lookup", ses, hex.EncodeToString(msg.Key)[:5])
+		}
+
+		if ses == nil {
+			protocol.WriteMessage(conn, protocol.ResponseNotFound)
+			conn.Close()
+			return
+		}
+
+		if !ses.AddConnection(conn) {
+			if debug {
+				log.Println("Failed to add", conn.RemoteAddr(), "to session", ses)
+			}
+			protocol.WriteMessage(conn, protocol.ResponseAlreadyConnected)
+			conn.Close()
+			return
+		}
+
+		if err := protocol.WriteMessage(conn, protocol.ResponseSuccess); err != nil {
+			if debug {
+				log.Println("Failed to send session join response to ", conn.RemoteAddr(), "for", ses)
+			}
+			return
+		}
+
+		if err := conn.SetDeadline(time.Time{}); err != nil {
+			if debug {
+				log.Println("Weird error setting deadline:", err, "on", conn.RemoteAddr())
+			}
+			conn.Close()
+			return
+		}
+
+	default:
+		if debug {
+			log.Println("Unexpected message from", conn.RemoteAddr(), message)
+		}
+		protocol.WriteMessage(conn, protocol.ResponseUnexpectedMessage)
+		conn.Close()
+	}
+}
+
+func messageReader(conn net.Conn, messages chan<- interface{}, errors chan<- error) {
+	atomic.AddInt64(&numConnections, 1)
+	defer atomic.AddInt64(&numConnections, -1)
+
+	for {
+		msg, err := protocol.ReadMessage(conn)
+		if err != nil {
+			errors <- err
+			return
+		}
+		messages <- msg
+	}
+}

cmd/relaysrv/main.go

@@ -0,0 +1,224 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors.
+
+package main
+
+import (
+	"crypto/tls"
+	"flag"
+	"fmt"
+	"log"
+	"net"
+	"net/url"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"syscall"
+	"time"
+
+	"github.com/juju/ratelimit"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/relay/protocol"
+	"github.com/syncthing/syncthing/lib/tlsutil"
+
+	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
+)
+
+var (
+	Version    string
+	BuildStamp string
+	BuildUser  string
+	BuildHost  string
+
+	BuildDate   time.Time
+	LongVersion string
+)
+
+func init() {
+	stamp, _ := strconv.Atoi(BuildStamp)
+	BuildDate = time.Unix(int64(stamp), 0)
+
+	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
+	LongVersion = fmt.Sprintf(`relaysrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
+}
+
+var (
+	listen string
+	debug  bool
+
+	sessionAddress []byte
+	sessionPort    uint16
+
+	networkTimeout = 2 * time.Minute
+	pingInterval   = time.Minute
+	messageTimeout = time.Minute
+
+	limitCheckTimer *time.Timer
+
+	sessionLimitBps int
+	globalLimitBps  int
+	overLimit       int32
+	descriptorLimit int64
+	sessionLimiter  *ratelimit.Bucket
+	globalLimiter   *ratelimit.Bucket
+
+	statusAddr       string
+	poolAddrs        string
+	pools            []string
+	providedBy       string
+	defaultPoolAddrs = "https://relays.syncthing.net/endpoint"
+)
+
+func main() {
+	log.SetFlags(log.Lshortfile | log.LstdFlags)
+
+	var dir, extAddress string
+
+	flag.StringVar(&listen, "listen", ":22067", "Protocol listen address")
+	flag.StringVar(&dir, "keys", ".", "Directory where cert.pem and key.pem is stored")
+	flag.DurationVar(&networkTimeout, "network-timeout", networkTimeout, "Timeout for network operations between the client and the relay.\n\tIf no data is received between the client and the relay in this period of time, the connection is terminated.\n\tFurthermore, if no data is sent between either clients being relayed within this period of time, the session is also terminated.")
+	flag.DurationVar(&pingInterval, "ping-interval", pingInterval, "How often pings are sent")
+	flag.DurationVar(&messageTimeout, "message-timeout", messageTimeout, "Maximum amount of time we wait for relevant messages to arrive")
+	flag.IntVar(&sessionLimitBps, "per-session-rate", sessionLimitBps, "Per session rate limit, in bytes/s")
+	flag.IntVar(&globalLimitBps, "global-rate", globalLimitBps, "Global rate limit, in bytes/s")
+	flag.BoolVar(&debug, "debug", debug, "Enable debug output")
+	flag.StringVar(&statusAddr, "status-srv", ":22070", "Listen address for status service (blank to disable)")
+	flag.StringVar(&poolAddrs, "pools", defaultPoolAddrs, "Comma separated list of relay pool addresses to join")
+	flag.StringVar(&providedBy, "provided-by", "", "An optional description about who provides the relay")
+	flag.StringVar(&extAddress, "ext-address", "", "An optional address to advertising as being available on.\n\tAllows listening on an unprivileged port with port forwarding from e.g. 443, and be connected to on port 443.")
+
+	flag.Parse()
+
+	if extAddress == "" {
+		extAddress = listen
+	}
+
+	addr, err := net.ResolveTCPAddr("tcp", extAddress)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	log.Println(LongVersion)
+
+	maxDescriptors, err := osutil.MaximizeOpenFileLimit()
+	if maxDescriptors > 0 {
+		// Assume that 20% of FD's are leaked/unaccounted for.
+		descriptorLimit = int64(maxDescriptors*80) / 100
+		log.Println("Connection limit", descriptorLimit)
+
+		go monitorLimits()
+	} else if err != nil && runtime.GOOS != "windows" {
+		log.Println("Assuming no connection limit, due to error retrievign rlimits:", err)
+	}
+
+	sessionAddress = addr.IP[:]
+	sessionPort = uint16(addr.Port)
+
+	certFile, keyFile := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem")
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		log.Println("Failed to load keypair. Generating one, this might take a while...")
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "relaysrv", 3072)
+		if err != nil {
+			log.Fatalln("Failed to generate X509 key pair:", err)
+		}
+	}
+
+	tlsCfg := &tls.Config{
+		Certificates:           []tls.Certificate{cert},
+		NextProtos:             []string{protocol.ProtocolName},
+		ClientAuth:             tls.RequestClientCert,
+		SessionTicketsDisabled: true,
+		InsecureSkipVerify:     true,
+		MinVersion:             tls.VersionTLS12,
+		CipherSuites: []uint16{
+			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+			tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+			tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+			tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+		},
+	}
+
+	id := syncthingprotocol.NewDeviceID(cert.Certificate[0])
+	if debug {
+		log.Println("ID:", id)
+	}
+
+	if sessionLimitBps > 0 {
+		sessionLimiter = ratelimit.NewBucketWithRate(float64(sessionLimitBps), int64(2*sessionLimitBps))
+	}
+	if globalLimitBps > 0 {
+		globalLimiter = ratelimit.NewBucketWithRate(float64(globalLimitBps), int64(2*globalLimitBps))
+	}
+
+	if statusAddr != "" {
+		go statusService(statusAddr)
+	}
+
+	uri, err := url.Parse(fmt.Sprintf("relay://%s/?id=%s&pingInterval=%s&networkTimeout=%s&sessionLimitBps=%d&globalLimitBps=%d&statusAddr=%s&providedBy=%s", extAddress, id, pingInterval, networkTimeout, sessionLimitBps, globalLimitBps, statusAddr, providedBy))
+	if err != nil {
+		log.Fatalln("Failed to construct URI", err)
+	}
+
+	log.Println("URI:", uri.String())
+
+	if poolAddrs == defaultPoolAddrs {
+		log.Println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+		log.Println("!!  Joining default relay pools, this relay will be available for public use. !!")
+		log.Println(`!!      Use the -pools="" command line option to make the relay private.      !!`)
+		log.Println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
+	}
+
+	pools = strings.Split(poolAddrs, ",")
+	for _, pool := range pools {
+		pool = strings.TrimSpace(pool)
+		if len(pool) > 0 {
+			go poolHandler(pool, uri)
+		}
+	}
+
+	go listener(listen, tlsCfg)
+
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+	<-sigs
+
+	// Gracefully close all connections, hoping that clients will be faster
+	// to realize that the relay is now gone.
+
+	sessionMut.RLock()
+	for _, session := range activeSessions {
+		session.CloseConns()
+	}
+
+	for _, session := range pendingSessions {
+		session.CloseConns()
+	}
+	sessionMut.RUnlock()
+
+	outboxesMut.RLock()
+	for _, outbox := range outboxes {
+		close(outbox)
+	}
+	outboxesMut.RUnlock()
+
+	time.Sleep(500 * time.Millisecond)
+}
+
+func monitorLimits() {
+	limitCheckTimer = time.NewTimer(time.Minute)
+	for _ = range limitCheckTimer.C {
+		if atomic.LoadInt64(&numConnections)+atomic.LoadInt64(&numProxies) > descriptorLimit {
+			atomic.StoreInt32(&overLimit, 1)
+			log.Println("Gone past our connection limits. Starting to refuse new/drop idle connections.")
+		} else if atomic.CompareAndSwapInt32(&overLimit, 1, 0) {
+			log.Println("Dropped below our connection limits. Accepting new connections.")
+		}
+		limitCheckTimer.Reset(time.Minute)
+	}
+}

cmd/relaysrv/pool.go

@@ -0,0 +1,63 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors.
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+func poolHandler(pool string, uri *url.URL) {
+	if debug {
+		log.Println("Joining", pool)
+	}
+	for {
+		var b bytes.Buffer
+		json.NewEncoder(&b).Encode(struct {
+			URL string `json:"url"`
+		}{
+			uri.String(),
+		})
+
+		resp, err := http.Post(pool, "application/json", &b)
+		if err != nil {
+			log.Println("Error joining pool", pool, err)
+		} else if resp.StatusCode == 500 {
+			bs, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				log.Println("Failed to join", pool, "due to an internal server error. Could not read response:", err)
+			} else {
+				log.Println("Failed to join", pool, "due to an internal server error:", string(bs))
+			}
+			resp.Body.Close()
+		} else if resp.StatusCode == 429 {
+			log.Println(pool, "under load, will retry in a minute")
+			time.Sleep(time.Minute)
+			continue
+		} else if resp.StatusCode == 403 {
+			log.Println(pool, "failed to join due to IP address not matching external address. Aborting")
+			return
+		} else if resp.StatusCode == 200 {
+			var x struct {
+				EvictionIn time.Duration `json:"evictionIn"`
+			}
+			err := json.NewDecoder(resp.Body).Decode(&x)
+			if err == nil {
+				rejoin := x.EvictionIn - (x.EvictionIn / 5)
+				log.Println("Joined", pool, "rejoining in", rejoin)
+				time.Sleep(rejoin)
+				continue
+			} else {
+				log.Println("Failed to deserialize response", err)
+			}
+		} else {
+			log.Println(pool, "unknown response type from server", resp.StatusCode)
+		}
+		time.Sleep(time.Hour)
+	}
+}

cmd/relaysrv/session.go

@@ -0,0 +1,326 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors.
+
+package main
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"log"
+	"net"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/juju/ratelimit"
+	"github.com/syncthing/syncthing/lib/relay/protocol"
+
+	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
+)
+
+var (
+	sessionMut      = sync.RWMutex{}
+	activeSessions  = make([]*session, 0)
+	pendingSessions = make(map[string]*session, 0)
+	numProxies      int64
+	bytesProxied    int64
+)
+
+func newSession(serverid, clientid syncthingprotocol.DeviceID, sessionRateLimit, globalRateLimit *ratelimit.Bucket) *session {
+	serverkey := make([]byte, 32)
+	_, err := rand.Read(serverkey)
+	if err != nil {
+		return nil
+	}
+
+	clientkey := make([]byte, 32)
+	_, err = rand.Read(clientkey)
+	if err != nil {
+		return nil
+	}
+
+	ses := &session{
+		serverkey: serverkey,
+		serverid:  serverid,
+		clientkey: clientkey,
+		clientid:  clientid,
+		rateLimit: makeRateLimitFunc(sessionRateLimit, globalRateLimit),
+		connsChan: make(chan net.Conn),
+		conns:     make([]net.Conn, 0, 2),
+	}
+
+	if debug {
+		log.Println("New session", ses)
+	}
+
+	sessionMut.Lock()
+	pendingSessions[string(ses.serverkey)] = ses
+	pendingSessions[string(ses.clientkey)] = ses
+	sessionMut.Unlock()
+
+	return ses
+}
+
+func findSession(key string) *session {
+	sessionMut.Lock()
+	defer sessionMut.Unlock()
+	ses, ok := pendingSessions[key]
+	if !ok {
+		return nil
+
+	}
+	delete(pendingSessions, key)
+	return ses
+}
+
+func dropSessions(id syncthingprotocol.DeviceID) {
+	sessionMut.RLock()
+	for _, session := range activeSessions {
+		if session.HasParticipant(id) {
+			if debug {
+				log.Println("Dropping session", session, "involving", id)
+			}
+			session.CloseConns()
+		}
+	}
+	sessionMut.RUnlock()
+}
+
+func hasSessions(id syncthingprotocol.DeviceID) bool {
+	sessionMut.RLock()
+	has := false
+	for _, session := range activeSessions {
+		if session.HasParticipant(id) {
+			has = true
+			break
+		}
+	}
+	sessionMut.RUnlock()
+	return has
+}
+
+type session struct {
+	mut sync.Mutex
+
+	serverkey []byte
+	serverid  syncthingprotocol.DeviceID
+
+	clientkey []byte
+	clientid  syncthingprotocol.DeviceID
+
+	rateLimit func(bytes int64)
+
+	connsChan chan net.Conn
+	conns     []net.Conn
+}
+
+func (s *session) AddConnection(conn net.Conn) bool {
+	if debug {
+		log.Println("New connection for", s, "from", conn.RemoteAddr())
+	}
+
+	select {
+	case s.connsChan <- conn:
+		return true
+	default:
+	}
+	return false
+}
+
+func (s *session) Serve() {
+	timedout := time.After(messageTimeout)
+
+	if debug {
+		log.Println("Session", s, "serving")
+	}
+
+	for {
+		select {
+		case conn := <-s.connsChan:
+			s.mut.Lock()
+			s.conns = append(s.conns, conn)
+			s.mut.Unlock()
+			// We're the only ones mutating s.conns, hence we are free to read it.
+			if len(s.conns) < 2 {
+				continue
+			}
+
+			close(s.connsChan)
+
+			if debug {
+				log.Println("Session", s, "starting between", s.conns[0].RemoteAddr(), "and", s.conns[1].RemoteAddr())
+			}
+
+			wg := sync.WaitGroup{}
+			wg.Add(2)
+
+			var err0 error
+			go func() {
+				err0 = s.proxy(s.conns[0], s.conns[1])
+				wg.Done()
+			}()
+
+			var err1 error
+			go func() {
+				err1 = s.proxy(s.conns[1], s.conns[0])
+				wg.Done()
+			}()
+
+			sessionMut.Lock()
+			activeSessions = append(activeSessions, s)
+			sessionMut.Unlock()
+
+			wg.Wait()
+
+			if debug {
+				log.Println("Session", s, "ended, outcomes:", err0, "and", err1)
+			}
+			goto done
+
+		case <-timedout:
+			if debug {
+				log.Println("Session", s, "timed out")
+			}
+			goto done
+		}
+	}
+done:
+	// We can end up here in 3 cases:
+	// 1. Timeout joining, in which case there are potentially entries in pendingSessions
+	// 2. General session end/timeout, in which case there are entries in activeSessions
+	// 3. Protocol handler calls dropSession as one of it's clients disconnects.
+
+	sessionMut.Lock()
+	delete(pendingSessions, string(s.serverkey))
+	delete(pendingSessions, string(s.clientkey))
+
+	for i, session := range activeSessions {
+		if session == s {
+			l := len(activeSessions) - 1
+			activeSessions[i] = activeSessions[l]
+			activeSessions[l] = nil
+			activeSessions = activeSessions[:l]
+		}
+	}
+	sessionMut.Unlock()
+
+	// If we are here because of case 2 or 3, we are potentially closing some or
+	// all connections a second time.
+	s.CloseConns()
+
+	if debug {
+		log.Println("Session", s, "stopping")
+	}
+}
+
+func (s *session) GetClientInvitationMessage() protocol.SessionInvitation {
+	return protocol.SessionInvitation{
+		From:         s.serverid[:],
+		Key:          []byte(s.clientkey),
+		Address:      sessionAddress,
+		Port:         sessionPort,
+		ServerSocket: false,
+	}
+}
+
+func (s *session) GetServerInvitationMessage() protocol.SessionInvitation {
+	return protocol.SessionInvitation{
+		From:         s.clientid[:],
+		Key:          []byte(s.serverkey),
+		Address:      sessionAddress,
+		Port:         sessionPort,
+		ServerSocket: true,
+	}
+}
+
+func (s *session) HasParticipant(id syncthingprotocol.DeviceID) bool {
+	return s.clientid == id || s.serverid == id
+}
+
+func (s *session) CloseConns() {
+	s.mut.Lock()
+	for _, conn := range s.conns {
+		conn.Close()
+	}
+	s.mut.Unlock()
+}
+
+func (s *session) proxy(c1, c2 net.Conn) error {
+	if debug {
+		log.Println("Proxy", c1.RemoteAddr(), "->", c2.RemoteAddr())
+	}
+
+	atomic.AddInt64(&numProxies, 1)
+	defer atomic.AddInt64(&numProxies, -1)
+
+	buf := make([]byte, 65536)
+	for {
+		c1.SetReadDeadline(time.Now().Add(networkTimeout))
+		n, err := c1.Read(buf)
+		if err != nil {
+			return err
+		}
+
+		atomic.AddInt64(&bytesProxied, int64(n))
+
+		if debug {
+			log.Printf("%d bytes from %s to %s", n, c1.RemoteAddr(), c2.RemoteAddr())
+		}
+
+		if s.rateLimit != nil {
+			s.rateLimit(int64(n))
+		}
+
+		c2.SetWriteDeadline(time.Now().Add(networkTimeout))
+		_, err = c2.Write(buf[:n])
+		if err != nil {
+			return err
+		}
+	}
+}
+
+func (s *session) String() string {
+	return fmt.Sprintf("<%s/%s>", hex.EncodeToString(s.clientkey)[:5], hex.EncodeToString(s.serverkey)[:5])
+}
+
+func makeRateLimitFunc(sessionRateLimit, globalRateLimit *ratelimit.Bucket) func(int64) {
+	// This may be a case of super duper premature optimization... We build an
+	// optimized function to do the rate limiting here based on what we need
+	// to do and then use it in the loop.
+
+	if sessionRateLimit == nil && globalRateLimit == nil {
+		// No limiting needed. We could equally well return a func(int64){} and
+		// not do a nil check were we use it, but I think the nil check there
+		// makes it clear that there will be no limiting if none is
+		// configured...
+		return nil
+	}
+
+	if sessionRateLimit == nil {
+		// We only have a global limiter
+		return func(bytes int64) {
+			globalRateLimit.Wait(bytes)
+		}
+	}
+
+	if globalRateLimit == nil {
+		// We only have a session limiter
+		return func(bytes int64) {
+			sessionRateLimit.Wait(bytes)
+		}
+	}
+
+	// We have both. Queue the bytes on both the global and session specific
+	// rate limiters. Wait for both in parallell, so that the actual send
+	// happens when both conditions are satisfied. In practice this just means
+	// wait the longer of the two times.
+	return func(bytes int64) {
+		t0 := sessionRateLimit.Take(bytes)
+		t1 := globalRateLimit.Take(bytes)
+		if t0 > t1 {
+			time.Sleep(t0)
+		} else {
+			time.Sleep(t1)
+		}
+	}
+}

cmd/relaysrv/status.go

@@ -0,0 +1,111 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors.
+
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"net/http"
+	"runtime"
+	"sync/atomic"
+	"time"
+)
+
+var rc *rateCalculator
+
+func statusService(addr string) {
+	rc = newRateCalculator(360, 10*time.Second, &bytesProxied)
+
+	http.HandleFunc("/status", getStatus)
+	if err := http.ListenAndServe(addr, nil); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func getStatus(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	status := make(map[string]interface{})
+
+	sessionMut.Lock()
+	// This can potentially be double the number of pending sessions, as each session has two keys, one for each side.
+	status["startTime"] = rc.startTime
+	status["uptimeSeconds"] = time.Since(rc.startTime) / time.Second
+	status["numPendingSessionKeys"] = len(pendingSessions)
+	status["numActiveSessions"] = len(activeSessions)
+	sessionMut.Unlock()
+	status["numConnections"] = atomic.LoadInt64(&numConnections)
+	status["numProxies"] = atomic.LoadInt64(&numProxies)
+	status["bytesProxied"] = atomic.LoadInt64(&bytesProxied)
+	status["goVersion"] = runtime.Version()
+	status["goOS"] = runtime.GOOS
+	status["goArch"] = runtime.GOARCH
+	status["goMaxProcs"] = runtime.GOMAXPROCS(-1)
+	status["goNumRoutine"] = runtime.NumGoroutine()
+	status["kbps10s1m5m15m30m60m"] = []int64{
+		rc.rate(10/10) * 8 / 1000,
+		rc.rate(60/10) * 8 / 1000,
+		rc.rate(5*60/10) * 8 / 1000,
+		rc.rate(15*60/10) * 8 / 1000,
+		rc.rate(30*60/10) * 8 / 1000,
+		rc.rate(60*60/10) * 8 / 1000,
+	}
+	status["options"] = map[string]interface{}{
+		"network-timeout":  networkTimeout / time.Second,
+		"ping-interval":    pingInterval / time.Second,
+		"message-timeout":  messageTimeout / time.Second,
+		"per-session-rate": sessionLimitBps,
+		"global-rate":      globalLimitBps,
+		"pools":            pools,
+		"provided-by":      providedBy,
+	}
+
+	bs, err := json.MarshalIndent(status, "", "    ")
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(bs)
+}
+
+type rateCalculator struct {
+	rates     []int64
+	prev      int64
+	counter   *int64
+	startTime time.Time
+}
+
+func newRateCalculator(keepIntervals int, interval time.Duration, counter *int64) *rateCalculator {
+	r := &rateCalculator{
+		rates:     make([]int64, keepIntervals),
+		counter:   counter,
+		startTime: time.Now(),
+	}
+
+	go r.updateRates(interval)
+
+	return r
+}
+
+func (r *rateCalculator) updateRates(interval time.Duration) {
+	for {
+		now := time.Now()
+		next := now.Truncate(interval).Add(interval)
+		time.Sleep(next.Sub(now))
+
+		cur := atomic.LoadInt64(r.counter)
+		rate := int64(float64(cur-r.prev) / interval.Seconds())
+		copy(r.rates[1:], r.rates)
+		r.rates[0] = rate
+		r.prev = cur
+	}
+}
+
+func (r *rateCalculator) rate(periods int) int64 {
+	var tot int64
+	for i := 0; i < periods; i++ {
+		tot += r.rates[i]
+	}
+	return tot / int64(periods)
+}

cmd/relaysrv/testutil/main.go

@@ -0,0 +1,152 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+package main
+
+import (
+	"bufio"
+	"crypto/tls"
+	"flag"
+	"log"
+	"net"
+	"net/url"
+	"os"
+	"path/filepath"
+	"time"
+
+	syncthingprotocol "github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/relay/client"
+	"github.com/syncthing/syncthing/lib/relay/protocol"
+)
+
+func main() {
+	log.SetOutput(os.Stdout)
+	log.SetFlags(log.LstdFlags | log.Lshortfile)
+
+	var connect, relay, dir string
+	var join, test bool
+
+	flag.StringVar(&connect, "connect", "", "Device ID to which to connect to")
+	flag.BoolVar(&join, "join", false, "Join relay")
+	flag.BoolVar(&test, "test", false, "Generic relay test")
+	flag.StringVar(&relay, "relay", "relay://127.0.0.1:22067", "Relay address")
+	flag.StringVar(&dir, "keys", ".", "Directory where cert.pem and key.pem is stored")
+
+	flag.Parse()
+
+	certFile, keyFile := filepath.Join(dir, "cert.pem"), filepath.Join(dir, "key.pem")
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		log.Fatalln("Failed to load X509 key pair:", err)
+	}
+
+	id := syncthingprotocol.NewDeviceID(cert.Certificate[0])
+	log.Println("ID:", id)
+
+	uri, err := url.Parse(relay)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	stdin := make(chan string)
+
+	go stdinReader(stdin)
+
+	if join {
+		log.Println("Creating client")
+		relay, err := client.NewClient(uri, []tls.Certificate{cert}, nil, 10*time.Second)
+		if err != nil {
+			log.Fatal(err)
+		}
+		log.Println("Created client")
+
+		go relay.Serve()
+
+		recv := make(chan protocol.SessionInvitation)
+
+		go func() {
+			log.Println("Starting invitation receiver")
+			for invite := range relay.Invitations() {
+				select {
+				case recv <- invite:
+					log.Println("Received invitation", invite)
+				default:
+					log.Println("Discarding invitation", invite)
+				}
+			}
+		}()
+
+		for {
+			conn, err := client.JoinSession(<-recv)
+			if err != nil {
+				log.Fatalln("Failed to join", err)
+			}
+			log.Println("Joined", conn.RemoteAddr(), conn.LocalAddr())
+			connectToStdio(stdin, conn)
+			log.Println("Finished", conn.RemoteAddr(), conn.LocalAddr())
+		}
+	} else if connect != "" {
+		id, err := syncthingprotocol.DeviceIDFromString(connect)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		invite, err := client.GetInvitationFromRelay(uri, id, []tls.Certificate{cert}, 10*time.Second)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		log.Println("Received invitation", invite)
+		conn, err := client.JoinSession(invite)
+		if err != nil {
+			log.Fatalln("Failed to join", err)
+		}
+		log.Println("Joined", conn.RemoteAddr(), conn.LocalAddr())
+		connectToStdio(stdin, conn)
+		log.Println("Finished", conn.RemoteAddr(), conn.LocalAddr())
+	} else if test {
+		if client.TestRelay(uri, []tls.Certificate{cert}, time.Second, 2*time.Second, 4) {
+			log.Println("OK")
+		} else {
+			log.Println("FAIL")
+		}
+	} else {
+		log.Fatal("Requires either join or connect")
+	}
+}
+
+func stdinReader(c chan<- string) {
+	scanner := bufio.NewScanner(os.Stdin)
+	for scanner.Scan() {
+		c <- scanner.Text()
+		c <- "\n"
+	}
+}
+
+func connectToStdio(stdin <-chan string, conn net.Conn) {
+	go func() {
+
+	}()
+
+	buf := make([]byte, 1024)
+	for {
+		conn.SetReadDeadline(time.Now().Add(time.Millisecond))
+		n, err := conn.Read(buf[0:])
+		if err != nil {
+			nerr, ok := err.(net.Error)
+			if !ok || !nerr.Timeout() {
+				log.Println(err)
+				return
+			}
+		}
+		os.Stdout.Write(buf[:n])
+
+		select {
+		case msg := <-stdin:
+			_, err := conn.Write([]byte(msg))
+			if err != nil {
+				return
+			}
+		default:
+		}
+	}
+}

cmd/relaysrv/utils.go

@@ -0,0 +1,28 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors.
+
+package main
+
+import (
+	"errors"
+	"net"
+)
+
+func setTCPOptions(conn net.Conn) error {
+	tcpConn, ok := conn.(*net.TCPConn)
+	if !ok {
+		return errors.New("Not a TCP connection")
+	}
+	if err := tcpConn.SetLinger(0); err != nil {
+		return err
+	}
+	if err := tcpConn.SetNoDelay(true); err != nil {
+		return err
+	}
+	if err := tcpConn.SetKeepAlivePeriod(networkTimeout); err != nil {
+		return err
+	}
+	if err := tcpConn.SetKeepAlive(true); err != nil {
+		return err
+	}
+	return nil
+}

cmd/stbench/main.go

@@ -0,0 +1,143 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// This doesn't build on Windows due to the Rusage stuff.
+
+// +build !windows
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"runtime"
+	"syscall"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/rc"
+)
+
+var homeDir = "h1"
+var syncthingBin = "./bin/syncthing"
+var test = "scan"
+
+func main() {
+	flag.StringVar(&homeDir, "home", homeDir, "Home directory location")
+	flag.StringVar(&syncthingBin, "bin", syncthingBin, "Binary location")
+	flag.StringVar(&test, "test", test, "Test to run")
+	flag.Parse()
+
+	switch test {
+	case "scan":
+		// scan measures the resource usage required to perform the initial
+		// scan, without cleaning away the database first.
+		testScan()
+	}
+}
+
+// testScan starts a process and reports on the resource usage required to
+// perform the initial scan.
+func testScan() {
+	log.Println("Starting...")
+	p := rc.NewProcess("127.0.0.1:8081")
+	if err := p.Start(syncthingBin, "-home", homeDir, "-no-browser"); err != nil {
+		log.Println(err)
+		return
+	}
+	defer p.Stop()
+
+	wallTime := awaitScanComplete(p)
+
+	report(p, wallTime)
+}
+
+// awaitScanComplete waits for a folder to transition idle->scanning and
+// then scanning->idle and returns the time taken for the scan.
+func awaitScanComplete(p *rc.Process) time.Duration {
+	log.Println("Awaiting scan completion...")
+	var t0, t1 time.Time
+	lastEvent := 0
+loop:
+	for {
+		evs, err := p.Events(lastEvent)
+		if err != nil {
+			continue
+		}
+
+		for _, ev := range evs {
+			if ev.Type == "StateChanged" {
+				data := ev.Data.(map[string]interface{})
+				log.Println(ev)
+
+				if data["to"].(string) == "scanning" {
+					t0 = ev.Time
+					continue
+				}
+
+				if !t0.IsZero() && data["to"].(string) == "idle" {
+					t1 = ev.Time
+					break loop
+				}
+			}
+			lastEvent = ev.ID
+		}
+
+		time.Sleep(250 * time.Millisecond)
+	}
+
+	return t1.Sub(t0)
+}
+
+// report stops the given process and reports on it's resource usage in two
+// ways: human readable to stderr, and CSV to stdout.
+func report(p *rc.Process, wallTime time.Duration) {
+	sv, err := p.SystemVersion()
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
+	ss, err := p.SystemStatus()
+	if err != nil {
+		log.Println(err)
+		return
+	}
+
+	proc, err := p.Stop()
+	if err != nil {
+		return
+	}
+
+	rusage, ok := proc.SysUsage().(*syscall.Rusage)
+	if !ok {
+		return
+	}
+
+	log.Println("Version:", sv.Version)
+	log.Println("Alloc:", ss.Alloc/1024, "KiB")
+	log.Println("Sys:", ss.Sys/1024, "KiB")
+	log.Println("Goroutines:", ss.Goroutines)
+	log.Println("Wall time:", wallTime)
+	log.Println("Utime:", time.Duration(rusage.Utime.Nano()))
+	log.Println("Stime:", time.Duration(rusage.Stime.Nano()))
+	if runtime.GOOS == "darwin" {
+		// Darwin reports in bytes, Linux seems to report in KiB even
+		// though the manpage says otherwise.
+		rusage.Maxrss /= 1024
+	}
+	log.Println("MaxRSS:", rusage.Maxrss, "KiB")
+
+	fmt.Printf("%s,%d,%d,%d,%.02f,%.02f,%.02f,%d\n",
+		sv.Version,
+		ss.Alloc/1024,
+		ss.Sys/1024,
+		ss.Goroutines,
+		wallTime.Seconds(),
+		time.Duration(rusage.Utime.Nano()).Seconds(),
+		time.Duration(rusage.Stime.Nano()).Seconds(),
+		rusage.Maxrss)
+}

cmd/stcompdirs/main.go

@@ -16,7 +16,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/syncthing/syncthing/internal/symlinks"
+	"github.com/syncthing/syncthing/lib/symlinks"
 )
 
 func main() {

cmd/stdisco/main.go

@@ -0,0 +1,126 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"crypto/rand"
+	"flag"
+	"log"
+	"strings"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/beacon"
+	"github.com/syncthing/syncthing/lib/discover"
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+var (
+	all  = false // print all packets, not just first from each device/source
+	fake = false // send fake packets to lure out other devices faster
+	mc   = "[ff12::8384]:21027"
+	bc   = 21027
+)
+
+var (
+	// Static prefix that we use when generating fake device IDs, so that we
+	// can recognize them ourselves. Also makes the device ID start with
+	// "STPROBE-" which is humanly recognizable.
+	randomPrefix = []byte{148, 223, 23, 4, 148}
+
+	// Our random, fake, device ID that we use when sending announcements.
+	myID = randomDeviceID()
+)
+
+func main() {
+	flag.BoolVar(&all, "all", all, "Print all received announcements (not only first)")
+	flag.BoolVar(&fake, "fake", fake, "Send fake announcements")
+	flag.StringVar(&mc, "mc", mc, "IPv6 multicast address")
+	flag.IntVar(&bc, "bc", bc, "IPv4 broadcast port number")
+	flag.Parse()
+
+	if fake {
+		log.Println("My ID:", protocol.DeviceIDFromBytes(myID))
+	}
+
+	runbeacon(beacon.NewMulticast(mc), fake)
+	runbeacon(beacon.NewBroadcast(bc), fake)
+
+	select {}
+}
+
+func runbeacon(bc beacon.Interface, fake bool) {
+	go bc.Serve()
+	go recv(bc)
+	if fake {
+		go send(bc)
+	}
+}
+
+// receives and prints discovery announcements
+func recv(bc beacon.Interface) {
+	seen := make(map[string]bool)
+	for {
+		data, src := bc.Recv()
+		var ann discover.Announce
+		ann.UnmarshalXDR(data)
+
+		if bytes.Equal(ann.This.ID, myID) {
+			// This is one of our own fake packets, don't print it.
+			continue
+		}
+
+		// Print announcement details for the first packet from a given
+		// device ID and source address, or if -all was given.
+		key := string(ann.This.ID) + src.String()
+		if all || !seen[key] {
+			log.Printf("Announcement from %v\n", src)
+			log.Printf(" %v at %s\n", protocol.DeviceIDFromBytes(ann.This.ID), strings.Join(addrStrs(ann.This), ", "))
+
+			for _, dev := range ann.Extra {
+				log.Printf(" %v at %s\n", protocol.DeviceIDFromBytes(dev.ID), strings.Join(addrStrs(dev), ", "))
+			}
+			seen[key] = true
+		}
+	}
+}
+
+// sends fake discovery announcements once every second
+func send(bc beacon.Interface) {
+	ann := discover.Announce{
+		Magic: discover.AnnouncementMagic,
+		This: discover.Device{
+			ID: myID,
+			Addresses: []discover.Address{
+				{URL: "tcp://fake.example.com:12345"},
+			},
+		},
+	}
+	bs, _ := ann.MarshalXDR()
+
+	for {
+		bc.Send(bs)
+		time.Sleep(time.Second)
+	}
+}
+
+// returns the list of address URLs
+func addrStrs(dev discover.Device) []string {
+	ss := make([]string, len(dev.Addresses))
+	for i, addr := range dev.Addresses {
+		ss[i] = addr.URL
+	}
+	return ss
+}
+
+// returns a random but recognizable device ID
+func randomDeviceID() []byte {
+	var id [32]byte
+	copy(id[:], randomPrefix)
+	rand.Read(id[len(randomPrefix):])
+	return id[:]
+}

cmd/stfileinfo/main.go

@@ -12,8 +12,8 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/internal/scanner"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/scanner"
 )
 
 func main() {
@@ -68,7 +68,7 @@ func main() {
 		if *standardBlocks || blockSize < protocol.BlockSize {
 			blockSize = protocol.BlockSize
 		}
-		bs, err := scanner.Blocks(fd, blockSize, fi.Size())
+		bs, err := scanner.Blocks(fd, blockSize, fi.Size(), nil)
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/stfinddevice/main.go

@@ -7,37 +7,101 @@
 package main
 
 import (
+	"crypto/tls"
+	"errors"
 	"flag"
-	"log"
+	"fmt"
+	"net/url"
 	"os"
+	"time"
 
-	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/internal/discover"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/discover"
+	"github.com/syncthing/syncthing/lib/protocol"
 )
 
-func main() {
-	log.SetFlags(0)
-	log.SetOutput(os.Stdout)
+var timeout = 5 * time.Second
 
+func main() {
 	var server string
 
-	flag.StringVar(&server, "server", "udp4://announce.syncthing.net:22026", "Announce server")
+	flag.StringVar(&server, "server", "", "Announce server (blank for default set)")
+	flag.DurationVar(&timeout, "timeout", timeout, "Query timeout")
+	flag.Usage = usage
 	flag.Parse()
 
-	if len(flag.Args()) != 1 || server == "" {
-		log.Printf("Usage: %s [-server=\"udp4://announce.syncthing.net:22026\"] <device>", os.Args[0])
+	if flag.NArg() != 1 {
+		flag.Usage()
 		os.Exit(64)
 	}
 
 	id, err := protocol.DeviceIDFromString(flag.Args()[0])
 	if err != nil {
-		log.Println(err)
+		fmt.Println(err)
 		os.Exit(1)
 	}
 
-	discoverer := discover.NewDiscoverer(protocol.LocalDeviceID, nil)
-	discoverer.StartGlobal([]string{server}, 1)
-	for _, addr := range discoverer.Lookup(id) {
-		log.Println(addr)
+	if server != "" {
+		checkServers(id, server)
+	} else {
+		checkServers(id, config.DefaultDiscoveryServers...)
 	}
 }
+
+type checkResult struct {
+	server    string
+	addresses []string
+	error
+}
+
+func checkServers(deviceID protocol.DeviceID, servers ...string) {
+	t0 := time.Now()
+	resc := make(chan checkResult)
+	for _, srv := range servers {
+		srv := srv
+		go func() {
+			res := checkServer(deviceID, srv)
+			res.server = srv
+			resc <- res
+		}()
+	}
+
+	for _ = range servers {
+		res := <-resc
+
+		u, _ := url.Parse(res.server)
+		fmt.Printf("%s (%v):\n", u.Host, time.Since(t0))
+
+		if res.error != nil {
+			fmt.Println("  " + res.error.Error())
+		}
+		for _, addr := range res.addresses {
+			fmt.Println("  address:", addr)
+		}
+	}
+}
+
+func checkServer(deviceID protocol.DeviceID, server string) checkResult {
+	disco, err := discover.NewGlobal(server, tls.Certificate{}, nil)
+	if err != nil {
+		return checkResult{error: err}
+	}
+
+	res := make(chan checkResult, 1)
+
+	time.AfterFunc(timeout, func() {
+		res <- checkResult{error: errors.New("timeout")}
+	})
+
+	go func() {
+		addresses, err := disco.Lookup(deviceID)
+		res <- checkResult{addresses: addresses, error: err}
+	}()
+
+	return <-res
+}
+
+func usage() {
+	fmt.Printf("Usage:\n\t%s [options] <device ID>\n\nOptions:\n", os.Args[0])
+	flag.PrintDefaults()
+}

cmd/stgenfiles/main.go

@@ -0,0 +1,124 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+func main() {
+	dir := flag.String("dir", "~/files", "Directory to generate into")
+	files := flag.Int("files", 1000, "Number of files to create")
+	maxExp := flag.Int("maxexp", 20, "Max size exponent")
+	src := flag.String("src", "/dev/urandom", "Source of file data")
+	flag.Parse()
+	if err := generateFiles(*dir, *files, *maxExp, *src); err != nil {
+		log.Println(err)
+	}
+}
+
+func generateFiles(dir string, files, maxexp int, srcname string) error {
+	fd, err := os.Open(srcname)
+	if err != nil {
+		return err
+	}
+
+	for i := 0; i < files; i++ {
+		n := randomName()
+
+		if rand.Float64() < 0.05 {
+			// Some files and directories are dotfiles
+			n = "." + n
+		}
+
+		p0 := filepath.Join(dir, string(n[0]), n[0:2])
+		err = os.MkdirAll(p0, 0755)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		p1 := filepath.Join(p0, n)
+
+		s := int64(1 << uint(rand.Intn(maxexp)))
+		a := int64(128 * 1024)
+		if a > s {
+			a = s
+		}
+		s += rand.Int63n(a)
+
+		if err := generateOneFile(fd, p1, s); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
+	src := io.LimitReader(&inifiteReader{fd}, int64(s))
+	dst, err := os.Create(p1)
+	if err != nil {
+		return err
+	}
+
+	_, err = io.Copy(dst, src)
+	if err != nil {
+		return err
+	}
+
+	err = dst.Close()
+	if err != nil {
+		return err
+	}
+
+	_ = os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
+
+	t := time.Now().Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
+	err = os.Chtimes(p1, t, t)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func randomName() string {
+	var b [16]byte
+	readRand(b[:])
+	return fmt.Sprintf("%x", b[:])
+}
+
+func readRand(bs []byte) (int, error) {
+	var r uint32
+	for i := range bs {
+		if i%4 == 0 {
+			r = uint32(rand.Int63())
+		}
+		bs[i] = byte(r >> uint((i%4)*8))
+	}
+	return len(bs), nil
+}
+
+type inifiteReader struct {
+	rd io.ReadSeeker
+}
+
+func (i *inifiteReader) Read(bs []byte) (int, error) {
+	n, err := i.rd.Read(bs)
+	if err == io.EOF {
+		err = nil
+		i.rd.Seek(0, 0)
+	}
+	return n, err
+}

cmd/stindex/dump.go

@@ -0,0 +1,65 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"encoding/binary"
+	"fmt"
+	"log"
+
+	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+func dump(ldb *leveldb.DB) {
+	it := ldb.NewIterator(nil, nil)
+	var dev protocol.DeviceID
+	for it.Next() {
+		key := it.Key()
+		switch key[0] {
+		case db.KeyTypeDevice:
+			folder := nulString(key[1 : 1+64])
+			devBytes := key[1+64 : 1+64+32]
+			name := nulString(key[1+64+32:])
+			copy(dev[:], devBytes)
+			fmt.Printf("[device] F:%q N:%q D:%v\n", folder, name, dev)
+
+			var f protocol.FileInfo
+			err := f.UnmarshalXDR(it.Value())
+			if err != nil {
+				log.Fatal(err)
+			}
+			fmt.Printf("  N:%q\n  F:%#o\n  M:%d\n  V:%v\n  S:%d\n  B:%d\n", f.Name, f.Flags, f.Modified, f.Version, f.Size(), len(f.Blocks))
+
+		case db.KeyTypeGlobal:
+			folder := nulString(key[1 : 1+64])
+			name := nulString(key[1+64:])
+			var flv db.VersionList
+			flv.UnmarshalXDR(it.Value())
+			fmt.Printf("[global] F:%q N:%q V: %s\n", folder, name, flv)
+
+		case db.KeyTypeBlock:
+			folder := nulString(key[1 : 1+64])
+			hash := key[1+64 : 1+64+32]
+			name := nulString(key[1+64+32:])
+			fmt.Printf("[block] F:%q H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
+
+		case db.KeyTypeDeviceStatistic:
+			fmt.Printf("[dstat]\n  %x\n  %x\n", it.Key(), it.Value())
+
+		case db.KeyTypeFolderStatistic:
+			fmt.Printf("[fstat]\n  %x\n  %x\n", it.Key(), it.Value())
+
+		case db.KeyTypeVirtualMtime:
+			fmt.Printf("[mtime]\n  %x\n  %x\n", it.Key(), it.Value())
+
+		default:
+			fmt.Printf("[???]\n  %x\n  %x\n", it.Key(), it.Value())
+		}
+	}
+}

cmd/stindex/dumpsize.go

@@ -0,0 +1,89 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"container/heap"
+	"fmt"
+
+	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type SizedElement struct {
+	key  string
+	size int
+}
+
+type ElementHeap []SizedElement
+
+func (h ElementHeap) Len() int           { return len(h) }
+func (h ElementHeap) Less(i, j int) bool { return h[i].size > h[j].size }
+func (h ElementHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
+
+func (h *ElementHeap) Push(x interface{}) {
+	*h = append(*h, x.(SizedElement))
+}
+
+func (h *ElementHeap) Pop() interface{} {
+	old := *h
+	n := len(old)
+	x := old[n-1]
+	*h = old[0 : n-1]
+	return x
+}
+
+func dumpsize(ldb *leveldb.DB) {
+	h := &ElementHeap{}
+	heap.Init(h)
+
+	it := ldb.NewIterator(nil, nil)
+	var dev protocol.DeviceID
+	var ele SizedElement
+	for it.Next() {
+		key := it.Key()
+		switch key[0] {
+		case db.KeyTypeDevice:
+			folder := nulString(key[1 : 1+64])
+			devBytes := key[1+64 : 1+64+32]
+			name := nulString(key[1+64+32:])
+			copy(dev[:], devBytes)
+			ele.key = fmt.Sprintf("DEVICE:%s:%s:%s", dev, folder, name)
+
+		case db.KeyTypeGlobal:
+			folder := nulString(key[1 : 1+64])
+			name := nulString(key[1+64:])
+			ele.key = fmt.Sprintf("GLOBAL:%s:%s", folder, name)
+
+		case db.KeyTypeBlock:
+			folder := nulString(key[1 : 1+64])
+			hash := key[1+64 : 1+64+32]
+			name := nulString(key[1+64+32:])
+			ele.key = fmt.Sprintf("BLOCK:%s:%x:%s", folder, hash, name)
+
+		case db.KeyTypeDeviceStatistic:
+			ele.key = fmt.Sprintf("DEVICESTATS:%s", key[1:])
+
+		case db.KeyTypeFolderStatistic:
+			ele.key = fmt.Sprintf("FOLDERSTATS:%s", key[1:])
+
+		case db.KeyTypeVirtualMtime:
+			ele.key = fmt.Sprintf("MTIME:%s", key[1:])
+
+		default:
+			ele.key = fmt.Sprintf("UNKNOWN:%x", key)
+		}
+		ele.size = len(it.Value())
+		heap.Push(h, ele)
+	}
+
+	for h.Len() > 0 {
+		ele = heap.Pop(h).(SizedElement)
+		fmt.Println(ele.key, ele.size)
+	}
+}

cmd/stindex/main.go

@@ -11,45 +11,42 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"path/filepath"
 
-	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/internal/db"
 	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
 )
 
 func main() {
+	var mode string
 	log.SetFlags(0)
 	log.SetOutput(os.Stdout)
 
-	folder := flag.String("folder", "default", "Folder ID")
-	device := flag.String("device", "", "Device ID (blank for global)")
+	flag.StringVar(&mode, "mode", "dump", "Mode of operation: dump, dumpsize")
+
 	flag.Parse()
 
-	ldb, err := leveldb.OpenFile(flag.Arg(0), nil)
+	path := flag.Arg(0)
+	if path == "" {
+		path = filepath.Join(defaultConfigDir(), "index-v0.11.0.db")
+	}
+
+	fmt.Println("Path:", path)
+
+	ldb, err := leveldb.OpenFile(path, &opt.Options{
+		ErrorIfMissing:         true,
+		Strict:                 opt.StrictAll,
+		OpenFilesCacheCapacity: 100,
+	})
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	fs := db.NewFileSet(*folder, ldb)
-
-	if *device == "" {
-		log.Printf("*** Global index for folder %q", *folder)
-		fs.WithGlobalTruncated(func(fi db.FileIntf) bool {
-			f := fi.(db.FileInfoTruncated)
-			fmt.Println(f)
-			fmt.Println("\t", fs.Availability(f.Name))
-			return true
-		})
+	if mode == "dump" {
+		dump(ldb)
+	} else if mode == "dumpsize" {
+		dumpsize(ldb)
 	} else {
-		n, err := protocol.DeviceIDFromString(*device)
-		if err != nil {
-			log.Fatal(err)
-		}
-		log.Printf("*** Have index for folder %q device %q", *folder, n)
-		fs.WithHaveTruncated(n, func(fi db.FileIntf) bool {
-			f := fi.(db.FileInfoTruncated)
-			fmt.Println(f)
-			return true
-		})
+		fmt.Println("Unknown mode")
 	}
 }

cmd/stindex/util.go

@@ -0,0 +1,52 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+
+	"github.com/syncthing/syncthing/lib/osutil"
+)
+
+func nulString(bs []byte) string {
+	for i := range bs {
+		if bs[i] == 0 {
+			return string(bs[:i])
+		}
+	}
+	return string(bs)
+}
+
+func defaultConfigDir() string {
+	switch runtime.GOOS {
+	case "windows":
+		if p := os.Getenv("LocalAppData"); p != "" {
+			return filepath.Join(p, "Syncthing")
+		}
+		return filepath.Join(os.Getenv("AppData"), "Syncthing")
+
+	case "darwin":
+		dir, err := osutil.ExpandTilde("~/Library/Application Support/Syncthing")
+		if err != nil {
+			log.Fatal(err)
+		}
+		return dir
+
+	default:
+		if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
+			return filepath.Join(xdgCfg, "syncthing")
+		}
+		dir, err := osutil.ExpandTilde("~/.config/syncthing")
+		if err != nil {
+			log.Fatal(err)
+		}
+		return dir
+	}
+}

cmd/stsigtool/main.go

@@ -0,0 +1,122 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"flag"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+
+	"github.com/syncthing/syncthing/lib/signature"
+	"github.com/syncthing/syncthing/lib/upgrade"
+)
+
+func main() {
+	log.SetFlags(0)
+	log.SetOutput(os.Stdout)
+
+	flag.Parse()
+
+	if flag.NArg() < 1 {
+		log.Println(`Usage:
+	stsigtool <command>
+
+Where command is one of:
+
+	gen
+		- generate a new key pair
+
+	sign <privkeyfile> [datafile]
+		- sign a file
+
+	verify <signaturefile> <datafile>
+		- verify a signature, using the built in public key
+
+	verify <signaturefile> <datafile> <pubkeyfile>
+		- verify a signature, using the specified public key file
+`)
+	}
+
+	switch flag.Arg(0) {
+	case "gen":
+		gen()
+	case "sign":
+		sign(flag.Arg(1), flag.Arg(2))
+	case "verify":
+		if flag.NArg() == 4 {
+			verifyWithFile(flag.Arg(1), flag.Arg(2), flag.Arg(3))
+		} else {
+			verifyWithKey(flag.Arg(1), flag.Arg(2), upgrade.SigningKey)
+		}
+	}
+}
+
+func gen() {
+	priv, pub, err := signature.GenerateKeys()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	os.Stdout.Write(priv)
+	os.Stdout.Write(pub)
+}
+
+func sign(keyname, dataname string) {
+	privkey, err := ioutil.ReadFile(keyname)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	var input io.Reader
+	if dataname == "-" || dataname == "" {
+		input = os.Stdin
+	} else {
+		fd, err := os.Open(dataname)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer fd.Close()
+		input = fd
+	}
+
+	sig, err := signature.Sign(privkey, input)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	os.Stdout.Write(sig)
+}
+
+func verifyWithFile(signame, dataname, keyname string) {
+	pubkey, err := ioutil.ReadFile(keyname)
+	if err != nil {
+		log.Fatal(err)
+	}
+	verifyWithKey(signame, dataname, pubkey)
+}
+
+func verifyWithKey(signame, dataname string, pubkey []byte) {
+	sig, err := ioutil.ReadFile(signame)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fd, err := os.Open(dataname)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer fd.Close()
+
+	err = signature.Verify(pubkey, sig, fd)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	log.Println("correct signature")
+}

cmd/stvanity/main.go

@@ -0,0 +1,212 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"flag"
+	"fmt"
+	"math/big"
+	mr "math/rand"
+	"os"
+	"runtime"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+type result struct {
+	id       protocol.DeviceID
+	priv     *ecdsa.PrivateKey
+	derBytes []byte
+}
+
+func main() {
+	flag.Parse()
+	prefix := strings.ToUpper(strings.Replace(flag.Arg(0), "-", "", -1))
+	if len(prefix) > 7 {
+		prefix = prefix[:7] + "-" + prefix[7:]
+	}
+
+	found := make(chan result)
+	stop := make(chan struct{})
+	var count int64
+
+	// Print periodic progress reports.
+	go printProgress(prefix, &count)
+
+	// Run one certificate generator per CPU core.
+	var wg sync.WaitGroup
+	for i := 0; i < runtime.GOMAXPROCS(-1); i++ {
+		wg.Add(1)
+		go func() {
+			generatePrefixed(prefix, &count, found, stop)
+			wg.Done()
+		}()
+	}
+
+	// Save the result, when one has been found.
+	res := <-found
+	close(stop)
+	wg.Wait()
+
+	fmt.Println("Found", res.id)
+	saveCert(res.priv, res.derBytes)
+	fmt.Println("Saved to cert.pem, key.pem")
+}
+
+// Try certificates until one is found that has the prefix at the start of
+// the resulting device ID. Increments count atomically, sends the result to
+// found, returns when stop is closed.
+func generatePrefixed(prefix string, count *int64, found chan<- result, stop <-chan struct{}) {
+	notBefore := time.Now()
+	notAfter := time.Date(2049, 12, 31, 23, 59, 59, 0, time.UTC)
+
+	template := x509.Certificate{
+		SerialNumber: new(big.Int).SetInt64(mr.Int63()),
+		Subject: pkix.Name{
+			CommonName: "syncthing",
+		},
+		NotBefore: notBefore,
+		NotAfter:  notAfter,
+
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth},
+		BasicConstraintsValid: true,
+	}
+
+	priv, err := ecdsa.GenerateKey(elliptic.P384(), rand.Reader)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	for {
+		select {
+		case <-stop:
+			return
+		default:
+		}
+
+		derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
+		id := protocol.NewDeviceID(derBytes)
+		atomic.AddInt64(count, 1)
+
+		if strings.HasPrefix(id.String(), prefix) {
+			select {
+			case found <- result{id, priv, derBytes}:
+			case <-stop:
+			}
+			return
+		}
+	}
+}
+
+func printProgress(prefix string, count *int64) {
+	started := time.Now()
+	wantBits := 5 * len(prefix)
+	if wantBits > 63 {
+		fmt.Printf("Want %d bits for prefix %q, refusing to boil the ocean.\n", wantBits, prefix)
+		os.Exit(1)
+	}
+	expectedIterations := float64(int(1) << uint(wantBits))
+	fmt.Printf("Want %d bits for prefix %q, about %.2g certs to test (statistically speaking)\n", wantBits, prefix, expectedIterations)
+
+	for _ = range time.NewTicker(15 * time.Second).C {
+		tried := atomic.LoadInt64(count)
+		elapsed := time.Since(started)
+		rate := float64(tried) / elapsed.Seconds()
+		expected := timeStr(expectedIterations / rate)
+		fmt.Printf("Trying %.0f certs/s, tested %d so far in %v, expect ~%s total time to complete\n", rate, tried, elapsed/time.Second*time.Second, expected)
+	}
+}
+
+func saveCert(priv interface{}, derBytes []byte) {
+	certOut, err := os.Create("cert.pem")
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	err = certOut.Close()
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	keyOut, err := os.OpenFile("key.pem", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	block, err := pemBlockForKey(priv)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	err = pem.Encode(keyOut, block)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	err = keyOut.Close()
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func pemBlockForKey(priv interface{}) (*pem.Block, error) {
+	switch k := priv.(type) {
+	case *rsa.PrivateKey:
+		return &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)}, nil
+	case *ecdsa.PrivateKey:
+		b, err := x509.MarshalECPrivateKey(k)
+		if err != nil {
+			return nil, err
+		}
+		return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b}, nil
+	default:
+		return nil, fmt.Errorf("unknown key type")
+	}
+}
+
+func timeStr(seconds float64) string {
+	if seconds < 60 {
+		return fmt.Sprintf("%.0fs", seconds)
+	}
+	if seconds < 3600 {
+		return fmt.Sprintf("%.0fm", seconds/60)
+	}
+	if seconds < 86400 {
+		return fmt.Sprintf("%.0fh", seconds/3600)
+	}
+	if seconds < 86400*365 {
+		return fmt.Sprintf("%.0f days", seconds/3600)
+	}
+	return fmt.Sprintf("%.0f years", seconds/86400/365)
+}

cmd/stwatchfile/main.go

@@ -0,0 +1,98 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"crypto/md5"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"time"
+)
+
+func getmd5(filePath string) ([]byte, error) {
+	var result []byte
+	file, err := os.Open(filePath)
+	if err != nil {
+		return result, err
+	}
+	defer file.Close()
+
+	hash := md5.New()
+	if _, err := io.Copy(hash, file); err != nil {
+		return result, err
+	}
+
+	return hash.Sum(result), nil
+}
+
+func main() {
+	period := flag.Duration("period", 200*time.Millisecond, "Sleep period between checks")
+	flag.Parse()
+
+	file := flag.Arg(0)
+
+	if file == "" {
+		fmt.Println("Expects a path as an argument")
+		return
+	}
+
+	exists := true
+	size := int64(0)
+	mtime := time.Time{}
+	hash := []byte{}
+
+	for {
+		time.Sleep(*period)
+
+		newExists := true
+		fi, err := os.Stat(file)
+		if err != nil && os.IsNotExist(err) {
+			newExists = false
+		} else if err != nil {
+			fmt.Println("stat:", err)
+			return
+		}
+
+		if newExists != exists {
+			exists = newExists
+			if !newExists {
+				fmt.Println(file, "does not exist")
+			} else {
+				fmt.Println(file, "appeared")
+			}
+		}
+
+		if !exists {
+			size = 0
+			mtime = time.Time{}
+			hash = []byte{}
+			continue
+		}
+
+		if fi.IsDir() {
+			fmt.Println(file, "is directory")
+			return
+		}
+		newSize := fi.Size()
+		newMtime := fi.ModTime()
+
+		newHash, err := getmd5(file)
+		if err != nil {
+			fmt.Println("getmd5:", err)
+		}
+
+		if newSize != size || newMtime != mtime || !bytes.Equal(newHash, hash) {
+			fmt.Println(file, "Size:", newSize, "Mtime:", newMtime, "Hash:", fmt.Sprintf("%x", newHash))
+			hash = newHash
+			size = newSize
+			mtime = newMtime
+		}
+	}
+}

cmd/syncthing/auditservice.go

@@ -10,20 +10,20 @@ import (
 	"encoding/json"
 	"io"
 
-	"github.com/syncthing/syncthing/internal/events"
+	"github.com/syncthing/syncthing/lib/events"
 )
 
-// The auditSvc subscribes to events and writes these in JSON format, one
+// The auditService subscribes to events and writes these in JSON format, one
 // event per line, to the specified writer.
-type auditSvc struct {
+type auditService struct {
 	w       io.Writer     // audit destination
 	stop    chan struct{} // signals time to stop
 	started chan struct{} // signals startup complete
 	stopped chan struct{} // signals stop complete
 }
 
-func newAuditSvc(w io.Writer) *auditSvc {
-	return &auditSvc{
+func newAuditService(w io.Writer) *auditService {
+	return &auditService{
 		w:       w,
 		stop:    make(chan struct{}),
 		started: make(chan struct{}),
@@ -32,7 +32,7 @@ func newAuditSvc(w io.Writer) *auditSvc {
 }
 
 // Serve runs the audit service.
-func (s *auditSvc) Serve() {
+func (s *auditService) Serve() {
 	defer close(s.stopped)
 	sub := events.Default.Subscribe(events.AllEvents)
 	defer events.Default.Unsubscribe(sub)
@@ -52,18 +52,18 @@ func (s *auditSvc) Serve() {
 }
 
 // Stop stops the audit service.
-func (s *auditSvc) Stop() {
+func (s *auditService) Stop() {
 	close(s.stop)
 }
 
 // WaitForStart returns once the audit service is ready to receive events, or
 // immediately if it's already running.
-func (s *auditSvc) WaitForStart() {
+func (s *auditService) WaitForStart() {
 	<-s.started
 }
 
 // WaitForStop returns once the audit service has stopped.
 // (Needed by the tests.)
-func (s *auditSvc) WaitForStop() {
+func (s *auditService) WaitForStop() {
 	<-s.stopped
 }

cmd/syncthing/auditservice_test.go

@@ -12,18 +12,18 @@ import (
 	"testing"
 	"time"
 
-	"github.com/syncthing/syncthing/internal/events"
+	"github.com/syncthing/syncthing/lib/events"
 )
 
 func TestAuditService(t *testing.T) {
 	buf := new(bytes.Buffer)
-	svc := newAuditSvc(buf)
+	service := newAuditService(buf)
 
 	// Event sent before start, will not be logged
 	events.Default.Log(events.Ping, "the first event")
 
-	go svc.Serve()
-	svc.WaitForStart()
+	go service.Serve()
+	service.WaitForStart()
 
 	// Event that should end up in the audit log
 	events.Default.Log(events.Ping, "the second event")
@@ -31,8 +31,8 @@ func TestAuditService(t *testing.T) {
 	// We need to give the events time to arrive, since the channels are buffered etc.
 	time.Sleep(10 * time.Millisecond)
 
-	svc.Stop()
-	svc.WaitForStop()
+	service.Stop()
+	service.WaitForStop()
 
 	// This event should not be logged, since we have stopped.
 	events.Default.Log(events.Ping, "the third event")

cmd/syncthing/connections.go

@@ -1,355 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package main
-
-import (
-	"crypto/tls"
-	"fmt"
-	"io"
-	"net"
-	"strings"
-	"time"
-
-	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/internal/config"
-	"github.com/syncthing/syncthing/internal/events"
-	"github.com/syncthing/syncthing/internal/model"
-	"github.com/thejerf/suture"
-)
-
-// The connection service listens on TLS and dials configured unconnected
-// devices. Successful connections are handed to the model.
-type connectionSvc struct {
-	*suture.Supervisor
-	cfg    *config.Wrapper
-	myID   protocol.DeviceID
-	model  *model.Model
-	tlsCfg *tls.Config
-	conns  chan *tls.Conn
-}
-
-func newConnectionSvc(cfg *config.Wrapper, myID protocol.DeviceID, model *model.Model, tlsCfg *tls.Config) *connectionSvc {
-	svc := &connectionSvc{
-		Supervisor: suture.NewSimple("connectionSvc"),
-		cfg:        cfg,
-		myID:       myID,
-		model:      model,
-		tlsCfg:     tlsCfg,
-		conns:      make(chan *tls.Conn),
-	}
-
-	// There are several moving parts here; one routine per listening address
-	// to handle incoming connections, one routine to periodically attempt
-	// outgoing connections, and lastly one routine to the the common handling
-	// regardless of whether the connection was incoming or outgoing. It ends
-	// up as in the diagram below. We embed a Supervisor to manage the
-	// routines (i.e. log and restart if they crash or exit, etc).
-	//
-	//                +-----------------+
-	//    Incoming    | +---------------+-+      +-----------------+
-	//   Connections  | |                 |      |                 |   Outgoing
-	// -------------->| |   svc.listen    |      |                 |  Connections
-	//                | |  (1 per listen  |      |   svc.connect   |-------------->
-	//                | |    address)     |      |                 |
-	//                +-+                 |      |                 |
-	//                  +-----------------+      +-----------------+
-	//                           v                        v
-	//                           |                        |
-	//                           |                        |
-	//                           +------------+-----------+
-	//                                        |
-	//                                        | svc.conns
-	//                                        v
-	//                               +-----------------+
-	//                               |                 |
-	//                               |                 |
-	//                               |   svc.handle    |------> model.AddConnection()
-	//                               |                 |
-	//                               |                 |
-	//                               +-----------------+
-	//
-	// TODO: Clean shutdown, and/or handling config changes on the fly. We
-	// partly do this now - new devices and addresses will be picked up, but
-	// not new listen addresses and we don't support disconnecting devices
-	// that are removed and so on...
-
-	svc.Add(serviceFunc(svc.connect))
-	for _, addr := range svc.cfg.Options().ListenAddress {
-		addr := addr
-		listener := serviceFunc(func() {
-			svc.listen(addr)
-		})
-		svc.Add(listener)
-	}
-	svc.Add(serviceFunc(svc.handle))
-
-	return svc
-}
-
-func (s *connectionSvc) handle() {
-next:
-	for conn := range s.conns {
-		cs := conn.ConnectionState()
-
-		// We should have negotiated the next level protocol "bep/1.0" as part
-		// of the TLS handshake. Unfortunately this can't be a hard error,
-		// because there are implementations out there that don't support
-		// protocol negotiation (iOS for one...).
-		if !cs.NegotiatedProtocolIsMutual || cs.NegotiatedProtocol != bepProtocolName {
-			l.Infof("Peer %s did not negotiate bep/1.0", conn.RemoteAddr())
-		}
-
-		// We should have received exactly one certificate from the other
-		// side. If we didn't, they don't have a device ID and we drop the
-		// connection.
-		certs := cs.PeerCertificates
-		if cl := len(certs); cl != 1 {
-			l.Infof("Got peer certificate list of length %d != 1 from %s; protocol error", cl, conn.RemoteAddr())
-			conn.Close()
-			continue
-		}
-		remoteCert := certs[0]
-		remoteID := protocol.NewDeviceID(remoteCert.Raw)
-
-		// The device ID should not be that of ourselves. It can happen
-		// though, especially in the presence of NAT hairpinning, multiple
-		// clients between the same NAT gateway, and global discovery.
-		if remoteID == myID {
-			l.Infof("Connected to myself (%s) - should not happen", remoteID)
-			conn.Close()
-			continue
-		}
-
-		// We should not already be connected to the other party. TODO: This
-		// could use some better handling. If the old connection is dead but
-		// hasn't timed out yet we may want to drop *that* connection and keep
-		// this one. But in case we are two devices connecting to each other
-		// in parallel we don't want to do that or we end up with no
-		// connections still established...
-		if s.model.ConnectedTo(remoteID) {
-			l.Infof("Connected to already connected device (%s)", remoteID)
-			conn.Close()
-			continue
-		}
-
-		for deviceID, deviceCfg := range s.cfg.Devices() {
-			if deviceID == remoteID {
-				// Verify the name on the certificate. By default we set it to
-				// "syncthing" when generating, but the user may have replaced
-				// the certificate and used another name.
-				certName := deviceCfg.CertName
-				if certName == "" {
-					certName = tlsDefaultCommonName
-				}
-				err := remoteCert.VerifyHostname(certName)
-				if err != nil {
-					// Incorrect certificate name is something the user most
-					// likely wants to know about, since it's an advanced
-					// config. Warn instead of Info.
-					l.Warnf("Bad certificate from %s (%v): %v", remoteID, conn.RemoteAddr(), err)
-					conn.Close()
-					continue next
-				}
-
-				// If rate limiting is set, and based on the address we should
-				// limit the connection, then we wrap it in a limiter.
-
-				limit := s.shouldLimit(conn.RemoteAddr())
-
-				wr := io.Writer(conn)
-				if limit && writeRateLimit != nil {
-					wr = &limitedWriter{conn, writeRateLimit}
-				}
-
-				rd := io.Reader(conn)
-				if limit && readRateLimit != nil {
-					rd = &limitedReader{conn, readRateLimit}
-				}
-
-				name := fmt.Sprintf("%s-%s", conn.LocalAddr(), conn.RemoteAddr())
-				protoConn := protocol.NewConnection(remoteID, rd, wr, s.model, name, deviceCfg.Compression)
-
-				l.Infof("Established secure connection to %s at %s", remoteID, name)
-				if debugNet {
-					l.Debugf("cipher suite: %04X in lan: %t", conn.ConnectionState().CipherSuite, !limit)
-				}
-				events.Default.Log(events.DeviceConnected, map[string]string{
-					"id":   remoteID.String(),
-					"addr": conn.RemoteAddr().String(),
-				})
-
-				s.model.AddConnection(conn, protoConn)
-				continue next
-			}
-		}
-
-		if !s.cfg.IgnoredDevice(remoteID) {
-			events.Default.Log(events.DeviceRejected, map[string]string{
-				"device":  remoteID.String(),
-				"address": conn.RemoteAddr().String(),
-			})
-			l.Infof("Connection from %s with unknown device ID %s", conn.RemoteAddr(), remoteID)
-		} else {
-			l.Infof("Connection from %s with ignored device ID %s", conn.RemoteAddr(), remoteID)
-		}
-
-		conn.Close()
-	}
-}
-
-func (s *connectionSvc) listen(addr string) {
-	if debugNet {
-		l.Debugln("listening on", addr)
-	}
-
-	tcaddr, err := net.ResolveTCPAddr("tcp", addr)
-	if err != nil {
-		l.Fatalln("listen (BEP):", err)
-	}
-	listener, err := net.ListenTCP("tcp", tcaddr)
-	if err != nil {
-		l.Fatalln("listen (BEP):", err)
-	}
-
-	for {
-		conn, err := listener.Accept()
-		if err != nil {
-			l.Warnln("Accepting connection:", err)
-			continue
-		}
-
-		if debugNet {
-			l.Debugln("connect from", conn.RemoteAddr())
-		}
-
-		tcpConn := conn.(*net.TCPConn)
-		s.setTCPOptions(tcpConn)
-
-		tc := tls.Server(conn, s.tlsCfg)
-		err = tc.Handshake()
-		if err != nil {
-			l.Infoln("TLS handshake:", err)
-			tc.Close()
-			continue
-		}
-
-		s.conns <- tc
-	}
-}
-
-func (s *connectionSvc) connect() {
-	delay := time.Second
-	for {
-	nextDevice:
-		for deviceID, deviceCfg := range s.cfg.Devices() {
-			if deviceID == myID {
-				continue
-			}
-
-			if s.model.ConnectedTo(deviceID) {
-				continue
-			}
-
-			var addrs []string
-			for _, addr := range deviceCfg.Addresses {
-				if addr == "dynamic" {
-					if discoverer != nil {
-						t := discoverer.Lookup(deviceID)
-						if len(t) == 0 {
-							continue
-						}
-						addrs = append(addrs, t...)
-					}
-				} else {
-					addrs = append(addrs, addr)
-				}
-			}
-
-			for _, addr := range addrs {
-				host, port, err := net.SplitHostPort(addr)
-				if err != nil && strings.HasPrefix(err.Error(), "missing port") {
-					// addr is on the form "1.2.3.4"
-					addr = net.JoinHostPort(addr, "22000")
-				} else if err == nil && port == "" {
-					// addr is on the form "1.2.3.4:"
-					addr = net.JoinHostPort(host, "22000")
-				}
-				if debugNet {
-					l.Debugln("dial", deviceCfg.DeviceID, addr)
-				}
-
-				raddr, err := net.ResolveTCPAddr("tcp", addr)
-				if err != nil {
-					if debugNet {
-						l.Debugln(err)
-					}
-					continue
-				}
-
-				conn, err := net.DialTCP("tcp", nil, raddr)
-				if err != nil {
-					if debugNet {
-						l.Debugln(err)
-					}
-					continue
-				}
-
-				s.setTCPOptions(conn)
-
-				tc := tls.Client(conn, s.tlsCfg)
-				err = tc.Handshake()
-				if err != nil {
-					l.Infoln("TLS handshake:", err)
-					tc.Close()
-					continue
-				}
-
-				s.conns <- tc
-				continue nextDevice
-			}
-		}
-
-		time.Sleep(delay)
-		delay *= 2
-		if maxD := time.Duration(s.cfg.Options().ReconnectIntervalS) * time.Second; delay > maxD {
-			delay = maxD
-		}
-	}
-}
-
-func (*connectionSvc) setTCPOptions(conn *net.TCPConn) {
-	var err error
-	if err = conn.SetLinger(0); err != nil {
-		l.Infoln(err)
-	}
-	if err = conn.SetNoDelay(false); err != nil {
-		l.Infoln(err)
-	}
-	if err = conn.SetKeepAlivePeriod(60 * time.Second); err != nil {
-		l.Infoln(err)
-	}
-	if err = conn.SetKeepAlive(true); err != nil {
-		l.Infoln(err)
-	}
-}
-
-func (s *connectionSvc) shouldLimit(addr net.Addr) bool {
-	if s.cfg.Options().LimitBandwidthInLan {
-		return true
-	}
-
-	tcpaddr, ok := addr.(*net.TCPAddr)
-	if !ok {
-		return true
-	}
-	for _, lan := range lans {
-		if lan.Contains(tcpaddr.IP) {
-			return false
-		}
-	}
-	return !tcpaddr.IP.IsLoopback()
-}

cmd/syncthing/debug.go

@@ -9,9 +9,20 @@ package main
 import (
 	"os"
 	"strings"
+
+	"github.com/syncthing/syncthing/lib/logger"
 )
 
 var (
-	debugNet  = strings.Contains(os.Getenv("STTRACE"), "net") || os.Getenv("STTRACE") == "all"
-	debugHTTP = strings.Contains(os.Getenv("STTRACE"), "http") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger.NewFacility("main", "Main package")
+	httpl = logger.DefaultLogger.NewFacility("http", "REST API")
 )
+
+func shouldDebugHTTP() bool {
+	return l.ShouldDebug("http")
+}
+
+func init() {
+	l.SetDebug("main", strings.Contains(os.Getenv("STTRACE"), "main") || os.Getenv("STTRACE") == "all")
+	l.SetDebug("http", strings.Contains(os.Getenv("STTRACE"), "http") || os.Getenv("STTRACE") == "all")
+}

cmd/syncthing/gui_auth.go

@@ -9,13 +9,14 @@ package main
 import (
 	"bytes"
 	"encoding/base64"
-	"math/rand"
 	"net/http"
 	"strings"
 	"time"
 
-	"github.com/syncthing/syncthing/internal/config"
-	"github.com/syncthing/syncthing/internal/sync"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/rand"
+	"github.com/syncthing/syncthing/lib/sync"
 	"golang.org/x/crypto/bcrypt"
 )
 
@@ -24,14 +25,21 @@ var (
 	sessionsMut = sync.NewMutex()
 )
 
-func basicAuthAndSessionMiddleware(cfg config.GUIConfiguration, next http.Handler) http.Handler {
+func emitLoginAttempt(success bool, username string) {
+	events.Default.Log(events.LoginAttempt, map[string]interface{}{
+		"success":  success,
+		"username": username,
+	})
+}
+
+func basicAuthAndSessionMiddleware(cookieName string, cfg config.GUIConfiguration, next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if cfg.APIKey != "" && r.Header.Get("X-API-Key") == cfg.APIKey {
+		if cfg.IsValidAPIKey(r.Header.Get("X-API-Key")) {
 			next.ServeHTTP(w, r)
 			return
 		}
 
-		cookie, err := r.Cookie("sessionid")
+		cookie, err := r.Cookie(cookieName)
 		if err == nil && cookie != nil {
 			sessionsMut.Lock()
 			_, ok := sessions[cookie.Value]
@@ -42,9 +50,7 @@ func basicAuthAndSessionMiddleware(cfg config.GUIConfiguration, next http.Handle
 			}
 		}
 
-		if debugHTTP {
-			l.Debugln("Sessionless HTTP request with authentication; this is expensive.")
-		}
+		httpl.Debugln("Sessionless HTTP request with authentication; this is expensive.")
 
 		error := func() {
 			time.Sleep(time.Duration(rand.Intn(100)+100) * time.Millisecond)
@@ -71,26 +77,65 @@ func basicAuthAndSessionMiddleware(cfg config.GUIConfiguration, next http.Handle
 			return
 		}
 
-		if string(fields[0]) != cfg.User {
-			error()
-			return
+		// Check if the username is correct, assuming it was sent as UTF-8
+		username := string(fields[0])
+		if username == cfg.User {
+			goto usernameOK
 		}
 
-		if err := bcrypt.CompareHashAndPassword([]byte(cfg.Password), fields[1]); err != nil {
-			error()
-			return
+		// ... check it again, converting it from assumed ISO-8859-1 to UTF-8
+		username = string(iso88591ToUTF8(fields[0]))
+		if username == cfg.User {
+			goto usernameOK
 		}
 
-		sessionid := randomString(32)
+		// Neither of the possible interpretations match the configured username
+		emitLoginAttempt(false, username)
+		error()
+		return
+
+	usernameOK:
+		// Check password as given (assumes UTF-8 encoding)
+		password := fields[1]
+		if err := bcrypt.CompareHashAndPassword([]byte(cfg.Password), password); err == nil {
+			goto passwordOK
+		}
+
+		// ... check it again, converting it from assumed ISO-8859-1 to UTF-8
+		password = iso88591ToUTF8(password)
+		if err := bcrypt.CompareHashAndPassword([]byte(cfg.Password), password); err == nil {
+			goto passwordOK
+		}
+
+		// Neither of the attempts to verify the password checked out
+		emitLoginAttempt(false, username)
+		error()
+		return
+
+	passwordOK:
+		sessionid := rand.String(32)
 		sessionsMut.Lock()
 		sessions[sessionid] = true
 		sessionsMut.Unlock()
 		http.SetCookie(w, &http.Cookie{
-			Name:   "sessionid",
+			Name:   cookieName,
 			Value:  sessionid,
 			MaxAge: 0,
 		})
 
+		emitLoginAttempt(true, username)
 		next.ServeHTTP(w, r)
 	})
 }
+
+// Convert an ISO-8859-1 encoded byte string to UTF-8. Works by the
+// principle that ISO-8859-1 bytes are equivalent to unicode code points,
+// that a rune slice is a list of code points, and that stringifying a slice
+// of runes generates UTF-8 in Go.
+func iso88591ToUTF8(s []byte) []byte {
+	runes := make([]rune, len(s))
+	for i := range s {
+		runes[i] = rune(s[i])
+	}
+	return []byte(string(runes))
+}

cmd/syncthing/gui_csrf.go

@@ -12,33 +12,43 @@ import (
 	"net/http"
 	"os"
 	"strings"
-	"time"
 
-	"github.com/syncthing/syncthing/internal/osutil"
-	"github.com/syncthing/syncthing/internal/sync"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/rand"
+	"github.com/syncthing/syncthing/lib/sync"
 )
 
+// csrfTokens is a list of valid tokens. It is sorted so that the most
+// recently used token is first in the list. New tokens are added to the front
+// of the list (as it is the most recently used at that time). The list is
+// pruned to a maximum of maxCsrfTokens, throwing away the least recently used
+// tokens.
 var csrfTokens []string
 var csrfMut = sync.NewMutex()
 
+const maxCsrfTokens = 25
+
 // Check for CSRF token on /rest/ URLs. If a correct one is not given, reject
 // the request with 403. For / and /index.html, set a new CSRF cookie if none
 // is currently set.
-func csrfMiddleware(prefix, apiKey string, next http.Handler) http.Handler {
+func csrfMiddleware(unique string, prefix string, cfg config.GUIConfiguration, next http.Handler) http.Handler {
 	loadCsrfTokens()
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// Allow requests carrying a valid API key
-		if apiKey != "" && r.Header.Get("X-API-Key") == apiKey {
+		if cfg.IsValidAPIKey(r.Header.Get("X-API-Key")) {
 			next.ServeHTTP(w, r)
 			return
 		}
 
-		// Allow requests for the front page, and set a CSRF cookie if there isn't already a valid one.
+		// Allow requests for anything not under the protected path prefix,
+		// and set a CSRF cookie if there isn't already a valid one.
 		if !strings.HasPrefix(r.URL.Path, prefix) {
-			cookie, err := r.Cookie("CSRF-Token")
+			cookie, err := r.Cookie("CSRF-Token-" + unique)
 			if err != nil || !validCsrfToken(cookie.Value) {
+				httpl.Debugln("new CSRF cookie in response to request for", r.URL)
 				cookie = &http.Cookie{
-					Name:  "CSRF-Token",
+					Name:  "CSRF-Token-" + unique,
 					Value: newCsrfToken(),
 				}
 				http.SetCookie(w, cookie)
@@ -47,14 +57,8 @@ func csrfMiddleware(prefix, apiKey string, next http.Handler) http.Handler {
 			return
 		}
 
-		if r.Method == "GET" {
-			// Allow GET requests unconditionally
-			next.ServeHTTP(w, r)
-			return
-		}
-
 		// Verify the CSRF token
-		token := r.Header.Get("X-CSRF-Token")
+		token := r.Header.Get("X-CSRF-Token-" + unique)
 		if !validCsrfToken(token) {
 			http.Error(w, "CSRF Error", 403)
 			return
@@ -67,8 +71,15 @@ func csrfMiddleware(prefix, apiKey string, next http.Handler) http.Handler {
 func validCsrfToken(token string) bool {
 	csrfMut.Lock()
 	defer csrfMut.Unlock()
-	for _, t := range csrfTokens {
+	for i, t := range csrfTokens {
 		if t == token {
+			if i > 0 {
+				// Move this token to the head of the list. Copy the tokens at
+				// the front one step to the right and then replace the token
+				// at the head.
+				copy(csrfTokens[1:], csrfTokens[:i+1])
+				csrfTokens[0] = token
+			}
 			return true
 		}
 	}
@@ -76,12 +87,12 @@ func validCsrfToken(token string) bool {
 }
 
 func newCsrfToken() string {
-	token := randomString(32)
+	token := rand.String(32)
 
 	csrfMut.Lock()
-	csrfTokens = append(csrfTokens, token)
-	if len(csrfTokens) > 10 {
-		csrfTokens = csrfTokens[len(csrfTokens)-10:]
+	csrfTokens = append([]string{token}, csrfTokens...)
+	if len(csrfTokens) > maxCsrfTokens {
+		csrfTokens = csrfTokens[:maxCsrfTokens]
 	}
 	defer csrfMut.Unlock()
 
@@ -91,28 +102,20 @@ func newCsrfToken() string {
 }
 
 func saveCsrfTokens() {
-	name := locations[locCsrfTokens]
-	tmp := fmt.Sprintf("%s.tmp.%d", name, time.Now().UnixNano())
+	// We're ignoring errors in here. It's not super critical and there's
+	// nothing relevant we can do about them anyway...
 
-	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
+	name := locations[locCsrfTokens]
+	f, err := osutil.CreateAtomic(name, 0600)
 	if err != nil {
 		return
 	}
-	defer os.Remove(tmp)
 
 	for _, t := range csrfTokens {
-		_, err := fmt.Fprintln(f, t)
-		if err != nil {
-			return
-		}
+		fmt.Fprintln(f, t)
 	}
 
-	err = f.Close()
-	if err != nil {
-		return
-	}
-
-	osutil.Rename(tmp, name)
+	f.Close()
 }
 
 func loadCsrfTokens() {

cmd/syncthing/gui_statics.go

@@ -0,0 +1,176 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io/ioutil"
+	"mime"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/syncthing/syncthing/lib/auto"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/sync"
+)
+
+type staticsServer struct {
+	assetDir        string
+	assets          map[string][]byte
+	availableThemes []string
+
+	mut   sync.RWMutex
+	theme string
+}
+
+func newStaticsServer(theme, assetDir string) *staticsServer {
+	s := &staticsServer{
+		assetDir: assetDir,
+		assets:   auto.Assets(),
+		mut:      sync.NewRWMutex(),
+		theme:    theme,
+	}
+
+	seen := make(map[string]struct{})
+	// Load themes from compiled in assets.
+	for file := range auto.Assets() {
+		theme := strings.Split(file, "/")[0]
+		if _, ok := seen[theme]; !ok {
+			seen[theme] = struct{}{}
+			s.availableThemes = append(s.availableThemes, theme)
+		}
+	}
+	if assetDir != "" {
+		// Load any extra themes from the asset override dir.
+		for _, dir := range dirNames(assetDir) {
+			if _, ok := seen[dir]; !ok {
+				seen[dir] = struct{}{}
+				s.availableThemes = append(s.availableThemes, dir)
+			}
+		}
+	}
+
+	return s
+}
+
+func (s *staticsServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	switch r.URL.Path {
+	case "/themes.json":
+		s.serveThemes(w, r)
+	default:
+		s.serveAsset(w, r)
+	}
+}
+
+func (s *staticsServer) serveAsset(w http.ResponseWriter, r *http.Request) {
+	file := r.URL.Path
+
+	if file[0] == '/' {
+		file = file[1:]
+	}
+
+	if len(file) == 0 {
+		file = "index.html"
+	}
+
+	s.mut.RLock()
+	theme := s.theme
+	s.mut.RUnlock()
+
+	// Check for an override for the current theme.
+	if s.assetDir != "" {
+		p := filepath.Join(s.assetDir, theme, filepath.FromSlash(file))
+		if _, err := os.Stat(p); err == nil {
+			http.ServeFile(w, r, p)
+			return
+		}
+	}
+
+	// Check for a compiled in asset for the current theme.
+	bs, ok := s.assets[theme+"/"+file]
+	if !ok {
+		// Check for an overridden default asset.
+		if s.assetDir != "" {
+			p := filepath.Join(s.assetDir, config.DefaultTheme, filepath.FromSlash(file))
+			if _, err := os.Stat(p); err == nil {
+				http.ServeFile(w, r, p)
+				return
+			}
+		}
+
+		// Check for a compiled in default asset.
+		bs, ok = s.assets[config.DefaultTheme+"/"+file]
+		if !ok {
+			http.NotFound(w, r)
+			return
+		}
+	}
+
+	mtype := s.mimeTypeForFile(file)
+	if len(mtype) != 0 {
+		w.Header().Set("Content-Type", mtype)
+	}
+	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
+		w.Header().Set("Content-Encoding", "gzip")
+	} else {
+		// ungzip if browser not send gzip accepted header
+		var gr *gzip.Reader
+		gr, _ = gzip.NewReader(bytes.NewReader(bs))
+		bs, _ = ioutil.ReadAll(gr)
+		gr.Close()
+	}
+	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(bs)))
+
+	w.Write(bs)
+}
+
+func (s *staticsServer) serveThemes(w http.ResponseWriter, r *http.Request) {
+	sendJSON(w, map[string][]string{
+		"themes": s.availableThemes,
+	})
+}
+
+func (s *staticsServer) mimeTypeForFile(file string) string {
+	// We use a built in table of the common types since the system
+	// TypeByExtension might be unreliable. But if we don't know, we delegate
+	// to the system.
+	ext := filepath.Ext(file)
+	switch ext {
+	case ".htm", ".html":
+		return "text/html"
+	case ".css":
+		return "text/css"
+	case ".js":
+		return "application/javascript"
+	case ".json":
+		return "application/json"
+	case ".png":
+		return "image/png"
+	case ".ttf":
+		return "application/x-font-ttf"
+	case ".woff":
+		return "application/x-font-woff"
+	case ".svg":
+		return "image/svg+xml"
+	default:
+		return mime.TypeByExtension(ext)
+	}
+}
+
+func (s *staticsServer) setTheme(theme string) {
+	s.mut.Lock()
+	s.theme = theme
+	s.mut.Unlock()
+}
+
+func (s *staticsServer) String() string {
+	return fmt.Sprintf("staticsServer@%p", s)
+}

cmd/syncthing/gui_test.go

@@ -0,0 +1,615 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/d4l3k/messagediff"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/thejerf/suture"
+)
+
+func TestCSRFToken(t *testing.T) {
+	t1 := newCsrfToken()
+	t2 := newCsrfToken()
+
+	t3 := newCsrfToken()
+	if !validCsrfToken(t3) {
+		t.Fatal("t3 should be valid")
+	}
+
+	for i := 0; i < 250; i++ {
+		if i%5 == 0 {
+			// t1 and t2 should remain valid by virtue of us checking them now
+			// and then.
+			if !validCsrfToken(t1) {
+				t.Fatal("t1 should be valid at iteration", i)
+			}
+			if !validCsrfToken(t2) {
+				t.Fatal("t2 should be valid at iteration", i)
+			}
+		}
+
+		// The newly generated token is always valid
+		t4 := newCsrfToken()
+		if !validCsrfToken(t4) {
+			t.Fatal("t4 should be valid at iteration", i)
+		}
+	}
+
+	if validCsrfToken(t3) {
+		t.Fatal("t3 should have expired by now")
+	}
+}
+
+func TestStopAfterBrokenConfig(t *testing.T) {
+	cfg := config.Configuration{
+		GUI: config.GUIConfiguration{
+			RawAddress: "127.0.0.1:0",
+			RawUseTLS:  false,
+		},
+	}
+	w := config.Wrap("/dev/null", cfg)
+
+	srv := newAPIService(protocol.LocalDeviceID, w, "../../test/h1/https-cert.pem", "../../test/h1/https-key.pem", "", nil, nil, nil, nil, nil, nil)
+	srv.started = make(chan string)
+
+	sup := suture.NewSimple("test")
+	sup.Add(srv)
+	sup.ServeBackground()
+
+	<-srv.started
+
+	// Service is now running, listening on a random port on localhost. Now we
+	// request a config change to a completely invalid listen address. The
+	// commit will fail and the service will be in a broken state.
+
+	newCfg := config.Configuration{
+		GUI: config.GUIConfiguration{
+			RawAddress: "totally not a valid address",
+			RawUseTLS:  false,
+		},
+	}
+	if err := srv.VerifyConfiguration(cfg, newCfg); err == nil {
+		t.Fatal("Verify config should have failed")
+	}
+
+	// Nonetheless, it should be fine to Stop() it without panic.
+
+	sup.Stop()
+}
+
+func TestAssetsDir(t *testing.T) {
+	// For any given request to $FILE, we should return the first found of
+	//  - assetsdir/$THEME/$FILE
+	//  - compiled in asset $THEME/$FILE
+	//  - assetsdir/default/$FILE
+	//  - compiled in asset default/$FILE
+
+	// The asset map contains compressed assets, so create a couple of gzip compressed assets here.
+	buf := new(bytes.Buffer)
+	gw := gzip.NewWriter(buf)
+	gw.Write([]byte("default"))
+	gw.Close()
+	def := buf.Bytes()
+
+	buf = new(bytes.Buffer)
+	gw = gzip.NewWriter(buf)
+	gw.Write([]byte("foo"))
+	gw.Close()
+	foo := buf.Bytes()
+
+	e := &staticsServer{
+		theme:    "foo",
+		mut:      sync.NewRWMutex(),
+		assetDir: "testdata",
+		assets: map[string][]byte{
+			"foo/a":     foo, // overridden in foo/a
+			"foo/b":     foo,
+			"default/a": def, // overridden in default/a (but foo/a takes precedence)
+			"default/b": def, // overridden in default/b (but foo/b takes precedence)
+			"default/c": def,
+		},
+	}
+
+	s := httptest.NewServer(e)
+	defer s.Close()
+
+	// assetsdir/foo/a exists, overrides compiled in
+	expectURLToContain(t, s.URL+"/a", "overridden-foo")
+
+	// foo/b is compiled in, default/b is overridden, return compiled in
+	expectURLToContain(t, s.URL+"/b", "foo")
+
+	// only exists as compiled in default/c so use that
+	expectURLToContain(t, s.URL+"/c", "default")
+
+	// only exists as overridden default/d so use that
+	expectURLToContain(t, s.URL+"/d", "overridden-default")
+}
+
+func expectURLToContain(t *testing.T, url, exp string) {
+	res, err := http.Get(url)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if res.StatusCode != 200 {
+		t.Errorf("Got %s instead of 200 OK", res.Status)
+		return
+	}
+
+	data, err := ioutil.ReadAll(res.Body)
+	res.Body.Close()
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if string(data) != exp {
+		t.Errorf("Got %q instead of %q on %q", data, exp, url)
+		return
+	}
+}
+
+func TestDirNames(t *testing.T) {
+	names := dirNames("testdata")
+	expected := []string{"default", "foo", "testfolder"}
+	if diff, equal := messagediff.PrettyDiff(expected, names); !equal {
+		t.Errorf("Unexpected dirNames return: %#v\n%s", names, diff)
+	}
+}
+
+type httpTestCase struct {
+	URL     string        // URL to check
+	Code    int           // Expected result code
+	Type    string        // Expected content type
+	Prefix  string        // Expected result prefix
+	Timeout time.Duration // Defaults to a second
+}
+
+func TestAPIServiceRequests(t *testing.T) {
+	const testAPIKey = "foobarbaz"
+	cfg := new(mockedConfig)
+	cfg.gui.APIKey = testAPIKey
+	baseURL, err := startHTTP(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cases := []httpTestCase{
+		// /rest/db
+		{
+			URL:    "/rest/db/completion?device=" + protocol.LocalDeviceID.String() + "&folder=default",
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "{",
+		},
+		{
+			URL:  "/rest/db/file?folder=default&file=something",
+			Code: 404,
+		},
+		{
+			URL:    "/rest/db/ignores?folder=default",
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "{",
+		},
+		{
+			URL:    "/rest/db/need?folder=default",
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "{",
+		},
+		{
+			URL:    "/rest/db/status?folder=default",
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "{",
+		},
+		{
+			URL:    "/rest/db/browse?folder=default",
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "null",
+		},
+
+		// /rest/stats
+		{
+			URL:    "/rest/stats/device",
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "null",
+		},
+		{
+			URL:    "/rest/stats/folder",
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "null",
+		},
+
+		// /rest/svc
+		{
+			URL:    "/rest/svc/deviceid?id=" + protocol.LocalDeviceID.String(),
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "{",
+		},
+		{
+			URL:    "/rest/svc/lang",
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "[",
+		},
+		{
+			URL:     "/rest/svc/report",
+			Code:    200,
+			Type:    "application/json",
+			Prefix:  "{",
+			Timeout: 5 * time.Second,
+		},
+
+		// /rest/system
+		{
+			URL:    "/rest/system/browse?current=~",
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "[",
+		},
+		{
+			URL:    "/rest/system/config",
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "{",
+		},
+		{
+			URL:    "/rest/system/config/insync",
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "{",
+		},
+		{
+			URL:    "/rest/system/connections",
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "null",
+		},
+		{
+			URL:    "/rest/system/discovery",
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "{",
+		},
+		{
+			URL:    "/rest/system/error?since=0",
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "{",
+		},
+		{
+			URL:    "/rest/system/ping",
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "{",
+		},
+		{
+			URL:    "/rest/system/status",
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "{",
+		},
+		{
+			URL:    "/rest/system/version",
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "{",
+		},
+		{
+			URL:    "/rest/system/debug",
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "{",
+		},
+		{
+			URL:    "/rest/system/log?since=0",
+			Code:   200,
+			Type:   "application/json",
+			Prefix: "{",
+		},
+		{
+			URL:    "/rest/system/log.txt?since=0",
+			Code:   200,
+			Type:   "text/plain",
+			Prefix: "",
+		},
+	}
+
+	for _, tc := range cases {
+		t.Log("Testing", tc.URL, "...")
+		testHTTPRequest(t, baseURL, tc, testAPIKey)
+	}
+}
+
+// testHTTPRequest tries the given test case, comparing the result code,
+// content type, and result prefix.
+func testHTTPRequest(t *testing.T, baseURL string, tc httpTestCase, apikey string) {
+	timeout := time.Second
+	if tc.Timeout > 0 {
+		timeout = tc.Timeout
+	}
+	cli := &http.Client{
+		Timeout: timeout,
+	}
+
+	req, err := http.NewRequest("GET", baseURL+tc.URL, nil)
+	if err != nil {
+		t.Errorf("Unexpected error requesting %s: %v", tc.URL, err)
+		return
+	}
+	req.Header.Set("X-API-Key", apikey)
+
+	resp, err := cli.Do(req)
+	if err != nil {
+		t.Errorf("Unexpected error requesting %s: %v", tc.URL, err)
+		return
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != tc.Code {
+		t.Errorf("Get on %s should have returned status code %d, not %s", tc.URL, tc.Code, resp.Status)
+		return
+	}
+
+	ct := resp.Header.Get("Content-Type")
+	if !strings.HasPrefix(ct, tc.Type) {
+		t.Errorf("The content type on %s should be %q, not %q", tc.URL, tc.Type, ct)
+		return
+	}
+
+	data, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Errorf("Unexpected error reading %s: %v", tc.URL, err)
+		return
+	}
+
+	if !bytes.HasPrefix(data, []byte(tc.Prefix)) {
+		t.Errorf("Returned data from %s does not have prefix %q: %s", tc.URL, tc.Prefix, data)
+		return
+	}
+}
+
+func TestHTTPLogin(t *testing.T) {
+	cfg := new(mockedConfig)
+	cfg.gui.User = "üser"
+	cfg.gui.Password = "$2a$10$IdIZTxTg/dCNuNEGlmLynOjqg4B1FvDKuIV5e0BB3pnWVHNb8.GSq" // bcrypt of "räksmörgås" in UTF-8
+	baseURL, err := startHTTP(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Verify rejection when not using authorization
+
+	req, _ := http.NewRequest("GET", baseURL, nil)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.StatusCode != http.StatusUnauthorized {
+		t.Errorf("Unexpected non-401 return code %d for unauthed request", resp.StatusCode)
+	}
+
+	// Verify that incorrect password is rejected
+
+	req.SetBasicAuth("üser", "rksmrgs")
+	resp, err = http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.StatusCode != http.StatusUnauthorized {
+		t.Errorf("Unexpected non-401 return code %d for incorrect password", resp.StatusCode)
+	}
+
+	// Verify that incorrect username is rejected
+
+	req.SetBasicAuth("user", "räksmörgås") // string literals in Go source code are in UTF-8
+	resp, err = http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.StatusCode != http.StatusUnauthorized {
+		t.Errorf("Unexpected non-401 return code %d for incorrect username", resp.StatusCode)
+	}
+
+	// Verify that UTF-8 auth works
+
+	req.SetBasicAuth("üser", "räksmörgås") // string literals in Go source code are in UTF-8
+	resp, err = http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		t.Errorf("Unexpected non-200 return code %d for authed request (UTF-8)", resp.StatusCode)
+	}
+
+	// Verify that ISO-8859-1 auth
+
+	req.SetBasicAuth("\xfcser", "r\xe4ksm\xf6rg\xe5s") // escaped ISO-8859-1
+	resp, err = http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		t.Errorf("Unexpected non-200 return code %d for authed request (ISO-8859-1)", resp.StatusCode)
+	}
+
+}
+
+func startHTTP(cfg *mockedConfig) (string, error) {
+	model := new(mockedModel)
+	httpsCertFile := "../../test/h1/https-cert.pem"
+	httpsKeyFile := "../../test/h1/https-key.pem"
+	assetDir := "../../gui"
+	eventSub := new(mockedEventSub)
+	discoverer := new(mockedCachingMux)
+	connections := new(mockedConnections)
+	errorLog := new(mockedLoggerRecorder)
+	systemLog := new(mockedLoggerRecorder)
+	addrChan := make(chan string)
+
+	// Instantiate the API service
+	svc := newAPIService(protocol.LocalDeviceID, cfg, httpsCertFile, httpsKeyFile, assetDir, model,
+		eventSub, discoverer, connections, errorLog, systemLog)
+	svc.started = addrChan
+
+	// Actually start the API service
+	supervisor := suture.NewSimple("API test")
+	supervisor.Add(svc)
+	supervisor.ServeBackground()
+
+	// Make sure the API service is listening, and get the URL to use.
+	addr := <-addrChan
+	tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
+	if err != nil {
+		return "", fmt.Errorf("Weird address from API service: %v", err)
+	}
+	baseURL := fmt.Sprintf("http://127.0.0.1:%d", tcpAddr.Port)
+
+	return baseURL, nil
+}
+
+func TestCSRFRequired(t *testing.T) {
+	const testAPIKey = "foobarbaz"
+	cfg := new(mockedConfig)
+	cfg.gui.APIKey = testAPIKey
+	baseURL, err := startHTTP(cfg)
+
+	cli := &http.Client{
+		Timeout: time.Second,
+	}
+
+	// Getting the base URL (i.e. "/") should succeed.
+
+	resp, err := cli.Get(baseURL)
+	if err != nil {
+		t.Fatal("Unexpected error from getting base URL:", err)
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatal("Getting base URL should succeed, not", resp.Status)
+	}
+
+	// Find the returned CSRF token for future use
+
+	var csrfTokenName, csrfTokenValue string
+	for _, cookie := range resp.Cookies() {
+		if strings.HasPrefix(cookie.Name, "CSRF-Token") {
+			csrfTokenName = cookie.Name
+			csrfTokenValue = cookie.Value
+			break
+		}
+	}
+
+	// Calling on /rest without a token should fail
+
+	resp, err = cli.Get(baseURL + "/rest/system/config")
+	if err != nil {
+		t.Fatal("Unexpected error from getting /rest/system/config:", err)
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusForbidden {
+		t.Fatal("Getting /rest/system/config without CSRF token should fail, not", resp.Status)
+	}
+
+	// Calling on /rest with a token should succeed
+
+	req, _ := http.NewRequest("GET", baseURL+"/rest/system/config", nil)
+	req.Header.Set("X-"+csrfTokenName, csrfTokenValue)
+	resp, err = cli.Do(req)
+	if err != nil {
+		t.Fatal("Unexpected error from getting /rest/system/config:", err)
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatal("Getting /rest/system/config with CSRF token should succeed, not", resp.Status)
+	}
+
+	// Calling on /rest with the API key should succeed
+
+	req, _ = http.NewRequest("GET", baseURL+"/rest/system/config", nil)
+	req.Header.Set("X-API-Key", testAPIKey)
+	resp, err = cli.Do(req)
+	if err != nil {
+		t.Fatal("Unexpected error from getting /rest/system/config:", err)
+	}
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatal("Getting /rest/system/config with API key should succeed, not", resp.Status)
+	}
+}
+
+func TestRandomString(t *testing.T) {
+	const testAPIKey = "foobarbaz"
+	cfg := new(mockedConfig)
+	cfg.gui.APIKey = testAPIKey
+	baseURL, err := startHTTP(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	cli := &http.Client{
+		Timeout: time.Second,
+	}
+
+	// The default should be to return a 32 character random string
+
+	for _, url := range []string{"/rest/svc/random/string", "/rest/svc/random/string?length=-1", "/rest/svc/random/string?length=yo"} {
+		req, _ := http.NewRequest("GET", baseURL+url, nil)
+		req.Header.Set("X-API-Key", testAPIKey)
+		resp, err := cli.Do(req)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		var res map[string]string
+		if err := json.NewDecoder(resp.Body).Decode(&res); err != nil {
+			t.Fatal(err)
+		}
+		if len(res["random"]) != 32 {
+			t.Errorf("Expected 32 random characters, got %q of length %d", res["random"], len(res["random"]))
+		}
+	}
+
+	// We can ask for a different length if we like
+
+	req, _ := http.NewRequest("GET", baseURL+"/rest/svc/random/string?length=27", nil)
+	req.Header.Set("X-API-Key", testAPIKey)
+	resp, err := cli.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var res map[string]string
+	if err := json.NewDecoder(resp.Body).Decode(&res); err != nil {
+		t.Fatal(err)
+	}
+	if len(res["random"]) != 27 {
+		t.Errorf("Expected 27 random characters, got %q of length %d", res["random"], len(res["random"]))
+	}
+}