Docs & translations update

An error on opening .stignore will satisfy os.IsNotExist() and not be
reported. Other errors will be reported and stop the folder, including
is-not-exist errors from #include as these are passed through fmt.Errorf.

Also fixes minor issue where we would not print cause of folder stopping
to the log.

Also fixes minor issue with capitalization of errors.

.gitattributes

@@ -2,7 +2,8 @@
 *   text=auto
 
 # Except the dependencies, which we leave alone
-vendor/**   -text=auto
+Godeps/**   -text=auto
 
 # Diffs on these files are meaningless
+gui.files.go    -diff
 *.svg           -diff

.gitignore

@@ -1,19 +1,16 @@
-/syncthing
-/stdiscosrv
+./syncthing
 syncthing.exe
-stdiscosrv.exe
 *.tar.gz
 *.zip
 *.asc
-*.deb
 .jshintrc
 coverage.out
 files/pidx
 bin
 perfstats*.csv
 coverage.xml
-syncthing.sig
+!gui/scripts/syncthing
+syncthing.md5
+syncthing.exe.md5
 RELEASE
 deb
-lib/auto/gui.files.go
-snapcraft.yaml

AUTHORS

@@ -1,112 +1,67 @@
 # This is the official list of Syncthing authors for copyright purposes.
-# The format is:
-#
-#    Name Name Name (nickname) <email1@example.com> <email2@example.com>
-#
-# The NICKS list is auto generated from this file.
 
-Aaron Bieber (qbit) <qbit@deftly.net>
-Adam Piggott (ProactiveServices) <aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com> <ProactiveServices@users.noreply.github.com>
-Adel Qalieh (adelq) <aqalieh95@gmail.com> <adelq@users.noreply.github.com>
-Alessandro G. (alessandro.g89) <alessandro.g89@gmail.com>
-Alexander Graf (alex2108) <register-github@alex-graf.de>
-Alexandre Viau (aviau) <alexandre@alexandreviau.net> <aviau@debian.org>
-Anderson Mesquita (andersonvom) <andersonvom@gmail.com>
-Andrew Dunham (andrew-d) <andrew@du.nham.ca>
-Andrey D (scienmind) <scintertech@cryptolab.net>
-Antoine Lamielle (0x010C) <antoine.lamielle@0x010c.fr> <gh@0x010c.fr>
-Antony Male (canton7) <antony.male@gmail.com>
-Arthur Axel fREW Schmidt (frioux) <frew@afoolishmanifesto.com> <frioux@gmail.com>
-Audrius Butkevicius (AudriusButkevicius) <audrius.butkevicius@gmail.com>
-Bart De Vries (mogwa1) <devriesb@gmail.com>
-Ben Curthoys (bencurthoys) <ben@bencurthoys.com>
-Ben Schulz (uok) <ueomkail@gmail.com> <uok@users.noreply.github.com>
-Ben Shepherd (benshep) <bjashepherd@gmail.com>
-Ben Sidhom (bsidhom) <bsidhom@gmail.com>
-Benny Ng (tpng) <benny.tpng@gmail.com>
-Brandon Philips (philips) <brandon@ifup.org>
-Brendan Long (brendanlong) <self@brendanlong.com>
-Brian R. Becker (brbecker) <brbecker@gmail.com>
-Caleb Callaway (cqcallaw) <enlightened.despot@gmail.com>
-Carsten Hagemann (Moter8) <moter8@gmail.com>
-Cathryne Linenweaver (Cathryne) <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
-Cedric Staniewski (xduugu) <cedric@gmx.ca>
-Chris Howie (cdhowie) <me@chrishowie.com>
-Chris Joel (cdata) <chris@scriptolo.gy>
-Colin Kennedy (moshen) <moshen.colin@gmail.com>
-Daniel Bergmann (brgmnn) <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
-Daniel Harte (norgeous) <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
-Daniel Martí (mvdan) <mvdan@mvdan.cc>
-David Rimmer (dinosore) <dinosore@dbrsoftware.co.uk>
-Denis A. (dva) <denisva@gmail.com>
-Dennis Wilson (snnd) <dw@risu.io>
-Dominik Heidler (asdil12) <dominik@heidler.eu>
-Elias Jarlebring (jarlebring) <jarlebring@gmail.com>
-Emil Hessman (ceh) <emil@hessman.se>
-Erik Meitner (WSGCSysadmin) <e.meitner@willystreet.coop>
-Federico Castagnini (facastagnini) <federico.castagnini@gmail.com>
-Felix Ableitner (Nutomic) <me@nutomic.com>
-Felix Unterpaintner (bigbear2nd) <bigbear2nd@gmail.com>
-Francois-Xavier Gsell (zukoo) <fxgsell@gmail.com>
-Frank Isemann (fti7) <frank@isemann.name>
-Gilli Sigurdsson (gillisig) <gilli@vx.is>
-Heiko Zuerker (Smiley73) <heiko@zuerker.org>
-Jaakko Hannikainen (jgke) <jgke@jgke.fi>
-Jacek Szafarkiewicz (hadogenes) <szafar@linux.pl>
-Jake Peterson (acogdev) <jake@acogdev.com>
-Jakob Borg (calmh) <jakob@nym.se> <jakob@kastelo.net>
-James Patterson (jpjp) <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
-Jaroslav Malec (dzarda) <dzardacz@gmail.com>
-Jens Diemer (jedie) <github.com@jensdiemer.de> <git@jensdiemer.de>
-Jochen Voss (seehuhn) <voss@seehuhn.de>
-Johan Vromans (sciurius) <jvromans@squirrel.nl>
-Karol Różycki (krozycki) <rozycki.karol@gmail.com>
-Kelong Cong (kc1212) <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
-Ken'ichi Kamada (kamadak) <kamada@nanohz.org>
-Kevin Allen (ironmig) <kma1660@gmail.com>
-Kevin White, Jr. (kwhite17) <kevinwhite1710@gmail.com>
-Kurt Fitzner (Kudalufi) <kurt@va1der.ca>
-Lars K.W. Gohlke (lkwg82) <lkwg82@gmx.de>
-Laurent Etiemble (letiemble) <laurent.etiemble@gmail.com> <laurent.etiemble@monobjc.net>
-Leo Arias (elopio) <yo@elopio.net>
-Lode Hoste (Zillode) <zillode@zillode.be>
-Lord Landon Agahnim (LordLandon) <lordlandon@gmail.com>
-Majed Abdulaziz (majedev) <majed.alhajry@gmail.com>
-Marc Laporte (marclaporte) <marc@marclaporte.com> <marc@laporte.name>
-Marc Pujol (kilburn) <kilburn@la3.org>
-Marcin Dziadus (marcindziadus) <dziadus.marcin@gmail.com>
-Mark Pulford (mpx) <mark@kyne.com.au>
-Mateusz Naściszewski (mateon1) <matin1111@wp.pl>
-Matt Burke (burkemw3) <mburke@amplify.com> <burkemw3@gmail.com>
-Max Schulze (kralo) <max.schulze@online.de> <kralo@users.noreply.github.com>
-Michael Jephcote (Rewt0r) <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
-Michael Ploujnikov (plouj) <ploujj@gmail.com>
-Michael Tilli (pyfisch) <pyfisch@gmail.com>
-Nate Morrison (nrm21) <natemorrison@gmail.com>
-Niels Peter Roest (Niller303) <nielsproest@hotmail.com> <seje.niels@hotmail.com>
-Pascal Jungblut (pascalj) <github@pascalj.com> <mail@pascal-jungblut.com>
-Peter Hoeg (peterhoeg) <peter@speartail.com>
-Philippe Schommers (filoozoom) <philippe@schommers.be>
-Phill Luby (pluby) <phill.luby@newredo.com>
-Piotr Bejda (piobpl) <piotrb10@gmail.com>
-Roman Zaynetdinov (zaynetro) <romanznet@gmail.com>
-Ryan Sullivan (KayoticSully) <kayoticsully@gmail.com>
-Scott Klupfel (kluppy) <kluppy@going2blue.com>
-Sergey Mishin (ralder) <ralder@yandex.ru>
-Simon Frei (imsodin) <freisim93@gmail.com>
-Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
-Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
-Tim Abell (timabell) <tim@timwise.co.uk>
-Tim Howes (timhowes) <timhowes@berkeley.edu>
-Tobias Nygren (tnn2) <tnn@nygren.pp.se>
-Tomas Cerveny (kozec) <kozec@kozec.com>
-Tully Robinson (tojrobinson) <tully@tojr.org>
-Tyler Brazier (tylerbrazier) <tyler@tylerbrazier.com>
-Unrud (Unrud) <unrud@openaliasbox.org> <Unrud@users.noreply.github.com>
-Veeti Paananen (veeti) <veeti.paananen@rojekti.fi>
-Victor Buinsky (buinsky) <vix_booja@tut.by>
-Vil Brekin (Vilbrekin) <vilbrekin@gmail.com>
-William A. Kennington III (wkennington) <william@wkennington.com>
-Wulf Weich (wweich) <wweich@users.noreply.github.com> <wweich@gmx.de>
-Xavier O. (damajor) <damajor@gmail.com>
-Yannic A. (eipiminus1) <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>
+Aaron Bieber <qbit@deftly.net>
+Adam Piggott	<aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com>
+Alexander Graf <register-github@alex-graf.de>
+Andrew Dunham <andrew@du.nham.ca>
+Antony Male <antony.male@gmail.com>
+Arthur Axel fREW Schmidt <frew@afoolishmanifesto.com> <frioux@gmail.com>
+Audrius Butkevicius <audrius.butkevicius@gmail.com>
+Bart De Vries <devriesb@gmail.com>
+Ben Curthoys <ben@bencurthoys.com>
+Ben Schulz <ueomkail@gmail.com> <uok@users.noreply.github.com>
+Ben Sidhom <bsidhom@gmail.com>
+Brandon Philips <brandon@ifup.org>
+Brendan Long <self@brendanlong.com>
+Brian R. Becker <brbecker@gmail.com>
+Caleb Callaway <enlightened.despot@gmail.com>
+Carsten Hagemann <moter8@gmail.com>
+Cathryne Linenweaver <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
+Chris Howie <me@chrishowie.com>
+Chris Joel <chris@scriptolo.gy>
+Colin Kennedy <moshen.colin@gmail.com>
+Daniel Bergmann <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
+Daniel Martí <mvdan@mvdan.cc>
+Denis A. <denisva@gmail.com>
+Dennis Wilson <dw@risu.io>
+Dominik Heidler <dominik@heidler.eu>
+Elias Jarlebring <jarlebring@gmail.com>
+Emil Hessman <emil@hessman.se>
+Erik Meitner <e.meitner@willystreet.coop>
+Federico Castagnini <federico.castagnini@gmail.com>
+Felix Ableitner <me@nutomic.com>
+Felix Unterpaintner <bigbear2nd@gmail.com>
+Francois-Xavier Gsell <fxgsell@gmail.com>
+Frank Isemann <frank@isemann.name>
+Gilli Sigurdsson <gilli@vx.is>
+Jacek Szafarkiewicz <szafar@linux.pl>
+Jakob Borg <jakob@nym.se>
+James Patterson <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
+Jaroslav Malec <dzardacz@gmail.com>
+Jens Diemer <github.com@jensdiemer.de> <git@jensdiemer.de>
+Jochen Voss <voss@seehuhn.de>
+Johan Vromans <jvromans@squirrel.nl>
+Karol Różycki <rozycki.karol@gmail.com>
+Ken'ichi Kamada <kamada@nanohz.org>
+Lode Hoste <zillode@zillode.be>
+Lord Landon Agahnim <lordlandon@gmail.com>
+Marc Laporte <marc@marclaporte.com> <marc@laporte.name>
+Marc Pujol <kilburn@la3.org>
+Marcin Dziadus <dziadus.marcin@gmail.com>
+Matt Burke <mburke@amplify.com>
+Michael Jephcote <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
+Michael Tilli <pyfisch@gmail.com>
+Pascal Jungblut <github@pascalj.com> <mail@pascal-jungblut.com>
+Peter Hoeg <peter@speartail.com>
+Philippe Schommers <philippe@schommers.be>
+Phill Luby <phill.luby@newredo.com>
+Piotr Bejda <piotrb10@gmail.com>
+Ryan Sullivan <kayoticsully@gmail.com>
+Sergey Mishin <ralder@yandex.ru>
+Stefan Tatschner <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
+Tim Abell <tim@timwise.co.uk>
+Tobias Nygren <tnn@nygren.pp.se>
+Tomas Cerveny <kozec@kozec.com>
+Tully Robinson <tully@tojr.org>
+Veeti Paananen <veeti.paananen@rojekti.fi>
+Vil Brekin <vilbrekin@gmail.com>

CONTRIBUTING.md

@@ -33,31 +33,20 @@ latest info on Transifex.
 
 Every contribution is welcome. If you want to contribute but are unsure
 where to start, any open issues are fair game! See the [Contribution
-Guidelines](https://docs.syncthing.net/dev/contributing.html) for the full
+Guidelines](http://docs.syncthing.net/dev/contributing.html) for the full
 story on committing code.
 
 ## Contributing Documentation
 
-Updates to the [documentation site](https://docs.syncthing.net/) can be
+Updates to the [documentation site](http://docs.syncthing.net/) can be
 made as pull requests on the [documentation
 repository](https://github.com/syncthing/docs).
 
 ## Licensing
 
-All contributions are made available under the same license as the already
-existing material being contributed to. For most of the project and unless
-otherwise stated this means MPLv2, but there are exceptions:
-
-- Certain commands (under cmd/...) may have a separate license, indicated by
-  the presence of a LICENSE file in the corresponding directory.
-
-- The documentation (man/...) is licensed under the Creative Commons
-  Attribution 4.0 International License.
-
-- Projects under vendor/... are copyright by and licensed from their
-  respective original authors. Contributions should be made to the original
-  project, not here.
-
-Regardless of the license in effect, you retain the copyright to your
-contribution.
+All contributions are made under the same MPLv2 license as the rest of
+the project, except documentation, user interface text and translation
+strings which are licensed under the Creative Commons Attribution 4.0
+International License. You retain the copyright to code you have
+written.
 

GOALS.md

@@ -1,83 +0,0 @@
-# The Syncthing Goals
-
-Syncthing is a **continuous file synchronization program**. It synchronizes
-files between two or more computers. We strive to fulfill the goals below.
-The goals are listed in order of importance, the most important one being
-the first.
-
-> "Syncing files" here is precise. It means we specifically exclude things
-> that are not files - calendar items, instant messages, and so on. If those
-> are in fact stored as files on disk, they can of course be synced as
-> files.
-
-Syncthing should be:
-
-### 1. Safe From Data Loss
-
-Protecting the user's data is paramount. We take every reasonable precaution
-to avoid corrupting the user's files.
-
-> This is the overriding goal, without which synchronizing files becomes
-> pointless. This means that we do not make unsafe trade offs for the sake
-> of performance or, in some cases, even usability.
-
-### 2. Secure Against Attackers
-
-Again, protecting the user's data is paramount. Regardless of our other
-goals we must never allow the user's data to be susceptible to eavesdropping
-or modification by unauthorized parties.
-
-> This should be understood in context. It is not necessarily reasonable to
-> expect Syncthing to be resistant against well equipped state level
-> attackers. We will however do our best. Note also that this is different
-> from anonymity which is not, currently, a goal.
-
-### 3. Easy to Use
-
-Syncthing should be approachable, understandable and inclusive.
-
-> Complex concepts and maths form the base of Synchting's functionality.
-> This should nonetheless be abstracted or hidden to a degree where
-> Syncthing is usable by the general public.
-
-### 4. Automatic
-
-User interaction should be required only when absolutely necessary.
-
-> Specifically this means that changes to files are picked up without
-> prompting, conflicts are resolved without prompting and connections are
-> maintained without prompting. We only prompt the user when it is required
-> to fulfill one of the (overriding) Secure, Safe or Easy goals.
-
-### 5. Universally Available
-
-Syncthing should run on every common computer. We are mindful that the
-latest technology is not always available to any given individual.
-
-> Computers include desktops, laptops, servers, virtual machines, small
-> general purpose computers such as Raspberry Pis and, *where possible*,
-> tablets and phones. NAS appliances, toasters, cars, firearms, thermostats
-> and so on may include computing capabitilies but it is not our goal for
-> Syncthing to run smoothly on these devices.
-
-### 6. For Individuals
-
-Syncthing is primarily about empowering the individual user with safe,
-secure and easy to use file synchronization.
-
-> We acknowledge that it's also useful in an enterprise setting and include
-> functionality to support that. If this is in conflict with the
-> requirements of the individual, those will however take priority.
-
-### 7. Everything Else
-
-There are many things we care about that don't make it on to the list. It is
-fine to optimize for these values as well, as long as they are not in
-conflict with the stated goals above.
-
-> For example, performance is a thing we care about. We just don't care more
-> about it than safety, security, etc. Maintainability of the code base and
-> providing entertainment value for the maintainers are also things that
-> matter. It is understood that there are aspects of Syncthing that are
-> suboptimal or even in opposition with the goals above. However, we
-> continuously strive to align Syncthing more and more with these goals.

Godeps/Godeps.json

@@ -0,0 +1,90 @@
+{
+	"ImportPath": "github.com/syncthing/syncthing",
+	"GoVersion": "go1.5",
+	"Packages": [
+		"./cmd/..."
+	],
+	"Deps": [
+		{
+			"ImportPath": "github.com/bkaradzic/go-lz4",
+			"Rev": "d47913b1412890a261b9fefae99d72d2bf5aebd8"
+		},
+		{
+			"ImportPath": "github.com/calmh/du",
+			"Rev": "3c0690cca16228b97741327b1b6781397afbdb24"
+		},
+		{
+			"ImportPath": "github.com/calmh/logger",
+			"Rev": "c96f6a1a8c7b6bf2f4860c667867d90174799eb2"
+		},
+		{
+			"ImportPath": "github.com/calmh/luhn",
+			"Rev": "0c8388ff95fa92d4094011e5a04fc99dea3d1632"
+		},
+		{
+			"ImportPath": "github.com/calmh/xdr",
+			"Rev": "5f7208e86762911861c94f1849eddbfc0a60cbf0"
+		},
+		{
+			"ImportPath": "github.com/golang/snappy",
+			"Rev": "723cc1e459b8eea2dea4583200fd60757d40097a"
+		},
+		{
+			"ImportPath": "github.com/juju/ratelimit",
+			"Rev": "772f5c38e468398c4511514f4f6aa9a4185bc0a0"
+		},
+		{
+			"ImportPath": "github.com/kardianos/osext",
+			"Rev": "6e7f843663477789fac7c02def0d0909e969b4e5"
+		},
+		{
+			"ImportPath": "github.com/syncthing/protocol",
+			"Rev": "388a29bbe21d8772ee4c29f4520aa8040309607d"
+		},
+		{
+			"ImportPath": "github.com/syndtr/goleveldb/leveldb",
+			"Rev": "b743d92d3215f11c9b5ce8830fafe1f16786adf4"
+		},
+		{
+			"ImportPath": "github.com/thejerf/suture",
+			"Comment": "v1.0.1",
+			"Rev": "99c1f2d613756768fc4299acd9dc621e11ed3fd7"
+		},
+		{
+			"ImportPath": "github.com/vitrun/qart/coding",
+			"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
+		},
+		{
+			"ImportPath": "github.com/vitrun/qart/gf256",
+			"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
+		},
+		{
+			"ImportPath": "github.com/vitrun/qart/qr",
+			"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
+		},
+		{
+			"ImportPath": "golang.org/x/crypto/bcrypt",
+			"Rev": "c16968172724c0b5e8bdc6ad33f5a79443a44cd7"
+		},
+		{
+			"ImportPath": "golang.org/x/crypto/blowfish",
+			"Rev": "c16968172724c0b5e8bdc6ad33f5a79443a44cd7"
+		},
+		{
+			"ImportPath": "golang.org/x/net/internal/iana",
+			"Rev": "66f0418ca41253f8d1a024eb9754e9441a8e79b9"
+		},
+		{
+			"ImportPath": "golang.org/x/net/ipv6",
+			"Rev": "66f0418ca41253f8d1a024eb9754e9441a8e79b9"
+		},
+		{
+			"ImportPath": "golang.org/x/text/transform",
+			"Rev": "723492b65e225eafcba054e76ba18bb9c5ac1ea2"
+		},
+		{
+			"ImportPath": "golang.org/x/text/unicode/norm",
+			"Rev": "723492b65e225eafcba054e76ba18bb9c5ac1ea2"
+		}
+	]
+}

Godeps/Readme

@@ -0,0 +1,5 @@
+This directory tree is generated automatically by godep.
+
+Please do not edit.
+
+See https://github.com/tools/godep for more information.

Godeps/_workspace/.gitignore

@@ -0,0 +1,2 @@
+/pkg
+/bin

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/.gitignore

@@ -0,0 +1 @@
+/lz4-example/lz4-example

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/.travis.yml

@@ -0,0 +1,8 @@
+language: go
+
+go:
+  - 1.1
+  - 1.2
+  - 1.3
+  - 1.4
+  - tip

Godeps/_workspace/src/github.com/calmh/logger/README.md

@@ -0,0 +1,15 @@
+logger
+======
+
+A small wrapper around `log` to provide log levels.
+
+Documentation
+-------------
+
+http://godoc.org/github.com/calmh/logger
+
+License
+-------
+
+MIT
+

Godeps/_workspace/src/github.com/calmh/logger/logger.go

@@ -0,0 +1,187 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+// Package logger implements a standardized logger with callback functionality
+package logger
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+	"sync"
+)
+
+type LogLevel int
+
+const (
+	LevelDebug LogLevel = iota
+	LevelVerbose
+	LevelInfo
+	LevelOK
+	LevelWarn
+	LevelFatal
+	NumLevels
+)
+
+// A MessageHandler is called with the log level and message text.
+type MessageHandler func(l LogLevel, msg string)
+
+type Logger struct {
+	logger   *log.Logger
+	handlers [NumLevels][]MessageHandler
+	mut      sync.Mutex
+}
+
+// The default logger logs to standard output with a time prefix.
+var DefaultLogger = New()
+
+func New() *Logger {
+	if os.Getenv("LOGGER_DISCARD") != "" {
+		// Hack to completely disable logging, for example when running benchmarks.
+		return &Logger{
+			logger: log.New(ioutil.Discard, "", 0),
+		}
+	}
+
+	return &Logger{
+		logger: log.New(os.Stdout, "", log.Ltime),
+	}
+}
+
+// AddHandler registers a new MessageHandler to receive messages with the
+// specified log level or above.
+func (l *Logger) AddHandler(level LogLevel, h MessageHandler) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	l.handlers[level] = append(l.handlers[level], h)
+}
+
+// See log.SetFlags
+func (l *Logger) SetFlags(flag int) {
+	l.logger.SetFlags(flag)
+}
+
+// See log.SetPrefix
+func (l *Logger) SetPrefix(prefix string) {
+	l.logger.SetPrefix(prefix)
+}
+
+func (l *Logger) callHandlers(level LogLevel, s string) {
+	for _, h := range l.handlers[level] {
+		h(level, strings.TrimSpace(s))
+	}
+}
+
+// Debugln logs a line with a DEBUG prefix.
+func (l *Logger) Debugln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "DEBUG: "+s)
+	l.callHandlers(LevelDebug, s)
+}
+
+// Debugf logs a formatted line with a DEBUG prefix.
+func (l *Logger) Debugf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "DEBUG: "+s)
+	l.callHandlers(LevelDebug, s)
+}
+
+// Infoln logs a line with a VERBOSE prefix.
+func (l *Logger) Verboseln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "VERBOSE: "+s)
+	l.callHandlers(LevelVerbose, s)
+}
+
+// Infof logs a formatted line with a VERBOSE prefix.
+func (l *Logger) Verbosef(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "VERBOSE: "+s)
+	l.callHandlers(LevelVerbose, s)
+}
+
+// Infoln logs a line with an INFO prefix.
+func (l *Logger) Infoln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "INFO: "+s)
+	l.callHandlers(LevelInfo, s)
+}
+
+// Infof logs a formatted line with an INFO prefix.
+func (l *Logger) Infof(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "INFO: "+s)
+	l.callHandlers(LevelInfo, s)
+}
+
+// Okln logs a line with an OK prefix.
+func (l *Logger) Okln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "OK: "+s)
+	l.callHandlers(LevelOK, s)
+}
+
+// Okf logs a formatted line with an OK prefix.
+func (l *Logger) Okf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "OK: "+s)
+	l.callHandlers(LevelOK, s)
+}
+
+// Warnln logs a formatted line with a WARNING prefix.
+func (l *Logger) Warnln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "WARNING: "+s)
+	l.callHandlers(LevelWarn, s)
+}
+
+// Warnf logs a formatted line with a WARNING prefix.
+func (l *Logger) Warnf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "WARNING: "+s)
+	l.callHandlers(LevelWarn, s)
+}
+
+// Fatalln logs a line with a FATAL prefix and exits the process with exit
+// code 1.
+func (l *Logger) Fatalln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "FATAL: "+s)
+	l.callHandlers(LevelFatal, s)
+	os.Exit(1)
+}
+
+// Fatalf logs a formatted line with a FATAL prefix and exits the process with
+// exit code 1.
+func (l *Logger) Fatalf(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "FATAL: "+s)
+	l.callHandlers(LevelFatal, s)
+	os.Exit(1)
+}

Godeps/_workspace/src/github.com/calmh/logger/logger_test.go

@@ -0,0 +1,58 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+package logger
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestAPI(t *testing.T) {
+	l := New()
+	l.SetFlags(0)
+	l.SetPrefix("testing")
+
+	debug := 0
+	l.AddHandler(LevelDebug, checkFunc(t, LevelDebug, "test 0", &debug))
+	info := 0
+	l.AddHandler(LevelInfo, checkFunc(t, LevelInfo, "test 1", &info))
+	warn := 0
+	l.AddHandler(LevelWarn, checkFunc(t, LevelWarn, "test 2", &warn))
+	ok := 0
+	l.AddHandler(LevelOK, checkFunc(t, LevelOK, "test 3", &ok))
+
+	l.Debugf("test %d", 0)
+	l.Debugln("test", 0)
+	l.Infof("test %d", 1)
+	l.Infoln("test", 1)
+	l.Warnf("test %d", 2)
+	l.Warnln("test", 2)
+	l.Okf("test %d", 3)
+	l.Okln("test", 3)
+
+	if debug != 2 {
+		t.Errorf("Debug handler called %d != 2 times", debug)
+	}
+	if info != 2 {
+		t.Errorf("Info handler called %d != 2 times", info)
+	}
+	if warn != 2 {
+		t.Errorf("Warn handler called %d != 2 times", warn)
+	}
+	if ok != 2 {
+		t.Errorf("Ok handler called %d != 2 times", ok)
+	}
+}
+
+func checkFunc(t *testing.T, expectl LogLevel, expectmsg string, counter *int) func(LogLevel, string) {
+	return func(l LogLevel, msg string) {
+		*counter++
+		if l != expectl {
+			t.Errorf("Incorrect message level %d != %d", l, expectl)
+		}
+		if !strings.HasSuffix(msg, expectmsg) {
+			t.Errorf("%q does not end with %q", msg, expectmsg)
+		}
+	}
+}

Godeps/_workspace/src/github.com/calmh/xdr/.gitignore

@@ -0,0 +1 @@
+coverage.out

Godeps/_workspace/src/github.com/calmh/xdr/.travis.yml

@@ -0,0 +1,19 @@
+language: go
+go:
+- tip
+
+install:
+- export PATH=$PATH:$HOME/gopath/bin
+- go get code.google.com/p/go.tools/cmd/cover
+- go get github.com/mattn/goveralls
+
+script:
+- ./generate.sh
+- go test -coverprofile=coverage.out
+
+after_success:
+- goveralls -coverprofile=coverage.out -service=travis-ci -package=calmh/xdr -repotoken="$COVERALLS_TOKEN"
+
+env:
+  global:
+    secure: SmgnrGfp2zLrA44ChRMpjPeujubt9veZ8Fx/OseMWECmacyV5N/TuDhzIbwo6QwV4xB0sBacoPzvxQbJRVjNKsPiSu72UbcQmQ7flN4Tf7nW09tSh1iW8NgrpBCq/3UYLoBu2iPBEBKm93IK0aGNAKs6oEkB0fU27iTVBwiTXOY=

Godeps/_workspace/src/github.com/calmh/xdr/README.md

@@ -1,10 +1,12 @@
 xdr
 ===
 
-[![Build Status](https://img.shields.io/circleci/project/calmh/xdr.svg?style=flat-square)](https://circleci.com/gh/calmh/xdr)
+[![Build Status](https://img.shields.io/travis/calmh/xdr.svg?style=flat)](https://travis-ci.org/calmh/xdr)
 [![Coverage Status](https://img.shields.io/coveralls/calmh/xdr.svg?style=flat)](https://coveralls.io/r/calmh/xdr?branch=master)
 [![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat)](http://godoc.org/github.com/calmh/xdr)
 [![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://opensource.org/licenses/MIT)
 
-This is an XDR marshalling/unmarshalling library. It uses code generation and
-not reflection.
+This is an XDR encoding/decoding library. It uses code generation and
+not reflection. It supports the IPDR bastardized XDR format when built
+with `-tags ipdr`.
+

Godeps/_workspace/src/github.com/calmh/xdr/bench_test.go

@@ -0,0 +1,117 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+package xdr_test
+
+import (
+	"io"
+	"io/ioutil"
+	"testing"
+
+	"github.com/calmh/xdr"
+)
+
+type XDRBenchStruct struct {
+	I1  uint64
+	I2  uint32
+	I3  uint16
+	I4  uint8
+	Bs0 []byte // max:128
+	Bs1 []byte
+	S0  string // max:128
+	S1  string
+}
+
+var res []byte // no to be optimized away
+var s = XDRBenchStruct{
+	I1:  42,
+	I2:  43,
+	I3:  44,
+	I4:  45,
+	Bs0: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18},
+	Bs1: []byte{11, 12, 13, 14, 15, 16, 17, 18, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
+	S0:  "Hello World! String one.",
+	S1:  "Hello World! String two.",
+}
+var e []byte
+
+func init() {
+	e, _ = s.MarshalXDR()
+}
+
+func BenchmarkThisMarshal(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		res, _ = s.MarshalXDR()
+	}
+}
+
+func BenchmarkThisUnmarshal(b *testing.B) {
+	var t XDRBenchStruct
+	for i := 0; i < b.N; i++ {
+		err := t.UnmarshalXDR(e)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkThisEncode(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		_, err := s.EncodeXDR(ioutil.Discard)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkThisEncoder(b *testing.B) {
+	w := xdr.NewWriter(ioutil.Discard)
+	for i := 0; i < b.N; i++ {
+		_, err := s.EncodeXDRInto(w)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+type repeatReader struct {
+	data []byte
+}
+
+func (r *repeatReader) Read(bs []byte) (n int, err error) {
+	if len(bs) > len(r.data) {
+		err = io.EOF
+	}
+	n = copy(bs, r.data)
+	r.data = r.data[n:]
+	return n, err
+}
+
+func (r *repeatReader) Reset(bs []byte) {
+	r.data = bs
+}
+
+func BenchmarkThisDecode(b *testing.B) {
+	rr := &repeatReader{e}
+	var t XDRBenchStruct
+	for i := 0; i < b.N; i++ {
+		err := t.DecodeXDR(rr)
+		if err != nil {
+			b.Fatal(err)
+		}
+		rr.Reset(e)
+	}
+}
+
+func BenchmarkThisDecoder(b *testing.B) {
+	rr := &repeatReader{e}
+	r := xdr.NewReader(rr)
+	var t XDRBenchStruct
+	for i := 0; i < b.N; i++ {
+		err := t.DecodeXDRFrom(r)
+		if err != nil {
+			b.Fatal(err)
+		}
+		rr.Reset(e)
+	}
+}

Godeps/_workspace/src/github.com/calmh/xdr/bench_xdr_test.go

@@ -0,0 +1,201 @@
+// ************************************************************
+// This file is automatically generated by genxdr. Do not edit.
+// ************************************************************
+
+package xdr_test
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/calmh/xdr"
+)
+
+/*
+
+XDRBenchStruct Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                         I1 (64 bits)                          +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                              I2                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|            0x0000             |              I3               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                        uint8 Structure                        \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of Bs0                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     Bs0 (variable length)                     \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of Bs1                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     Bs1 (variable length)                     \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of S0                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     S0 (variable length)                      \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of S1                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     S1 (variable length)                      \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct XDRBenchStruct {
+	unsigned hyper I1;
+	unsigned int I2;
+	unsigned int I3;
+	uint8 I4;
+	opaque Bs0<128>;
+	opaque Bs1<>;
+	string S0<128>;
+	string S1<>;
+}
+
+*/
+
+func (o XDRBenchStruct) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
+}
+
+func (o XDRBenchStruct) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o XDRBenchStruct) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o XDRBenchStruct) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
+}
+
+func (o XDRBenchStruct) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	xw.WriteUint64(o.I1)
+	xw.WriteUint32(o.I2)
+	xw.WriteUint16(o.I3)
+	xw.WriteUint8(o.I4)
+	if l := len(o.Bs0); l > 128 {
+		return xw.Tot(), xdr.ElementSizeExceeded("Bs0", l, 128)
+	}
+	xw.WriteBytes(o.Bs0)
+	xw.WriteBytes(o.Bs1)
+	if l := len(o.S0); l > 128 {
+		return xw.Tot(), xdr.ElementSizeExceeded("S0", l, 128)
+	}
+	xw.WriteString(o.S0)
+	xw.WriteString(o.S1)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *XDRBenchStruct) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *XDRBenchStruct) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *XDRBenchStruct) DecodeXDRFrom(xr *xdr.Reader) error {
+	o.I1 = xr.ReadUint64()
+	o.I2 = xr.ReadUint32()
+	o.I3 = xr.ReadUint16()
+	o.I4 = xr.ReadUint8()
+	o.Bs0 = xr.ReadBytesMax(128)
+	o.Bs1 = xr.ReadBytes()
+	o.S0 = xr.ReadStringMax(128)
+	o.S1 = xr.ReadString()
+	return xr.Error()
+}
+
+/*
+
+repeatReader Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                        Length of data                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                    data (variable length)                     \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct repeatReader {
+	opaque data<>;
+}
+
+*/
+
+func (o repeatReader) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
+}
+
+func (o repeatReader) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o repeatReader) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o repeatReader) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
+}
+
+func (o repeatReader) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	xw.WriteBytes(o.data)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *repeatReader) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *repeatReader) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *repeatReader) DecodeXDRFrom(xr *xdr.Reader) error {
+	o.data = xr.ReadBytes()
+	return xr.Error()
+}

Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go

@@ -28,7 +28,6 @@ type fieldInfo struct {
 	Encoder   string // the encoder name, i.e. "Uint64" for Read/WriteUint64
 	Convert   string // what to convert to when encoding, i.e. "uint64"
 	Max       int    // max size for slices and strings
-	Submax    int    // max size for strings inside slices
 }
 
 type structInfo struct {
@@ -36,73 +35,31 @@ type structInfo struct {
 	Fields []fieldInfo
 }
 
-func (i structInfo) SizeExpr() string {
-	var xdrSizes = map[string]int{
-		"int8":   4,
-		"uint8":  4,
-		"int16":  4,
-		"uint16": 4,
-		"int32":  4,
-		"uint32": 4,
-		"int64":  8,
-		"uint64": 8,
-		"int":    8,
-		"bool":   4,
-	}
-
-	var terms []string
-	nl := ""
-	for _, f := range i.Fields {
-		if size := xdrSizes[f.FieldType]; size > 0 {
-			if f.IsSlice {
-				terms = append(terms, nl+"4+len(o."+f.Name+")*"+strconv.Itoa(size))
-			} else {
-				terms = append(terms, strconv.Itoa(size))
-			}
-		} else {
-			switch f.FieldType {
-			case "string", "[]byte":
-				if f.IsSlice {
-					terms = append(terms, nl+"4+xdr.SizeOfSlice(o."+f.Name+")")
-				} else {
-					terms = append(terms, nl+"4+len(o."+f.Name+")+xdr.Padding(len(o."+f.Name+"))")
-				}
-			default:
-				if f.IsSlice {
-					terms = append(terms, nl+"4+xdr.SizeOfSlice(o."+f.Name+")")
-				} else {
-					terms = append(terms, nl+"o."+f.Name+".XDRSize()")
-				}
-			}
-		}
-		nl = "\n"
-	}
-	return strings.Join(terms, "+")
-}
-
-var headerData = `// ************************************************************
+var headerTpl = template.Must(template.New("header").Parse(`// ************************************************************
 // This file is automatically generated by genxdr. Do not edit.
 // ************************************************************
 
 package {{.Package}}
 
 import (
+	"bytes"
+	"io"
+
 	"github.com/calmh/xdr"
 )
-`
+`))
 
-var encoderData = `
-func (o {{.Name}}) XDRSize() int {
-	return {{.SizeExpr}}
+var encodeTpl = template.Must(template.New("encoder").Parse(`
+func (o {{.TypeName}}) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
 }//+n
 
-func (o {{.Name}}) MarshalXDR() ([]byte, error) {
-	buf:= make([]byte, o.XDRSize())
-	m := &xdr.Marshaller{Data: buf}
-	return buf, o.MarshalXDRInto(m)
+func (o {{.TypeName}}) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
 }//+n
 
-func (o {{.Name}}) MustMarshalXDR() []byte {
+func (o {{.TypeName}}) MustMarshalXDR() []byte {
 	bs, err := o.MarshalXDR()
 	if err != nil {
 		panic(err)
@@ -110,157 +67,106 @@ func (o {{.Name}}) MustMarshalXDR() []byte {
 	return bs
 }//+n
 
-func (o {{.Name}}) MarshalXDRInto(m *xdr.Marshaller) error {
-	{{range $fi := .Fields}}
-		{{if $fi.IsSlice}}
-			{{template "marshalSlice" $fi}}
-		{{else}}
-			{{template "marshalValue" $fi}}
-		{{end}}
-	{{end}}
-	return m.Error
+func (o {{.TypeName}}) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
 }//+n
 
-{{define "marshalValue"}}
-	{{if ne .Convert ""}}
-		m.Marshal{{.Encoder}}({{.Convert}}(o.{{.Name}}))
-	{{else if .IsBasic}}
-		{{if ge .Max 1}}
-			if l := len(o.{{.Name}}); l > {{.Max}} {
-				return xdr.ElementSizeExceeded("{{.Name}}", l, {{.Max}})
+func (o {{.TypeName}}) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	{{range $fieldInfo := .Fields}}
+		{{if not $fieldInfo.IsSlice}}
+			{{if ne $fieldInfo.Convert ""}}
+				xw.Write{{$fieldInfo.Encoder}}({{$fieldInfo.Convert}}(o.{{$fieldInfo.Name}}))
+			{{else if $fieldInfo.IsBasic}}
+				{{if ge $fieldInfo.Max 1}}
+					if l := len(o.{{$fieldInfo.Name}}); l > {{$fieldInfo.Max}} {
+						return xw.Tot(), xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", l, {{$fieldInfo.Max}})
+					}
+				{{end}}
+				xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}})
+			{{else}}
+				_, err := o.{{$fieldInfo.Name}}.EncodeXDRInto(xw)
+				if err != nil {
+					return xw.Tot(), err
+				}
+			{{end}}
+		{{else}}
+			{{if ge $fieldInfo.Max 1}}
+				if l := len(o.{{$fieldInfo.Name}}); l > {{$fieldInfo.Max}} {
+					return xw.Tot(), xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", l, {{$fieldInfo.Max}})
+				}
+			{{end}}
+			xw.WriteUint32(uint32(len(o.{{$fieldInfo.Name}})))
+			for i := range o.{{$fieldInfo.Name}} {
+			{{if ne $fieldInfo.Convert ""}}
+				xw.Write{{$fieldInfo.Encoder}}({{$fieldInfo.Convert}}(o.{{$fieldInfo.Name}}[i]))
+			{{else if $fieldInfo.IsBasic}}
+				xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}}[i])
+			{{else}}
+				_, err := o.{{$fieldInfo.Name}}[i].EncodeXDRInto(xw)
+				if err != nil {
+					return xw.Tot(), err
+				}
+			{{end}}
 			}
 		{{end}}
-		m.Marshal{{.Encoder}}(o.{{.Name}})
-	{{else}}
-		if err := o.{{.Name}}.MarshalXDRInto(m); err != nil {
-			return err
-		}
 	{{end}}
-{{end}}
-
-{{define "marshalSlice"}}
-	{{if ge .Max 1}}
-		if l := len(o.{{.Name}}); l > {{.Max}} {
-			return xdr.ElementSizeExceeded("{{.Name}}", l, {{.Max}})
-		}
-	{{end}}
-
-	m.MarshalUint32(uint32(len(o.{{.Name}})))
-	for i := range o.{{.Name}} {
-		{{if ne .Convert ""}}
-			m.Marshal{{.Encoder}}({{.Convert}}(o.{{.Name}}[i]))
-		{{else if .IsBasic}}
-			m.Marshal{{.Encoder}}(o.{{.Name}}[i])
-		{{else}}
-			if err := o.{{.Name}}[i].MarshalXDRInto(m); err != nil {
-				return err
-			}
-		{{end}}
-	}
-{{end}}
-
-func (o *{{.Name}}) UnmarshalXDR(bs []byte) error {
-	u := &xdr.Unmarshaller{Data: bs}
-	return o.UnmarshalXDRFrom(u)
-}
-
-func (o *{{.Name}}) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	{{range $fi := .Fields}}
-		{{if $fi.IsSlice}}
-			{{template "unmarshalSlice" $fi}}
-		{{else}}
-			{{template "unmarshalValue" $fi}}
-		{{end}}
-	{{end}}
-	return u.Error
+	return xw.Tot(), xw.Error()
 }//+n
 
-{{define "unmarshalValue"}}
-	{{if ne .Convert ""}}
-		o.{{.Name}} = {{.FieldType}}(u.Unmarshal{{.Encoder}}())
-	{{else if .IsBasic}}
-		{{if ge .Max 1}}
-			o.{{.Name}} = u.Unmarshal{{.Encoder}}Max({{.Max}})
-		{{else}}
-			o.{{.Name}} = u.Unmarshal{{.Encoder}}()
-		{{end}}
-	{{else}}
-		(&o.{{.Name}}).UnmarshalXDRFrom(u)
-	{{end}}
-{{end}}
+func (o *{{.TypeName}}) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}//+n
 
-{{define "unmarshalSlice"}}
-	_{{.Name}}Size := int(u.UnmarshalUint32())
-	if _{{.Name}}Size < 0 {
-		return xdr.ElementSizeExceeded("{{.Name}}", _{{.Name}}Size, {{.Max}})
-	} else if _{{.Name}}Size == 0 {
-		o.{{.Name}} = nil
-	} else {
-		{{if ge .Max 1}}
-			if _{{.Name}}Size > {{.Max}} {
-				return xdr.ElementSizeExceeded("{{.Name}}", _{{.Name}}Size, {{.Max}})
-			}
-		{{end}}
-		if _{{.Name}}Size <= len(o.{{.Name}}) {
-			{{if eq .FieldType "string"}}
-				for i := _{{.Name}}Size; i < len(o.{{.Name}}); i++ { o.{{.Name}}[i] = "" }
-			{{end}}
-			{{if eq .FieldType "[]byte"}}
-				for i := _{{.Name}}Size; i < len(o.{{.Name}}); i++ { o.{{.Name}}[i] = nil }
-			{{end}}
-			o.{{.Name}} = o.{{.Name}}[:_{{.Name}}Size]
-		} else {
-			o.{{.Name}} = make([]{{.FieldType}}, _{{.Name}}Size)
-		}
-		for i := range o.{{.Name}} {
-			{{if ne .Convert ""}}
-				o.{{.Name}}[i] = {{.FieldType}}(u.Unmarshal{{.Encoder}}())
-			{{else if .IsBasic}}
-				{{if ge .Submax 1}}
-					o.{{.Name}}[i] = u.Unmarshal{{.Encoder}}Max({{.Submax}})
+func (o *{{.TypeName}}) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}//+n
+
+func (o *{{.TypeName}}) DecodeXDRFrom(xr *xdr.Reader) error {
+	{{range $fieldInfo := .Fields}}
+		{{if not $fieldInfo.IsSlice}}
+			{{if ne $fieldInfo.Convert ""}}
+				o.{{$fieldInfo.Name}} = {{$fieldInfo.FieldType}}(xr.Read{{$fieldInfo.Encoder}}())
+			{{else if $fieldInfo.IsBasic}}
+				{{if ge $fieldInfo.Max 1}}
+					o.{{$fieldInfo.Name}} = xr.Read{{$fieldInfo.Encoder}}Max({{$fieldInfo.Max}})
 				{{else}}
-					o.{{.Name}}[i] = u.Unmarshal{{.Encoder}}()
+					o.{{$fieldInfo.Name}} = xr.Read{{$fieldInfo.Encoder}}()
 				{{end}}
 			{{else}}
-				(&o.{{.Name}}[i]).UnmarshalXDRFrom(u)
+				(&o.{{$fieldInfo.Name}}).DecodeXDRFrom(xr)
 			{{end}}
-		}
-	}
-{{end}}
-`
+		{{else}}
+			_{{$fieldInfo.Name}}Size := int(xr.ReadUint32())
+			if _{{$fieldInfo.Name}}Size < 0 {
+				return xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", _{{$fieldInfo.Name}}Size, {{$fieldInfo.Max}})
+			}
+			{{if ge $fieldInfo.Max 1}}
+				if _{{$fieldInfo.Name}}Size > {{$fieldInfo.Max}} {
+					return xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", _{{$fieldInfo.Name}}Size, {{$fieldInfo.Max}})
+				}
+			{{end}}
+			o.{{$fieldInfo.Name}} = make([]{{$fieldInfo.FieldType}}, _{{$fieldInfo.Name}}Size)
+			for i := range o.{{$fieldInfo.Name}} {
+				{{if ne $fieldInfo.Convert ""}}
+					o.{{$fieldInfo.Name}}[i] = {{$fieldInfo.FieldType}}(xr.Read{{$fieldInfo.Encoder}}())
+				{{else if $fieldInfo.IsBasic}}
+					o.{{$fieldInfo.Name}}[i] = xr.Read{{$fieldInfo.Encoder}}()
+				{{else}}
+					(&o.{{$fieldInfo.Name}}[i]).DecodeXDRFrom(xr)
+				{{end}}
+			}
+		{{end}}
+	{{end}}
+	return xr.Error()
+}`))
 
-var (
-	encodeTpl = template.Must(template.New("encoder").Parse(encoderData))
-	headerTpl = template.Must(template.New("header").Parse(headerData))
-)
-
-var emptyTypeTpl = template.Must(template.New("encoder").Parse(`
-func (o {{.Name}}) XDRSize() int {
-	return 0
-}
-
-func (o {{.Name}}) MarshalXDR() ([]byte, error) {
-	return nil, nil
-}//+n
-
-func (o {{.Name}}) MustMarshalXDR() []byte {
-	return nil
-}//+n
-
-func (o {{.Name}}) MarshalXDRInto(m *xdr.Marshaller) error {
-	return nil
-}//+n
-
-func (o *{{.Name}}) UnmarshalXDR(bs []byte) error {
-	return nil
-}//+n
-
-func (o *{{.Name}}) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	return nil
-}//+n
-`))
-
-var maxRe = regexp.MustCompile(`(?:\Wmax:)(\d+)(?:\s*,\s*(\d+))?`)
+var maxRe = regexp.MustCompile(`\Wmax:(\d+)`)
 
 type typeSet struct {
 	Type    string
@@ -292,15 +198,11 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 		}
 
 		fn := sf.Names[0].Name
-		var max1, max2 int
+		var max = 0
 		if sf.Comment != nil {
 			c := sf.Comment.List[0].Text
-			m := maxRe.FindStringSubmatch(c)
-			if len(m) >= 2 {
-				max1, _ = strconv.Atoi(m[1])
-			}
-			if len(m) >= 3 {
-				max2, _ = strconv.Atoi(m[2])
+			if m := maxRe.FindStringSubmatch(c); m != nil {
+				max, _ = strconv.Atoi(m[1])
 			}
 			if strings.Contains(c, "noencode") {
 				continue
@@ -318,16 +220,14 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 					FieldType: tn,
 					Encoder:   enc.Encoder,
 					Convert:   enc.Type,
-					Max:       max1,
-					Submax:    max2,
+					Max:       max,
 				}
 			} else {
 				f = fieldInfo{
 					Name:      fn,
 					IsBasic:   false,
 					FieldType: tn,
-					Max:       max1,
-					Submax:    max2,
+					Max:       max,
 				}
 			}
 
@@ -342,11 +242,10 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 				f = fieldInfo{
 					Name:      fn,
 					IsBasic:   true,
-					FieldType: "[]" + tn,
+					FieldType: tn,
 					Encoder:   enc.Encoder,
 					Convert:   enc.Type,
-					Max:       max1,
-					Submax:    max2,
+					Max:       max,
 				}
 			} else if enc, ok := xdrEncoders[tn]; ok {
 				f = fieldInfo{
@@ -356,16 +255,14 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 					FieldType: tn,
 					Encoder:   enc.Encoder,
 					Convert:   enc.Type,
-					Max:       max1,
-					Submax:    max2,
+					Max:       max,
 				}
 			} else {
 				f = fieldInfo{
 					Name:      fn,
 					IsSlice:   true,
 					FieldType: tn,
-					Max:       max1,
-					Submax:    max2,
+					Max:       max,
 				}
 			}
 
@@ -373,8 +270,7 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 			f = fieldInfo{
 				Name:      fn,
 				FieldType: ft.Sel.Name,
-				Max:       max1,
-				Submax:    max2,
+				Max:       max,
 			}
 		}
 
@@ -385,22 +281,23 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 }
 
 func generateCode(output io.Writer, s structInfo) {
+	name := s.Name
+	fs := s.Fields
+
 	var buf bytes.Buffer
-	var err error
-	if len(s.Fields) == 0 {
-		// This is an empty type. We can create a quite simple codec for it.
-		err = emptyTypeTpl.Execute(&buf, s)
-	} else {
-		// Generate with the default template.
-		err = encodeTpl.Execute(&buf, s)
-	}
+	err := encodeTpl.Execute(&buf, map[string]interface{}{"TypeName": name, "Fields": fs})
 	if err != nil {
 		panic(err)
 	}
 
 	bs := regexp.MustCompile(`(\s*\n)+`).ReplaceAll(buf.Bytes(), []byte("\n"))
 	bs = bytes.Replace(bs, []byte("//+n"), []byte("\n"), -1)
-	output.Write(bs)
+
+	bs, err = format.Source(bs)
+	if err != nil {
+		panic(err)
+	}
+	fmt.Fprintln(output, string(bs))
 }
 
 func uncamelize(s string) string {
@@ -414,14 +311,6 @@ func generateDiagram(output io.Writer, s structInfo) {
 	fs := s.Fields
 
 	fmt.Fprintln(output, sn+" Structure:")
-
-	if len(fs) == 0 {
-		fmt.Fprintln(output, "(contains no fields)")
-		fmt.Fprintln(output)
-		fmt.Fprintln(output)
-		return
-	}
-
 	fmt.Fprintln(output)
 	fmt.Fprintln(output, " 0                   1                   2                   3")
 	fmt.Fprintln(output, " 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1")
@@ -432,46 +321,46 @@ func generateDiagram(output io.Writer, s structInfo) {
 		tn := f.FieldType
 		name := uncamelize(f.Name)
 
-		suffix := ""
 		if f.IsSlice {
 			fmt.Fprintf(output, "| %s |\n", center("Number of "+name, 61))
 			fmt.Fprintln(output, line)
-			suffix = " (n items)"
-			fmt.Fprintf(output, "/ %s /\n", center("", 61))
 		}
 		switch tn {
 		case "bool":
 			fmt.Fprintf(output, "| %s |V|\n", center(name+" (V=0 or 1)", 59))
+			fmt.Fprintln(output, line)
 		case "int16", "uint16":
-			fmt.Fprintf(output, "| %s | %s |\n", center("16 zero bits", 29), center(name, 29))
-		case "int8", "uint8":
-			fmt.Fprintf(output, "| %s | %s |\n", center("24 zero bits", 45), center(name, 13))
+			fmt.Fprintf(output, "| %s | %s |\n", center("0x0000", 29), center(name, 29))
+			fmt.Fprintln(output, line)
 		case "int32", "uint32":
-			fmt.Fprintf(output, "| %s |\n", center(name+suffix, 61))
+			fmt.Fprintf(output, "| %s |\n", center(name, 61))
+			fmt.Fprintln(output, line)
 		case "int64", "uint64":
 			fmt.Fprintf(output, "| %-61s |\n", "")
 			fmt.Fprintf(output, "+ %s +\n", center(name+" (64 bits)", 61))
 			fmt.Fprintf(output, "| %-61s |\n", "")
-		case "string", "[]byte":
+			fmt.Fprintln(output, line)
+		case "string", "byte": // XXX We assume slice of byte!
+			fmt.Fprintf(output, "| %s |\n", center("Length of "+name, 61))
+			fmt.Fprintln(output, line)
 			fmt.Fprintf(output, "/ %61s /\n", "")
-			fmt.Fprintf(output, "\\ %s \\\n", center(name+" (length + padded data)", 61))
+			fmt.Fprintf(output, "\\ %s \\\n", center(name+" (variable length)", 61))
 			fmt.Fprintf(output, "/ %61s /\n", "")
+			fmt.Fprintln(output, line)
 		default:
 			if f.IsSlice {
 				tn = "Zero or more " + tn + " Structures"
+				fmt.Fprintf(output, "/ %s /\n", center("", 61))
 				fmt.Fprintf(output, "\\ %s \\\n", center(tn, 61))
+				fmt.Fprintf(output, "/ %s /\n", center("", 61))
 			} else {
 				tn = tn + " Structure"
 				fmt.Fprintf(output, "/ %s /\n", center("", 61))
 				fmt.Fprintf(output, "\\ %s \\\n", center(tn, 61))
 				fmt.Fprintf(output, "/ %s /\n", center("", 61))
 			}
+			fmt.Fprintln(output, line)
 		}
-
-		if f.IsSlice {
-			fmt.Fprintf(output, "/ %s /\n", center("", 61))
-		}
-		fmt.Fprintln(output, line)
 	}
 	fmt.Fprintln(output)
 	fmt.Fprintln(output)
@@ -496,9 +385,9 @@ func generateXdr(output io.Writer, s structInfo) {
 		}
 
 		switch tn {
-		case "int8", "int16", "int32":
+		case "int16", "int32":
 			fmt.Fprintf(output, "\tint %s%s;\n", fn, suf)
-		case "uint8", "uint16", "uint32":
+		case "uint16", "uint32":
 			fmt.Fprintf(output, "\tunsigned int %s%s;\n", fn, suf)
 		case "int64":
 			fmt.Fprintf(output, "\thyper %s%s;\n", fn, suf)
@@ -506,7 +395,7 @@ func generateXdr(output io.Writer, s structInfo) {
 			fmt.Fprintf(output, "\tunsigned hyper %s%s;\n", fn, suf)
 		case "string":
 			fmt.Fprintf(output, "\tstring %s<%s>;\n", fn, l)
-		case "[]byte":
+		case "byte":
 			fmt.Fprintf(output, "\topaque %s<%s>;\n", fn, l)
 		default:
 			fmt.Fprintf(output, "\t%s %s%s;\n", tn, fn, suf)
@@ -558,22 +447,6 @@ func main() {
 	i := inspector(&structs)
 	ast.Inspect(f, i)
 
-	buf := new(bytes.Buffer)
-	headerTpl.Execute(buf, map[string]string{"Package": f.Name.Name})
-	for _, s := range structs {
-		fmt.Fprintf(buf, "\n/*\n\n")
-		generateDiagram(buf, s)
-		generateXdr(buf, s)
-		fmt.Fprintf(buf, "*/\n")
-		generateCode(buf, s)
-	}
-
-	bs, err := format.Source(buf.Bytes())
-	if err != nil {
-		log.Print(buf.String())
-		log.Fatal(err)
-	}
-
 	var output io.Writer = os.Stdout
 	if *outputFile != "" {
 		fd, err := os.Create(*outputFile)
@@ -582,5 +455,13 @@ func main() {
 		}
 		output = fd
 	}
-	output.Write(bs)
+
+	headerTpl.Execute(output, map[string]string{"Package": f.Name.Name})
+	for _, s := range structs {
+		fmt.Fprintf(output, "\n/*\n\n")
+		generateDiagram(output, s)
+		generateXdr(output, s)
+		fmt.Fprintf(output, "*/\n")
+		generateCode(output, s)
+	}
 }

Godeps/_workspace/src/github.com/calmh/xdr/debug.go

@@ -0,0 +1,16 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+package xdr
+
+import (
+	"log"
+	"os"
+)
+
+var (
+	debug = len(os.Getenv("XDRTRACE")) > 0
+	dl    = log.New(os.Stdout, "xdr: ", log.Lshortfile|log.Ltime|log.Lmicroseconds)
+)
+
+const maxDebugBytes = 32

Godeps/_workspace/src/github.com/calmh/xdr/doc.go

@@ -1,5 +1,5 @@
 // Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
 // is governed by an MIT-style license that can be found in the LICENSE file.
 
-// Package xdr implements an XDR (RFC 4506) marshaller/unmarshaller.
+// Package xdr implements an XDR (RFC 4506) encoder/decoder.
 package xdr

Godeps/_workspace/src/github.com/calmh/xdr/encdec_test.go

@@ -0,0 +1,79 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+package xdr_test
+
+import (
+	"bytes"
+	"math/rand"
+	"reflect"
+	"testing"
+	"testing/quick"
+
+	"github.com/calmh/xdr"
+)
+
+// Contains all supported types
+type TestStruct struct {
+	I    int
+	I8   int8
+	UI8  uint8
+	I16  int16
+	UI16 uint16
+	I32  int32
+	UI32 uint32
+	I64  int64
+	UI64 uint64
+	BS   []byte // max:1024
+	S    string // max:1024
+	C    Opaque
+	SS   []string // max:1024
+}
+
+type Opaque [32]byte
+
+func (u *Opaque) EncodeXDRInto(w *xdr.Writer) (int, error) {
+	return w.WriteRaw(u[:])
+}
+
+func (u *Opaque) DecodeXDRFrom(r *xdr.Reader) (int, error) {
+	return r.ReadRaw(u[:])
+}
+
+func (Opaque) Generate(rand *rand.Rand, size int) reflect.Value {
+	var u Opaque
+	for i := range u[:] {
+		u[i] = byte(rand.Int())
+	}
+	return reflect.ValueOf(u)
+}
+
+func TestEncDec(t *testing.T) {
+	fn := func(t0 TestStruct) bool {
+		bs, err := t0.MarshalXDR()
+		if err != nil {
+			t.Fatal(err)
+		}
+		var t1 TestStruct
+		err = t1.UnmarshalXDR(bs)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Not comparing with DeepEqual since we'll unmarshal nil slices as empty
+		if t0.I != t1.I ||
+			t0.I16 != t1.I16 || t0.UI16 != t1.UI16 ||
+			t0.I32 != t1.I32 || t0.UI32 != t1.UI32 ||
+			t0.I64 != t1.I64 || t0.UI64 != t1.UI64 ||
+			bytes.Compare(t0.BS, t1.BS) != 0 ||
+			t0.S != t1.S || t0.C != t1.C {
+			t.Logf("%#v", t0)
+			t.Logf("%#v", t1)
+			return false
+		}
+		return true
+	}
+	if err := quick.Check(fn, nil); err != nil {
+		t.Error(err)
+	}
+}

Godeps/_workspace/src/github.com/calmh/xdr/encdec_xdr_test.go

@@ -0,0 +1,185 @@
+// ************************************************************
+// This file is automatically generated by genxdr. Do not edit.
+// ************************************************************
+
+package xdr_test
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/calmh/xdr"
+)
+
+/*
+
+TestStruct Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                         int Structure                         \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                        int8 Structure                         \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                        uint8 Structure                        \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|            0x0000             |              I16              |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|            0x0000             |             UI16              |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                              I32                              |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             UI32                              |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                         I64 (64 bits)                         +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                        UI64 (64 bits)                         +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of BS                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     BS (variable length)                      \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                          Length of S                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                      S (variable length)                      \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                       Opaque Structure                        \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Number of SS                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of SS                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     SS (variable length)                      \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct TestStruct {
+	int I;
+	int8 I8;
+	uint8 UI8;
+	int I16;
+	unsigned int UI16;
+	int I32;
+	unsigned int UI32;
+	hyper I64;
+	unsigned hyper UI64;
+	opaque BS<1024>;
+	string S<1024>;
+	Opaque C;
+	string SS<1024>;
+}
+
+*/
+
+func (o TestStruct) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
+}
+
+func (o TestStruct) MarshalXDR() ([]byte, error) {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o TestStruct) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o TestStruct) AppendXDR(bs []byte) ([]byte, error) {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	_, err := o.EncodeXDRInto(xw)
+	return []byte(aw), err
+}
+
+func (o TestStruct) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	xw.WriteUint64(uint64(o.I))
+	xw.WriteUint8(uint8(o.I8))
+	xw.WriteUint8(o.UI8)
+	xw.WriteUint16(uint16(o.I16))
+	xw.WriteUint16(o.UI16)
+	xw.WriteUint32(uint32(o.I32))
+	xw.WriteUint32(o.UI32)
+	xw.WriteUint64(uint64(o.I64))
+	xw.WriteUint64(o.UI64)
+	if l := len(o.BS); l > 1024 {
+		return xw.Tot(), xdr.ElementSizeExceeded("BS", l, 1024)
+	}
+	xw.WriteBytes(o.BS)
+	if l := len(o.S); l > 1024 {
+		return xw.Tot(), xdr.ElementSizeExceeded("S", l, 1024)
+	}
+	xw.WriteString(o.S)
+	_, err := o.C.EncodeXDRInto(xw)
+	if err != nil {
+		return xw.Tot(), err
+	}
+	if l := len(o.SS); l > 1024 {
+		return xw.Tot(), xdr.ElementSizeExceeded("SS", l, 1024)
+	}
+	xw.WriteUint32(uint32(len(o.SS)))
+	for i := range o.SS {
+		xw.WriteString(o.SS[i])
+	}
+	return xw.Tot(), xw.Error()
+}
+
+func (o *TestStruct) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *TestStruct) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.DecodeXDRFrom(xr)
+}
+
+func (o *TestStruct) DecodeXDRFrom(xr *xdr.Reader) error {
+	o.I = int(xr.ReadUint64())
+	o.I8 = int8(xr.ReadUint8())
+	o.UI8 = xr.ReadUint8()
+	o.I16 = int16(xr.ReadUint16())
+	o.UI16 = xr.ReadUint16()
+	o.I32 = int32(xr.ReadUint32())
+	o.UI32 = xr.ReadUint32()
+	o.I64 = int64(xr.ReadUint64())
+	o.UI64 = xr.ReadUint64()
+	o.BS = xr.ReadBytesMax(1024)
+	o.S = xr.ReadStringMax(1024)
+	(&o.C).DecodeXDRFrom(xr)
+	_SSSize := int(xr.ReadUint32())
+	if _SSSize < 0 {
+		return xdr.ElementSizeExceeded("SS", _SSSize, 1024)
+	}
+	if _SSSize > 1024 {
+		return xdr.ElementSizeExceeded("SS", _SSSize, 1024)
+	}
+	o.SS = make([]string, _SSSize)
+	for i := range o.SS {
+		o.SS[i] = xr.ReadString()
+	}
+	return xr.Error()
+}

Godeps/_workspace/src/github.com/calmh/xdr/pad_ipdr.go

@@ -0,0 +1,10 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+// +build ipdr
+
+package xdr
+
+func pad(l int) int {
+	return 0
+}

Godeps/_workspace/src/github.com/calmh/xdr/pad_xdr.go

@@ -0,0 +1,14 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+// +build !ipdr
+
+package xdr
+
+func pad(l int) int {
+	d := l % 4
+	if d == 0 {
+		return 0
+	}
+	return 4 - d
+}

Godeps/_workspace/src/github.com/calmh/xdr/reader.go

@@ -0,0 +1,171 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package xdr
+
+import (
+	"fmt"
+	"io"
+	"reflect"
+	"unsafe"
+)
+
+type Reader struct {
+	r   io.Reader
+	err error
+	b   [8]byte
+}
+
+func NewReader(r io.Reader) *Reader {
+	return &Reader{
+		r: r,
+	}
+}
+
+func (r *Reader) ReadRaw(bs []byte) (int, error) {
+	if r.err != nil {
+		return 0, r.err
+	}
+
+	var n int
+	n, r.err = io.ReadFull(r.r, bs)
+	return n, r.err
+}
+
+func (r *Reader) ReadString() string {
+	return r.ReadStringMax(0)
+}
+
+func (r *Reader) ReadStringMax(max int) string {
+	buf := r.ReadBytesMaxInto(max, nil)
+	bh := (*reflect.SliceHeader)(unsafe.Pointer(&buf))
+	sh := reflect.StringHeader{
+		Data: bh.Data,
+		Len:  bh.Len,
+	}
+	return *((*string)(unsafe.Pointer(&sh)))
+}
+
+func (r *Reader) ReadBytes() []byte {
+	return r.ReadBytesInto(nil)
+}
+
+func (r *Reader) ReadBytesMax(max int) []byte {
+	return r.ReadBytesMaxInto(max, nil)
+}
+
+func (r *Reader) ReadBytesInto(dst []byte) []byte {
+	return r.ReadBytesMaxInto(0, dst)
+}
+
+func (r *Reader) ReadBytesMaxInto(max int, dst []byte) []byte {
+	if r.err != nil {
+		return nil
+	}
+
+	l := int(r.ReadUint32())
+	if r.err != nil {
+		return nil
+	}
+	if l < 0 || max > 0 && l > max {
+		// l may be negative on 32 bit builds
+		r.err = ElementSizeExceeded("bytes field", l, max)
+		return nil
+	}
+
+	if fullLen := l + pad(l); fullLen > len(dst) {
+		dst = make([]byte, fullLen)
+	} else {
+		dst = dst[:fullLen]
+	}
+
+	var n int
+	n, r.err = io.ReadFull(r.r, dst)
+	if r.err != nil {
+		if debug {
+			dl.Printf("rd bytes (%d): %v", len(dst), r.err)
+		}
+		return nil
+	}
+
+	if debug {
+		if n > maxDebugBytes {
+			dl.Printf("rd bytes (%d): %x...", len(dst), dst[:maxDebugBytes])
+		} else {
+			dl.Printf("rd bytes (%d): %x", len(dst), dst)
+		}
+	}
+	return dst[:l]
+}
+
+func (r *Reader) ReadBool() bool {
+	return r.ReadUint8() != 0
+}
+
+func (r *Reader) ReadUint32() uint32 {
+	if r.err != nil {
+		return 0
+	}
+
+	_, r.err = io.ReadFull(r.r, r.b[:4])
+	if r.err != nil {
+		if debug {
+			dl.Printf("rd uint32: %v", r.err)
+		}
+		return 0
+	}
+
+	v := uint32(r.b[3]) | uint32(r.b[2])<<8 | uint32(r.b[1])<<16 | uint32(r.b[0])<<24
+
+	if debug {
+		dl.Printf("rd uint32=%d (0x%08x)", v, v)
+	}
+	return v
+}
+
+func (r *Reader) ReadUint64() uint64 {
+	if r.err != nil {
+		return 0
+	}
+
+	_, r.err = io.ReadFull(r.r, r.b[:8])
+	if r.err != nil {
+		if debug {
+			dl.Printf("rd uint64: %v", r.err)
+		}
+		return 0
+	}
+
+	v := uint64(r.b[7]) | uint64(r.b[6])<<8 | uint64(r.b[5])<<16 | uint64(r.b[4])<<24 |
+		uint64(r.b[3])<<32 | uint64(r.b[2])<<40 | uint64(r.b[1])<<48 | uint64(r.b[0])<<56
+
+	if debug {
+		dl.Printf("rd uint64=%d (0x%016x)", v, v)
+	}
+	return v
+}
+
+type XDRError struct {
+	op  string
+	err error
+}
+
+func (e XDRError) Error() string {
+	return "xdr " + e.op + ": " + e.err.Error()
+}
+
+func (e XDRError) IsEOF() bool {
+	return e.err == io.EOF
+}
+
+func (r *Reader) Error() error {
+	if r.err == nil {
+		return nil
+	}
+	return XDRError{"read", r.err}
+}
+
+func ElementSizeExceeded(field string, size, limit int) error {
+	return fmt.Errorf("%s exceeds size limit; %d > %d", field, size, limit)
+}

Godeps/_workspace/src/github.com/calmh/xdr/reader_ipdr.go

@@ -0,0 +1,49 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// +build ipdr
+
+package xdr
+
+import "io"
+
+func (r *Reader) ReadUint8() uint8 {
+	if r.err != nil {
+		return 0
+	}
+
+	_, r.err = io.ReadFull(r.r, r.b[:1])
+	if r.err != nil {
+		if debug {
+			dl.Printf("rd uint8: %v", r.err)
+		}
+		return 0
+	}
+
+	if debug {
+		dl.Printf("rd uint8=%d (0x%02x)", r.b[0], r.b[0])
+	}
+	return r.b[0]
+}
+
+func (r *Reader) ReadUint16() uint16 {
+	if r.err != nil {
+		return 0
+	}
+
+	_, r.err = io.ReadFull(r.r, r.b[:2])
+	if r.err != nil {
+		if debug {
+			dl.Printf("rd uint16: %v", r.err)
+		}
+		return 0
+	}
+
+	v := uint16(r.b[1]) | uint16(r.b[0])<<8
+
+	if debug {
+		dl.Printf("rd uint16=%d (0x%04x)", v, v)
+	}
+	return v
+}

Godeps/_workspace/src/github.com/calmh/xdr/reader_xdr.go

@@ -0,0 +1,15 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !ipdr
+
+package xdr
+
+func (r *Reader) ReadUint8() uint8 {
+	return uint8(r.ReadUint32())
+}
+
+func (r *Reader) ReadUint16() uint16 {
+	return uint16(r.ReadUint32())
+}

Godeps/_workspace/src/github.com/calmh/xdr/refl_test.go

@@ -0,0 +1,44 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+// +build refl
+
+package xdr_test
+
+import (
+	"bytes"
+	"testing"
+
+	refl "github.com/davecgh/go-xdr/xdr"
+)
+
+func TestCompareMarshals(t *testing.T) {
+	e0 := s.MarshalXDR()
+	e1, err := refl.Marshal(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if bytes.Compare(e0, e1) != 0 {
+		t.Fatalf("Encoding mismatch;\n\t%x (this)\n\t%x (refl)", e0, e1)
+	}
+}
+
+func BenchmarkReflMarshal(b *testing.B) {
+	var err error
+	for i := 0; i < b.N; i++ {
+		res, err = refl.Marshal(s)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkReflUnmarshal(b *testing.B) {
+	var t XDRBenchStruct
+	for i := 0; i < b.N; i++ {
+		_, err := refl.Unmarshal(e, &t)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

Godeps/_workspace/src/github.com/calmh/xdr/writer.go

@@ -0,0 +1,146 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+package xdr
+
+import (
+	"io"
+	"reflect"
+	"unsafe"
+)
+
+var padBytes = []byte{0, 0, 0}
+
+type Writer struct {
+	w   io.Writer
+	tot int
+	err error
+	b   [8]byte
+}
+
+type AppendWriter []byte
+
+func (w *AppendWriter) Write(bs []byte) (int, error) {
+	*w = append(*w, bs...)
+	return len(bs), nil
+}
+
+func NewWriter(w io.Writer) *Writer {
+	return &Writer{
+		w: w,
+	}
+}
+
+func (w *Writer) WriteRaw(bs []byte) (int, error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+
+	var n int
+	n, w.err = w.w.Write(bs)
+	return n, w.err
+}
+
+func (w *Writer) WriteString(s string) (int, error) {
+	sh := *((*reflect.StringHeader)(unsafe.Pointer(&s)))
+	bh := reflect.SliceHeader{
+		Data: sh.Data,
+		Len:  sh.Len,
+		Cap:  sh.Len,
+	}
+	return w.WriteBytes(*(*[]byte)(unsafe.Pointer(&bh)))
+}
+
+func (w *Writer) WriteBytes(bs []byte) (int, error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+
+	w.WriteUint32(uint32(len(bs)))
+	if w.err != nil {
+		return 0, w.err
+	}
+
+	if debug {
+		if len(bs) > maxDebugBytes {
+			dl.Printf("wr bytes (%d): %x...", len(bs), bs[:maxDebugBytes])
+		} else {
+			dl.Printf("wr bytes (%d): %x", len(bs), bs)
+		}
+	}
+
+	var l, n int
+	n, w.err = w.w.Write(bs)
+	l += n
+
+	if p := pad(len(bs)); w.err == nil && p > 0 {
+		n, w.err = w.w.Write(padBytes[:p])
+		l += n
+	}
+
+	w.tot += l
+	return l, w.err
+}
+
+func (w *Writer) WriteBool(v bool) (int, error) {
+	if v {
+		return w.WriteUint8(1)
+	} else {
+		return w.WriteUint8(0)
+	}
+}
+
+func (w *Writer) WriteUint32(v uint32) (int, error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+
+	if debug {
+		dl.Printf("wr uint32=%d", v)
+	}
+
+	w.b[0] = byte(v >> 24)
+	w.b[1] = byte(v >> 16)
+	w.b[2] = byte(v >> 8)
+	w.b[3] = byte(v)
+
+	var l int
+	l, w.err = w.w.Write(w.b[:4])
+	w.tot += l
+	return l, w.err
+}
+
+func (w *Writer) WriteUint64(v uint64) (int, error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+
+	if debug {
+		dl.Printf("wr uint64=%d", v)
+	}
+
+	w.b[0] = byte(v >> 56)
+	w.b[1] = byte(v >> 48)
+	w.b[2] = byte(v >> 40)
+	w.b[3] = byte(v >> 32)
+	w.b[4] = byte(v >> 24)
+	w.b[5] = byte(v >> 16)
+	w.b[6] = byte(v >> 8)
+	w.b[7] = byte(v)
+
+	var l int
+	l, w.err = w.w.Write(w.b[:8])
+	w.tot += l
+	return l, w.err
+}
+
+func (w *Writer) Tot() int {
+	return w.tot
+}
+
+func (w *Writer) Error() error {
+	if w.err == nil {
+		return nil
+	}
+	return XDRError{"write", w.err}
+}

Godeps/_workspace/src/github.com/calmh/xdr/writer_ipdr.go

@@ -0,0 +1,41 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+// +build ipdr
+
+package xdr
+
+func (w *Writer) WriteUint8(v uint8) (int, error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+
+	if debug {
+		dl.Printf("wr uint8=%d", v)
+	}
+
+	w.b[0] = byte(v)
+
+	var l int
+	l, w.err = w.w.Write(w.b[:1])
+	w.tot += l
+	return l, w.err
+}
+
+func (w *Writer) WriteUint16(v uint16) (int, error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+
+	if debug {
+		dl.Printf("wr uint8=%d", v)
+	}
+
+	w.b[0] = byte(v >> 8)
+	w.b[1] = byte(v)
+
+	var l int
+	l, w.err = w.w.Write(w.b[:2])
+	w.tot += l
+	return l, w.err
+}

Godeps/_workspace/src/github.com/calmh/xdr/writer_xdr.go

@@ -0,0 +1,14 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+// +build !ipdr
+
+package xdr
+
+func (w *Writer) WriteUint8(v uint8) (int, error) {
+	return w.WriteUint32(uint32(v))
+}
+
+func (w *Writer) WriteUint16(v uint16) (int, error) {
+	return w.WriteUint32(uint32(v))
+}

Godeps/_workspace/src/github.com/calmh/xdr/xdr_test.go

@@ -0,0 +1,93 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+package xdr
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+	"testing/quick"
+)
+
+func TestBytesNil(t *testing.T) {
+	fn := func(bs []byte) bool {
+		var b = new(bytes.Buffer)
+		var w = NewWriter(b)
+		var r = NewReader(b)
+		w.WriteBytes(bs)
+		w.WriteBytes(bs)
+		r.ReadBytes()
+		res := r.ReadBytes()
+		return bytes.Compare(bs, res) == 0
+	}
+	if err := quick.Check(fn, nil); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestBytesGiven(t *testing.T) {
+	fn := func(bs []byte) bool {
+		var b = new(bytes.Buffer)
+		var w = NewWriter(b)
+		var r = NewReader(b)
+		w.WriteBytes(bs)
+		w.WriteBytes(bs)
+		res := make([]byte, 12)
+		res = r.ReadBytesInto(res)
+		res = r.ReadBytesInto(res)
+		return bytes.Compare(bs, res) == 0
+	}
+	if err := quick.Check(fn, nil); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestReadBytesMaxInto(t *testing.T) {
+	var max = 64
+	for tot := 32; tot < 128; tot++ {
+		for diff := -32; diff <= 32; diff++ {
+			var b = new(bytes.Buffer)
+			var r = NewReader(b)
+			var w = NewWriter(b)
+
+			var toWrite = make([]byte, tot)
+			w.WriteBytes(toWrite)
+
+			var buf = make([]byte, tot+diff)
+			var bs = r.ReadBytesMaxInto(max, buf)
+
+			if tot <= max {
+				if read := len(bs); read != tot {
+					t.Errorf("Incorrect read bytes, wrote=%d, buf=%d, max=%d, read=%d", tot, tot+diff, max, read)
+				}
+			} else if !strings.Contains(r.err.Error(), "exceeds size") {
+				t.Errorf("Unexpected non-ErrElementSizeExceeded error for wrote=%d, max=%d: %v", tot, max, r.err)
+			}
+		}
+	}
+}
+
+func TestReadStringMax(t *testing.T) {
+	for tot := 42; tot < 72; tot++ {
+		for max := 0; max < 128; max++ {
+			var b = new(bytes.Buffer)
+			var r = NewReader(b)
+			var w = NewWriter(b)
+
+			var toWrite = make([]byte, tot)
+			w.WriteBytes(toWrite)
+
+			var str = r.ReadStringMax(max)
+			var read = len(str)
+
+			if max == 0 || tot <= max {
+				if read != tot {
+					t.Errorf("Incorrect read bytes, wrote=%d, max=%d, read=%d", tot, max, read)
+				}
+			} else if !strings.Contains(r.err.Error(), "exceeds size") {
+				t.Errorf("Unexpected non-ErrElementSizeExceeded error for wrote=%d, max=%d, read=%d: %v", tot, max, read, r.err)
+			}
+		}
+	}
+}

Godeps/_workspace/src/github.com/golang/snappy/AUTHORS

@@ -11,5 +11,4 @@
 Damian Gryski <dgryski@gmail.com>
 Google Inc.
 Jan Mercl <0xjnml@gmail.com>
-Rodolfo Carvalho <rhcarvalho@gmail.com>
 Sebastien Binet <seb.binet@gmail.com>

Godeps/_workspace/src/github.com/golang/snappy/CONTRIBUTORS

@@ -32,6 +32,5 @@ Kai Backman <kaib@golang.org>
 Marc-Antoine Ruel <maruel@chromium.org>
 Nigel Tao <nigeltao@golang.org>
 Rob Pike <r@golang.org>
-Rodolfo Carvalho <rhcarvalho@gmail.com>
 Russ Cox <rsc@golang.org>
 Sebastien Binet <seb.binet@gmail.com>

Godeps/_workspace/src/github.com/golang/snappy/decode.go

@@ -17,9 +17,6 @@ var (
 	ErrTooLarge = errors.New("snappy: decoded block is too large")
 	// ErrUnsupported reports that the input isn't supported.
 	ErrUnsupported = errors.New("snappy: unsupported input")
-
-	errUnsupportedCopy4Tag      = errors.New("snappy: unsupported COPY_4 tag")
-	errUnsupportedLiteralLength = errors.New("snappy: unsupported literal length")
 )
 
 // DecodedLen returns the length of the decoded block.
@@ -43,36 +40,96 @@ func decodedLen(src []byte) (blockLen, headerLen int, err error) {
 	return int(v), n, nil
 }
 
-const (
-	decodeErrCodeCorrupt                  = 1
-	decodeErrCodeUnsupportedLiteralLength = 2
-	decodeErrCodeUnsupportedCopy4Tag      = 3
-)
-
 // Decode returns the decoded form of src. The returned slice may be a sub-
 // slice of dst if dst was large enough to hold the entire decoded block.
 // Otherwise, a newly allocated slice will be returned.
-//
-// The dst and src must not overlap. It is valid to pass a nil dst.
+// It is valid to pass a nil dst.
 func Decode(dst, src []byte) ([]byte, error) {
 	dLen, s, err := decodedLen(src)
 	if err != nil {
 		return nil, err
 	}
-	if dLen <= len(dst) {
-		dst = dst[:dLen]
-	} else {
+	if len(dst) < dLen {
 		dst = make([]byte, dLen)
 	}
-	switch decode(dst, src[s:]) {
-	case 0:
-		return dst, nil
-	case decodeErrCodeUnsupportedLiteralLength:
-		return nil, errUnsupportedLiteralLength
-	case decodeErrCodeUnsupportedCopy4Tag:
-		return nil, errUnsupportedCopy4Tag
+
+	var d, offset, length int
+	for s < len(src) {
+		switch src[s] & 0x03 {
+		case tagLiteral:
+			x := uint(src[s] >> 2)
+			switch {
+			case x < 60:
+				s++
+			case x == 60:
+				s += 2
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-1])
+			case x == 61:
+				s += 3
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-2]) | uint(src[s-1])<<8
+			case x == 62:
+				s += 4
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-3]) | uint(src[s-2])<<8 | uint(src[s-1])<<16
+			case x == 63:
+				s += 5
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-4]) | uint(src[s-3])<<8 | uint(src[s-2])<<16 | uint(src[s-1])<<24
+			}
+			length = int(x + 1)
+			if length <= 0 {
+				return nil, errors.New("snappy: unsupported literal length")
+			}
+			if length > len(dst)-d || length > len(src)-s {
+				return nil, ErrCorrupt
+			}
+			copy(dst[d:], src[s:s+length])
+			d += length
+			s += length
+			continue
+
+		case tagCopy1:
+			s += 2
+			if s > len(src) {
+				return nil, ErrCorrupt
+			}
+			length = 4 + int(src[s-2])>>2&0x7
+			offset = int(src[s-2])&0xe0<<3 | int(src[s-1])
+
+		case tagCopy2:
+			s += 3
+			if s > len(src) {
+				return nil, ErrCorrupt
+			}
+			length = 1 + int(src[s-3])>>2
+			offset = int(src[s-2]) | int(src[s-1])<<8
+
+		case tagCopy4:
+			return nil, errors.New("snappy: unsupported COPY_4 tag")
+		}
+
+		end := d + length
+		if offset > d || end > len(dst) {
+			return nil, ErrCorrupt
+		}
+		for ; d < end; d++ {
+			dst[d] = dst[d-offset]
+		}
 	}
-	return nil, ErrCorrupt
+	if d != dLen {
+		return nil, ErrCorrupt
+	}
+	return dst[:d], nil
 }
 
 // NewReader returns a new Reader that decompresses from r, using the framing
@@ -81,12 +138,12 @@ func Decode(dst, src []byte) ([]byte, error) {
 func NewReader(r io.Reader) *Reader {
 	return &Reader{
 		r:       r,
-		decoded: make([]byte, maxBlockSize),
-		buf:     make([]byte, maxEncodedLenOfMaxBlockSize+checksumSize),
+		decoded: make([]byte, maxUncompressedChunkLen),
+		buf:     make([]byte, MaxEncodedLen(maxUncompressedChunkLen)+checksumSize),
 	}
 }
 
-// Reader is an io.Reader that can read Snappy-compressed bytes.
+// Reader is an io.Reader than can read Snappy-compressed bytes.
 type Reader struct {
 	r       io.Reader
 	err     error

Godeps/_workspace/src/github.com/golang/snappy/encode.go

@@ -0,0 +1,254 @@
+// Copyright 2011 The Snappy-Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package snappy
+
+import (
+	"encoding/binary"
+	"io"
+)
+
+// We limit how far copy back-references can go, the same as the C++ code.
+const maxOffset = 1 << 15
+
+// emitLiteral writes a literal chunk and returns the number of bytes written.
+func emitLiteral(dst, lit []byte) int {
+	i, n := 0, uint(len(lit)-1)
+	switch {
+	case n < 60:
+		dst[0] = uint8(n)<<2 | tagLiteral
+		i = 1
+	case n < 1<<8:
+		dst[0] = 60<<2 | tagLiteral
+		dst[1] = uint8(n)
+		i = 2
+	case n < 1<<16:
+		dst[0] = 61<<2 | tagLiteral
+		dst[1] = uint8(n)
+		dst[2] = uint8(n >> 8)
+		i = 3
+	case n < 1<<24:
+		dst[0] = 62<<2 | tagLiteral
+		dst[1] = uint8(n)
+		dst[2] = uint8(n >> 8)
+		dst[3] = uint8(n >> 16)
+		i = 4
+	case int64(n) < 1<<32:
+		dst[0] = 63<<2 | tagLiteral
+		dst[1] = uint8(n)
+		dst[2] = uint8(n >> 8)
+		dst[3] = uint8(n >> 16)
+		dst[4] = uint8(n >> 24)
+		i = 5
+	default:
+		panic("snappy: source buffer is too long")
+	}
+	if copy(dst[i:], lit) != len(lit) {
+		panic("snappy: destination buffer is too short")
+	}
+	return i + len(lit)
+}
+
+// emitCopy writes a copy chunk and returns the number of bytes written.
+func emitCopy(dst []byte, offset, length int) int {
+	i := 0
+	for length > 0 {
+		x := length - 4
+		if 0 <= x && x < 1<<3 && offset < 1<<11 {
+			dst[i+0] = uint8(offset>>8)&0x07<<5 | uint8(x)<<2 | tagCopy1
+			dst[i+1] = uint8(offset)
+			i += 2
+			break
+		}
+
+		x = length
+		if x > 1<<6 {
+			x = 1 << 6
+		}
+		dst[i+0] = uint8(x-1)<<2 | tagCopy2
+		dst[i+1] = uint8(offset)
+		dst[i+2] = uint8(offset >> 8)
+		i += 3
+		length -= x
+	}
+	return i
+}
+
+// Encode returns the encoded form of src. The returned slice may be a sub-
+// slice of dst if dst was large enough to hold the entire encoded block.
+// Otherwise, a newly allocated slice will be returned.
+// It is valid to pass a nil dst.
+func Encode(dst, src []byte) []byte {
+	if n := MaxEncodedLen(len(src)); len(dst) < n {
+		dst = make([]byte, n)
+	}
+
+	// The block starts with the varint-encoded length of the decompressed bytes.
+	d := binary.PutUvarint(dst, uint64(len(src)))
+
+	// Return early if src is short.
+	if len(src) <= 4 {
+		if len(src) != 0 {
+			d += emitLiteral(dst[d:], src)
+		}
+		return dst[:d]
+	}
+
+	// Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive.
+	const maxTableSize = 1 << 14
+	shift, tableSize := uint(32-8), 1<<8
+	for tableSize < maxTableSize && tableSize < len(src) {
+		shift--
+		tableSize *= 2
+	}
+	var table [maxTableSize]int
+
+	// Iterate over the source bytes.
+	var (
+		s   int // The iterator position.
+		t   int // The last position with the same hash as s.
+		lit int // The start position of any pending literal bytes.
+	)
+	for s+3 < len(src) {
+		// Update the hash table.
+		b0, b1, b2, b3 := src[s], src[s+1], src[s+2], src[s+3]
+		h := uint32(b0) | uint32(b1)<<8 | uint32(b2)<<16 | uint32(b3)<<24
+		p := &table[(h*0x1e35a7bd)>>shift]
+		// We need to to store values in [-1, inf) in table. To save
+		// some initialization time, (re)use the table's zero value
+		// and shift the values against this zero: add 1 on writes,
+		// subtract 1 on reads.
+		t, *p = *p-1, s+1
+		// If t is invalid or src[s:s+4] differs from src[t:t+4], accumulate a literal byte.
+		if t < 0 || s-t >= maxOffset || b0 != src[t] || b1 != src[t+1] || b2 != src[t+2] || b3 != src[t+3] {
+			s++
+			continue
+		}
+		// Otherwise, we have a match. First, emit any pending literal bytes.
+		if lit != s {
+			d += emitLiteral(dst[d:], src[lit:s])
+		}
+		// Extend the match to be as long as possible.
+		s0 := s
+		s, t = s+4, t+4
+		for s < len(src) && src[s] == src[t] {
+			s++
+			t++
+		}
+		// Emit the copied bytes.
+		d += emitCopy(dst[d:], s-t, s-s0)
+		lit = s
+	}
+
+	// Emit any final pending literal bytes and return.
+	if lit != len(src) {
+		d += emitLiteral(dst[d:], src[lit:])
+	}
+	return dst[:d]
+}
+
+// MaxEncodedLen returns the maximum length of a snappy block, given its
+// uncompressed length.
+func MaxEncodedLen(srcLen int) int {
+	// Compressed data can be defined as:
+	//    compressed := item* literal*
+	//    item       := literal* copy
+	//
+	// The trailing literal sequence has a space blowup of at most 62/60
+	// since a literal of length 60 needs one tag byte + one extra byte
+	// for length information.
+	//
+	// Item blowup is trickier to measure. Suppose the "copy" op copies
+	// 4 bytes of data. Because of a special check in the encoding code,
+	// we produce a 4-byte copy only if the offset is < 65536. Therefore
+	// the copy op takes 3 bytes to encode, and this type of item leads
+	// to at most the 62/60 blowup for representing literals.
+	//
+	// Suppose the "copy" op copies 5 bytes of data. If the offset is big
+	// enough, it will take 5 bytes to encode the copy op. Therefore the
+	// worst case here is a one-byte literal followed by a five-byte copy.
+	// That is, 6 bytes of input turn into 7 bytes of "compressed" data.
+	//
+	// This last factor dominates the blowup, so the final estimate is:
+	return 32 + srcLen + srcLen/6
+}
+
+// NewWriter returns a new Writer that compresses to w, using the framing
+// format described at
+// https://github.com/google/snappy/blob/master/framing_format.txt
+func NewWriter(w io.Writer) *Writer {
+	return &Writer{
+		w:   w,
+		enc: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)),
+	}
+}
+
+// Writer is an io.Writer than can write Snappy-compressed bytes.
+type Writer struct {
+	w           io.Writer
+	err         error
+	enc         []byte
+	buf         [checksumSize + chunkHeaderSize]byte
+	wroteHeader bool
+}
+
+// Reset discards the writer's state and switches the Snappy writer to write to
+// w. This permits reusing a Writer rather than allocating a new one.
+func (w *Writer) Reset(writer io.Writer) {
+	w.w = writer
+	w.err = nil
+	w.wroteHeader = false
+}
+
+// Write satisfies the io.Writer interface.
+func (w *Writer) Write(p []byte) (n int, errRet error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+	if !w.wroteHeader {
+		copy(w.enc, magicChunk)
+		if _, err := w.w.Write(w.enc[:len(magicChunk)]); err != nil {
+			w.err = err
+			return n, err
+		}
+		w.wroteHeader = true
+	}
+	for len(p) > 0 {
+		var uncompressed []byte
+		if len(p) > maxUncompressedChunkLen {
+			uncompressed, p = p[:maxUncompressedChunkLen], p[maxUncompressedChunkLen:]
+		} else {
+			uncompressed, p = p, nil
+		}
+		checksum := crc(uncompressed)
+
+		// Compress the buffer, discarding the result if the improvement
+		// isn't at least 12.5%.
+		chunkType := uint8(chunkTypeCompressedData)
+		chunkBody := Encode(w.enc, uncompressed)
+		if len(chunkBody) >= len(uncompressed)-len(uncompressed)/8 {
+			chunkType, chunkBody = chunkTypeUncompressedData, uncompressed
+		}
+
+		chunkLen := 4 + len(chunkBody)
+		w.buf[0] = chunkType
+		w.buf[1] = uint8(chunkLen >> 0)
+		w.buf[2] = uint8(chunkLen >> 8)
+		w.buf[3] = uint8(chunkLen >> 16)
+		w.buf[4] = uint8(checksum >> 0)
+		w.buf[5] = uint8(checksum >> 8)
+		w.buf[6] = uint8(checksum >> 16)
+		w.buf[7] = uint8(checksum >> 24)
+		if _, err := w.w.Write(w.buf[:]); err != nil {
+			w.err = err
+			return n, err
+		}
+		if _, err := w.w.Write(chunkBody); err != nil {
+			w.err = err
+			return n, err
+		}
+		n += len(uncompressed)
+	}
+	return n, nil
+}

Godeps/_workspace/src/github.com/golang/snappy/snappy.go

@@ -6,7 +6,7 @@
 // It aims for very high speeds and reasonable compression.
 //
 // The C++ snappy implementation is at https://github.com/google/snappy
-package snappy // import "github.com/golang/snappy"
+package snappy
 
 import (
 	"hash/crc32"
@@ -46,25 +46,9 @@ const (
 	chunkHeaderSize = 4
 	magicChunk      = "\xff\x06\x00\x00" + magicBody
 	magicBody       = "sNaPpY"
-
-	// maxBlockSize is the maximum size of the input to encodeBlock. It is not
-	// part of the wire format per se, but some parts of the encoder assume
-	// that an offset fits into a uint16.
-	//
-	// Also, for the framing format (Writer type instead of Encode function),
 	// https://github.com/google/snappy/blob/master/framing_format.txt says
-	// that "the uncompressed data in a chunk must be no longer than 65536
-	// bytes".
-	maxBlockSize = 65536
-
-	// maxEncodedLenOfMaxBlockSize equals MaxEncodedLen(maxBlockSize), but is
-	// hard coded to be a const instead of a variable, so that obufLen can also
-	// be a const. Their equivalence is confirmed by
-	// TestMaxEncodedLenOfMaxBlockSize.
-	maxEncodedLenOfMaxBlockSize = 76490
-
-	obufHeaderLen = len(magicChunk) + checksumSize + chunkHeaderSize
-	obufLen       = obufHeaderLen + maxEncodedLenOfMaxBlockSize
+	// that "the uncompressed data in a chunk must be no longer than 65536 bytes".
+	maxUncompressedChunkLen = 65536
 )
 
 const (

Godeps/_workspace/src/github.com/golang/snappy/snappy_test.go

@@ -0,0 +1,377 @@
+// Copyright 2011 The Snappy-Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package snappy
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+var (
+	download = flag.Bool("download", false, "If true, download any missing files before running benchmarks")
+	testdata = flag.String("testdata", "testdata", "Directory containing the test data")
+)
+
+func roundtrip(b, ebuf, dbuf []byte) error {
+	d, err := Decode(dbuf, Encode(ebuf, b))
+	if err != nil {
+		return fmt.Errorf("decoding error: %v", err)
+	}
+	if !bytes.Equal(b, d) {
+		return fmt.Errorf("roundtrip mismatch:\n\twant %v\n\tgot  %v", b, d)
+	}
+	return nil
+}
+
+func TestEmpty(t *testing.T) {
+	if err := roundtrip(nil, nil, nil); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestSmallCopy(t *testing.T) {
+	for _, ebuf := range [][]byte{nil, make([]byte, 20), make([]byte, 64)} {
+		for _, dbuf := range [][]byte{nil, make([]byte, 20), make([]byte, 64)} {
+			for i := 0; i < 32; i++ {
+				s := "aaaa" + strings.Repeat("b", i) + "aaaabbbb"
+				if err := roundtrip([]byte(s), ebuf, dbuf); err != nil {
+					t.Errorf("len(ebuf)=%d, len(dbuf)=%d, i=%d: %v", len(ebuf), len(dbuf), i, err)
+				}
+			}
+		}
+	}
+}
+
+func TestSmallRand(t *testing.T) {
+	rng := rand.New(rand.NewSource(27354294))
+	for n := 1; n < 20000; n += 23 {
+		b := make([]byte, n)
+		for i := range b {
+			b[i] = uint8(rng.Uint32())
+		}
+		if err := roundtrip(b, nil, nil); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestSmallRegular(t *testing.T) {
+	for n := 1; n < 20000; n += 23 {
+		b := make([]byte, n)
+		for i := range b {
+			b[i] = uint8(i%10 + 'a')
+		}
+		if err := roundtrip(b, nil, nil); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+func TestInvalidVarint(t *testing.T) {
+	data := []byte("\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x00")
+	if _, err := DecodedLen(data); err != ErrCorrupt {
+		t.Errorf("DecodedLen: got %v, want ErrCorrupt", err)
+	}
+	if _, err := Decode(nil, data); err != ErrCorrupt {
+		t.Errorf("Decode: got %v, want ErrCorrupt", err)
+	}
+
+	// The encoded varint overflows 32 bits
+	data = []byte("\xff\xff\xff\xff\xff\x00")
+
+	if _, err := DecodedLen(data); err != ErrCorrupt {
+		t.Errorf("DecodedLen: got %v, want ErrCorrupt", err)
+	}
+	if _, err := Decode(nil, data); err != ErrCorrupt {
+		t.Errorf("Decode: got %v, want ErrCorrupt", err)
+	}
+}
+
+func cmp(a, b []byte) error {
+	if len(a) != len(b) {
+		return fmt.Errorf("got %d bytes, want %d", len(a), len(b))
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return fmt.Errorf("byte #%d: got 0x%02x, want 0x%02x", i, a[i], b[i])
+		}
+	}
+	return nil
+}
+
+func TestFramingFormat(t *testing.T) {
+	// src is comprised of alternating 1e5-sized sequences of random
+	// (incompressible) bytes and repeated (compressible) bytes. 1e5 was chosen
+	// because it is larger than maxUncompressedChunkLen (64k).
+	src := make([]byte, 1e6)
+	rng := rand.New(rand.NewSource(1))
+	for i := 0; i < 10; i++ {
+		if i%2 == 0 {
+			for j := 0; j < 1e5; j++ {
+				src[1e5*i+j] = uint8(rng.Intn(256))
+			}
+		} else {
+			for j := 0; j < 1e5; j++ {
+				src[1e5*i+j] = uint8(i)
+			}
+		}
+	}
+
+	buf := new(bytes.Buffer)
+	if _, err := NewWriter(buf).Write(src); err != nil {
+		t.Fatalf("Write: encoding: %v", err)
+	}
+	dst, err := ioutil.ReadAll(NewReader(buf))
+	if err != nil {
+		t.Fatalf("ReadAll: decoding: %v", err)
+	}
+	if err := cmp(dst, src); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestReaderReset(t *testing.T) {
+	gold := bytes.Repeat([]byte("All that is gold does not glitter,\n"), 10000)
+	buf := new(bytes.Buffer)
+	if _, err := NewWriter(buf).Write(gold); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+	encoded, invalid, partial := buf.String(), "invalid", "partial"
+	r := NewReader(nil)
+	for i, s := range []string{encoded, invalid, partial, encoded, partial, invalid, encoded, encoded} {
+		if s == partial {
+			r.Reset(strings.NewReader(encoded))
+			if _, err := r.Read(make([]byte, 101)); err != nil {
+				t.Errorf("#%d: %v", i, err)
+				continue
+			}
+			continue
+		}
+		r.Reset(strings.NewReader(s))
+		got, err := ioutil.ReadAll(r)
+		switch s {
+		case encoded:
+			if err != nil {
+				t.Errorf("#%d: %v", i, err)
+				continue
+			}
+			if err := cmp(got, gold); err != nil {
+				t.Errorf("#%d: %v", i, err)
+				continue
+			}
+		case invalid:
+			if err == nil {
+				t.Errorf("#%d: got nil error, want non-nil", i)
+				continue
+			}
+		}
+	}
+}
+
+func TestWriterReset(t *testing.T) {
+	gold := bytes.Repeat([]byte("Not all those who wander are lost;\n"), 10000)
+	var gots, wants [][]byte
+	const n = 20
+	w, failed := NewWriter(nil), false
+	for i := 0; i <= n; i++ {
+		buf := new(bytes.Buffer)
+		w.Reset(buf)
+		want := gold[:len(gold)*i/n]
+		if _, err := w.Write(want); err != nil {
+			t.Errorf("#%d: Write: %v", i, err)
+			failed = true
+			continue
+		}
+		got, err := ioutil.ReadAll(NewReader(buf))
+		if err != nil {
+			t.Errorf("#%d: ReadAll: %v", i, err)
+			failed = true
+			continue
+		}
+		gots = append(gots, got)
+		wants = append(wants, want)
+	}
+	if failed {
+		return
+	}
+	for i := range gots {
+		if err := cmp(gots[i], wants[i]); err != nil {
+			t.Errorf("#%d: %v", i, err)
+		}
+	}
+}
+
+func benchDecode(b *testing.B, src []byte) {
+	encoded := Encode(nil, src)
+	// Bandwidth is in amount of uncompressed data.
+	b.SetBytes(int64(len(src)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		Decode(src, encoded)
+	}
+}
+
+func benchEncode(b *testing.B, src []byte) {
+	// Bandwidth is in amount of uncompressed data.
+	b.SetBytes(int64(len(src)))
+	dst := make([]byte, MaxEncodedLen(len(src)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		Encode(dst, src)
+	}
+}
+
+func readFile(b testing.TB, filename string) []byte {
+	src, err := ioutil.ReadFile(filename)
+	if err != nil {
+		b.Skipf("skipping benchmark: %v", err)
+	}
+	if len(src) == 0 {
+		b.Fatalf("%s has zero length", filename)
+	}
+	return src
+}
+
+// expand returns a slice of length n containing repeated copies of src.
+func expand(src []byte, n int) []byte {
+	dst := make([]byte, n)
+	for x := dst; len(x) > 0; {
+		i := copy(x, src)
+		x = x[i:]
+	}
+	return dst
+}
+
+func benchWords(b *testing.B, n int, decode bool) {
+	// Note: the file is OS-language dependent so the resulting values are not
+	// directly comparable for non-US-English OS installations.
+	data := expand(readFile(b, "/usr/share/dict/words"), n)
+	if decode {
+		benchDecode(b, data)
+	} else {
+		benchEncode(b, data)
+	}
+}
+
+func BenchmarkWordsDecode1e3(b *testing.B) { benchWords(b, 1e3, true) }
+func BenchmarkWordsDecode1e4(b *testing.B) { benchWords(b, 1e4, true) }
+func BenchmarkWordsDecode1e5(b *testing.B) { benchWords(b, 1e5, true) }
+func BenchmarkWordsDecode1e6(b *testing.B) { benchWords(b, 1e6, true) }
+func BenchmarkWordsEncode1e3(b *testing.B) { benchWords(b, 1e3, false) }
+func BenchmarkWordsEncode1e4(b *testing.B) { benchWords(b, 1e4, false) }
+func BenchmarkWordsEncode1e5(b *testing.B) { benchWords(b, 1e5, false) }
+func BenchmarkWordsEncode1e6(b *testing.B) { benchWords(b, 1e6, false) }
+
+// testFiles' values are copied directly from
+// https://raw.githubusercontent.com/google/snappy/master/snappy_unittest.cc
+// The label field is unused in snappy-go.
+var testFiles = []struct {
+	label    string
+	filename string
+}{
+	{"html", "html"},
+	{"urls", "urls.10K"},
+	{"jpg", "fireworks.jpeg"},
+	{"jpg_200", "fireworks.jpeg"},
+	{"pdf", "paper-100k.pdf"},
+	{"html4", "html_x_4"},
+	{"txt1", "alice29.txt"},
+	{"txt2", "asyoulik.txt"},
+	{"txt3", "lcet10.txt"},
+	{"txt4", "plrabn12.txt"},
+	{"pb", "geo.protodata"},
+	{"gaviota", "kppkn.gtb"},
+}
+
+// The test data files are present at this canonical URL.
+const baseURL = "https://raw.githubusercontent.com/google/snappy/master/testdata/"
+
+func downloadTestdata(b *testing.B, basename string) (errRet error) {
+	filename := filepath.Join(*testdata, basename)
+	if stat, err := os.Stat(filename); err == nil && stat.Size() != 0 {
+		return nil
+	}
+
+	if !*download {
+		b.Skipf("test data not found; skipping benchmark without the -download flag")
+	}
+	// Download the official snappy C++ implementation reference test data
+	// files for benchmarking.
+	if err := os.Mkdir(*testdata, 0777); err != nil && !os.IsExist(err) {
+		return fmt.Errorf("failed to create testdata: %s", err)
+	}
+
+	f, err := os.Create(filename)
+	if err != nil {
+		return fmt.Errorf("failed to create %s: %s", filename, err)
+	}
+	defer f.Close()
+	defer func() {
+		if errRet != nil {
+			os.Remove(filename)
+		}
+	}()
+	url := baseURL + basename
+	resp, err := http.Get(url)
+	if err != nil {
+		return fmt.Errorf("failed to download %s: %s", url, err)
+	}
+	defer resp.Body.Close()
+	if s := resp.StatusCode; s != http.StatusOK {
+		return fmt.Errorf("downloading %s: HTTP status code %d (%s)", url, s, http.StatusText(s))
+	}
+	_, err = io.Copy(f, resp.Body)
+	if err != nil {
+		return fmt.Errorf("failed to download %s to %s: %s", url, filename, err)
+	}
+	return nil
+}
+
+func benchFile(b *testing.B, n int, decode bool) {
+	if err := downloadTestdata(b, testFiles[n].filename); err != nil {
+		b.Fatalf("failed to download testdata: %s", err)
+	}
+	data := readFile(b, filepath.Join(*testdata, testFiles[n].filename))
+	if decode {
+		benchDecode(b, data)
+	} else {
+		benchEncode(b, data)
+	}
+}
+
+// Naming convention is kept similar to what snappy's C++ implementation uses.
+func Benchmark_UFlat0(b *testing.B)  { benchFile(b, 0, true) }
+func Benchmark_UFlat1(b *testing.B)  { benchFile(b, 1, true) }
+func Benchmark_UFlat2(b *testing.B)  { benchFile(b, 2, true) }
+func Benchmark_UFlat3(b *testing.B)  { benchFile(b, 3, true) }
+func Benchmark_UFlat4(b *testing.B)  { benchFile(b, 4, true) }
+func Benchmark_UFlat5(b *testing.B)  { benchFile(b, 5, true) }
+func Benchmark_UFlat6(b *testing.B)  { benchFile(b, 6, true) }
+func Benchmark_UFlat7(b *testing.B)  { benchFile(b, 7, true) }
+func Benchmark_UFlat8(b *testing.B)  { benchFile(b, 8, true) }
+func Benchmark_UFlat9(b *testing.B)  { benchFile(b, 9, true) }
+func Benchmark_UFlat10(b *testing.B) { benchFile(b, 10, true) }
+func Benchmark_UFlat11(b *testing.B) { benchFile(b, 11, true) }
+func Benchmark_ZFlat0(b *testing.B)  { benchFile(b, 0, false) }
+func Benchmark_ZFlat1(b *testing.B)  { benchFile(b, 1, false) }
+func Benchmark_ZFlat2(b *testing.B)  { benchFile(b, 2, false) }
+func Benchmark_ZFlat3(b *testing.B)  { benchFile(b, 3, false) }
+func Benchmark_ZFlat4(b *testing.B)  { benchFile(b, 4, false) }
+func Benchmark_ZFlat5(b *testing.B)  { benchFile(b, 5, false) }
+func Benchmark_ZFlat6(b *testing.B)  { benchFile(b, 6, false) }
+func Benchmark_ZFlat7(b *testing.B)  { benchFile(b, 7, false) }
+func Benchmark_ZFlat8(b *testing.B)  { benchFile(b, 8, false) }
+func Benchmark_ZFlat9(b *testing.B)  { benchFile(b, 9, false) }
+func Benchmark_ZFlat10(b *testing.B) { benchFile(b, 10, false) }
+func Benchmark_ZFlat11(b *testing.B) { benchFile(b, 11, false) }

Godeps/_workspace/src/github.com/juju/ratelimit/LICENSE

@@ -0,0 +1,191 @@
+All files in this repository are licensed as follows. If you contribute
+to this repository, it is assumed that you license your contribution
+under the same license unless you state otherwise.
+
+All files Copyright (C) 2015 Canonical Ltd. unless otherwise specified in the file.
+
+This software is licensed under the LGPLv3, included below.
+
+As a special exception to the GNU Lesser General Public License version 3
+("LGPL3"), the copyright holders of this Library give you permission to
+convey to a third party a Combined Work that links statically or dynamically
+to this Library without providing any Minimal Corresponding Source or
+Minimal Application Code as set out in 4d or providing the installation
+information set out in section 4e, provided that you comply with the other
+provisions of LGPL3 and provided that you meet, for the Application the
+terms and conditions of the license(s) which apply to the Application.
+
+Except as stated in this special exception, the provisions of LGPL3 will
+continue to comply in full to this Library. If you modify this Library, you
+may apply this exception to your version of this Library, but you are not
+obliged to do so. If you do not wish to do so, delete this exception
+statement from your version. This exception does not (and cannot) modify any
+license terms which apply to the Application, with which you must still
+comply.
+
+
+                   GNU LESSER GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+
+  This version of the GNU Lesser General Public License incorporates
+the terms and conditions of version 3 of the GNU General Public
+License, supplemented by the additional permissions listed below.
+
+  0. Additional Definitions.
+
+  As used herein, "this License" refers to version 3 of the GNU Lesser
+General Public License, and the "GNU GPL" refers to version 3 of the GNU
+General Public License.
+
+  "The Library" refers to a covered work governed by this License,
+other than an Application or a Combined Work as defined below.
+
+  An "Application" is any work that makes use of an interface provided
+by the Library, but which is not otherwise based on the Library.
+Defining a subclass of a class defined by the Library is deemed a mode
+of using an interface provided by the Library.
+
+  A "Combined Work" is a work produced by combining or linking an
+Application with the Library.  The particular version of the Library
+with which the Combined Work was made is also called the "Linked
+Version".
+
+  The "Minimal Corresponding Source" for a Combined Work means the
+Corresponding Source for the Combined Work, excluding any source code
+for portions of the Combined Work that, considered in isolation, are
+based on the Application, and not on the Linked Version.
+
+  The "Corresponding Application Code" for a Combined Work means the
+object code and/or source code for the Application, including any data
+and utility programs needed for reproducing the Combined Work from the
+Application, but excluding the System Libraries of the Combined Work.
+
+  1. Exception to Section 3 of the GNU GPL.
+
+  You may convey a covered work under sections 3 and 4 of this License
+without being bound by section 3 of the GNU GPL.
+
+  2. Conveying Modified Versions.
+
+  If you modify a copy of the Library, and, in your modifications, a
+facility refers to a function or data to be supplied by an Application
+that uses the facility (other than as an argument passed when the
+facility is invoked), then you may convey a copy of the modified
+version:
+
+   a) under this License, provided that you make a good faith effort to
+   ensure that, in the event an Application does not supply the
+   function or data, the facility still operates, and performs
+   whatever part of its purpose remains meaningful, or
+
+   b) under the GNU GPL, with none of the additional permissions of
+   this License applicable to that copy.
+
+  3. Object Code Incorporating Material from Library Header Files.
+
+  The object code form of an Application may incorporate material from
+a header file that is part of the Library.  You may convey such object
+code under terms of your choice, provided that, if the incorporated
+material is not limited to numerical parameters, data structure
+layouts and accessors, or small macros, inline functions and templates
+(ten or fewer lines in length), you do both of the following:
+
+   a) Give prominent notice with each copy of the object code that the
+   Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the object code with a copy of the GNU GPL and this license
+   document.
+
+  4. Combined Works.
+
+  You may convey a Combined Work under terms of your choice that,
+taken together, effectively do not restrict modification of the
+portions of the Library contained in the Combined Work and reverse
+engineering for debugging such modifications, if you also do each of
+the following:
+
+   a) Give prominent notice with each copy of the Combined Work that
+   the Library is used in it and that the Library and its use are
+   covered by this License.
+
+   b) Accompany the Combined Work with a copy of the GNU GPL and this license
+   document.
+
+   c) For a Combined Work that displays copyright notices during
+   execution, include the copyright notice for the Library among
+   these notices, as well as a reference directing the user to the
+   copies of the GNU GPL and this license document.
+
+   d) Do one of the following:
+
+       0) Convey the Minimal Corresponding Source under the terms of this
+       License, and the Corresponding Application Code in a form
+       suitable for, and under terms that permit, the user to
+       recombine or relink the Application with a modified version of
+       the Linked Version to produce a modified Combined Work, in the
+       manner specified by section 6 of the GNU GPL for conveying
+       Corresponding Source.
+
+       1) Use a suitable shared library mechanism for linking with the
+       Library.  A suitable mechanism is one that (a) uses at run time
+       a copy of the Library already present on the user's computer
+       system, and (b) will operate properly with a modified version
+       of the Library that is interface-compatible with the Linked
+       Version.
+
+   e) Provide Installation Information, but only if you would otherwise
+   be required to provide such information under section 6 of the
+   GNU GPL, and only to the extent that such information is
+   necessary to install and execute a modified version of the
+   Combined Work produced by recombining or relinking the
+   Application with a modified version of the Linked Version. (If
+   you use option 4d0, the Installation Information must accompany
+   the Minimal Corresponding Source and Corresponding Application
+   Code. If you use option 4d1, you must provide the Installation
+   Information in the manner specified by section 6 of the GNU GPL
+   for conveying Corresponding Source.)
+
+  5. Combined Libraries.
+
+  You may place library facilities that are a work based on the
+Library side by side in a single library together with other library
+facilities that are not Applications and are not covered by this
+License, and convey such a combined library under terms of your
+choice, if you do both of the following:
+
+   a) Accompany the combined library with a copy of the same work based
+   on the Library, uncombined with any other library facilities,
+   conveyed under the terms of this License.
+
+   b) Give prominent notice with the combined library that part of it
+   is a work based on the Library, and explaining where to find the
+   accompanying uncombined form of the same work.
+
+  6. Revised Versions of the GNU Lesser General Public License.
+
+  The Free Software Foundation may publish revised and/or new versions
+of the GNU Lesser General Public License from time to time. Such new
+versions will be similar in spirit to the present version, but may
+differ in detail to address new problems or concerns.
+
+  Each version is given a distinguishing version number. If the
+Library as you received it specifies that a certain numbered version
+of the GNU Lesser General Public License "or any later version"
+applies to it, you have the option of following the terms and
+conditions either of that published version or of any later version
+published by the Free Software Foundation. If the Library as you
+received it does not specify a version number of the GNU Lesser
+General Public License, you may choose any version of the GNU Lesser
+General Public License ever published by the Free Software Foundation.
+
+  If the Library as you received it specifies that a proxy can decide
+whether future versions of the GNU Lesser General Public License shall
+apply, that proxy's public statement of acceptance of any version is
+permanent authorization for you to choose that version for the
+Library.

Godeps/_workspace/src/github.com/juju/ratelimit/README.md

@@ -0,0 +1,117 @@
+# ratelimit
+--
+    import "github.com/juju/ratelimit"
+
+The ratelimit package provides an efficient token bucket implementation. See
+http://en.wikipedia.org/wiki/Token_bucket.
+
+## Usage
+
+#### func  Reader
+
+```go
+func Reader(r io.Reader, bucket *Bucket) io.Reader
+```
+Reader returns a reader that is rate limited by the given token bucket. Each
+token in the bucket represents one byte.
+
+#### func  Writer
+
+```go
+func Writer(w io.Writer, bucket *Bucket) io.Writer
+```
+Writer returns a writer that is rate limited by the given token bucket. Each
+token in the bucket represents one byte.
+
+#### type Bucket
+
+```go
+type Bucket struct {
+}
+```
+
+Bucket represents a token bucket that fills at a predetermined rate. Methods on
+Bucket may be called concurrently.
+
+#### func  NewBucket
+
+```go
+func NewBucket(fillInterval time.Duration, capacity int64) *Bucket
+```
+NewBucket returns a new token bucket that fills at the rate of one token every
+fillInterval, up to the given maximum capacity. Both arguments must be positive.
+The bucket is initially full.
+
+#### func  NewBucketWithQuantum
+
+```go
+func NewBucketWithQuantum(fillInterval time.Duration, capacity, quantum int64) *Bucket
+```
+NewBucketWithQuantum is similar to NewBucket, but allows the specification of
+the quantum size - quantum tokens are added every fillInterval.
+
+#### func  NewBucketWithRate
+
+```go
+func NewBucketWithRate(rate float64, capacity int64) *Bucket
+```
+NewBucketWithRate returns a token bucket that fills the bucket at the rate of
+rate tokens per second up to the given maximum capacity. Because of limited
+clock resolution, at high rates, the actual rate may be up to 1% different from
+the specified rate.
+
+#### func (*Bucket) Rate
+
+```go
+func (tb *Bucket) Rate() float64
+```
+Rate returns the fill rate of the bucket, in tokens per second.
+
+#### func (*Bucket) Take
+
+```go
+func (tb *Bucket) Take(count int64) time.Duration
+```
+Take takes count tokens from the bucket without blocking. It returns the time
+that the caller should wait until the tokens are actually available.
+
+Note that if the request is irrevocable - there is no way to return tokens to
+the bucket once this method commits us to taking them.
+
+#### func (*Bucket) TakeAvailable
+
+```go
+func (tb *Bucket) TakeAvailable(count int64) int64
+```
+TakeAvailable takes up to count immediately available tokens from the bucket. It
+returns the number of tokens removed, or zero if there are no available tokens.
+It does not block.
+
+#### func (*Bucket) TakeMaxDuration
+
+```go
+func (tb *Bucket) TakeMaxDuration(count int64, maxWait time.Duration) (time.Duration, bool)
+```
+TakeMaxDuration is like Take, except that it will only take tokens from the
+bucket if the wait time for the tokens is no greater than maxWait.
+
+If it would take longer than maxWait for the tokens to become available, it does
+nothing and reports false, otherwise it returns the time that the caller should
+wait until the tokens are actually available, and reports true.
+
+#### func (*Bucket) Wait
+
+```go
+func (tb *Bucket) Wait(count int64)
+```
+Wait takes count tokens from the bucket, waiting until they are available.
+
+#### func (*Bucket) WaitMaxDuration
+
+```go
+func (tb *Bucket) WaitMaxDuration(count int64, maxWait time.Duration) bool
+```
+WaitMaxDuration is like Wait except that it will only take tokens from the
+bucket if it needs to wait for no greater than maxWait. It reports whether any
+tokens have been removed from the bucket If no tokens have been removed, it
+returns immediately.

Godeps/_workspace/src/github.com/juju/ratelimit/ratelimit.go

@@ -0,0 +1,221 @@
+// Copyright 2014 Canonical Ltd.
+// Licensed under the LGPLv3 with static-linking exception.
+// See LICENCE file for details.
+
+// The ratelimit package provides an efficient token bucket implementation
+// that can be used to limit the rate of arbitrary things.
+// See http://en.wikipedia.org/wiki/Token_bucket.
+package ratelimit
+
+import (
+	"strconv"
+	"sync"
+	"time"
+	"math"
+)
+
+// Bucket represents a token bucket that fills at a predetermined rate.
+// Methods on Bucket may be called concurrently.
+type Bucket struct {
+	startTime    time.Time
+	capacity     int64
+	quantum      int64
+	fillInterval time.Duration
+
+	// The mutex guards the fields following it.
+	mu sync.Mutex
+
+	// avail holds the number of available tokens
+	// in the bucket, as of availTick ticks from startTime.
+	// It will be negative when there are consumers
+	// waiting for tokens.
+	avail     int64
+	availTick int64
+}
+
+// NewBucket returns a new token bucket that fills at the
+// rate of one token every fillInterval, up to the given
+// maximum capacity. Both arguments must be
+// positive. The bucket is initially full.
+func NewBucket(fillInterval time.Duration, capacity int64) *Bucket {
+	return NewBucketWithQuantum(fillInterval, capacity, 1)
+}
+
+// rateMargin specifes the allowed variance of actual
+// rate from specified rate. 1% seems reasonable.
+const rateMargin = 0.01
+
+// NewBucketWithRate returns a token bucket that fills the bucket
+// at the rate of rate tokens per second up to the given
+// maximum capacity. Because of limited clock resolution,
+// at high rates, the actual rate may be up to 1% different from the
+// specified rate.
+func NewBucketWithRate(rate float64, capacity int64) *Bucket {
+	for quantum := int64(1); quantum < 1<<50; quantum = nextQuantum(quantum) {
+		fillInterval := time.Duration(1e9 * float64(quantum) / rate)
+		if fillInterval <= 0 {
+			continue
+		}
+		tb := NewBucketWithQuantum(fillInterval, capacity, quantum)
+		if diff := math.Abs(tb.Rate() - rate); diff/rate <= rateMargin {
+			return tb
+		}
+	}
+	panic("cannot find suitable quantum for " + strconv.FormatFloat(rate, 'g', -1, 64))
+}
+
+// nextQuantum returns the next quantum to try after q.
+// We grow the quantum exponentially, but slowly, so we
+// get a good fit in the lower numbers.
+func nextQuantum(q int64) int64 {
+	q1 := q * 11 / 10
+	if q1 == q {
+		q1++
+	}
+	return q1
+}
+
+// NewBucketWithQuantum is similar to NewBucket, but allows
+// the specification of the quantum size - quantum tokens
+// are added every fillInterval.
+func NewBucketWithQuantum(fillInterval time.Duration, capacity, quantum int64) *Bucket {
+	if fillInterval <= 0 {
+		panic("token bucket fill interval is not > 0")
+	}
+	if capacity <= 0 {
+		panic("token bucket capacity is not > 0")
+	}
+	if quantum <= 0 {
+		panic("token bucket quantum is not > 0")
+	}
+	return &Bucket{
+		startTime:    time.Now(),
+		capacity:     capacity,
+		quantum:      quantum,
+		avail:        capacity,
+		fillInterval: fillInterval,
+	}
+}
+
+// Wait takes count tokens from the bucket, waiting until they are
+// available.
+func (tb *Bucket) Wait(count int64) {
+	if d := tb.Take(count); d > 0 {
+		time.Sleep(d)
+	}
+}
+
+// WaitMaxDuration is like Wait except that it will
+// only take tokens from the bucket if it needs to wait
+// for no greater than maxWait. It reports whether
+// any tokens have been removed from the bucket
+// If no tokens have been removed, it returns immediately.
+func (tb *Bucket) WaitMaxDuration(count int64, maxWait time.Duration) bool {
+	d, ok := tb.TakeMaxDuration(count, maxWait)
+	if d > 0 {
+		time.Sleep(d)
+	}
+	return ok
+}
+
+const infinityDuration time.Duration = 0x7fffffffffffffff
+
+// Take takes count tokens from the bucket without blocking. It returns
+// the time that the caller should wait until the tokens are actually
+// available.
+//
+// Note that if the request is irrevocable - there is no way to return
+// tokens to the bucket once this method commits us to taking them.
+func (tb *Bucket) Take(count int64) time.Duration {
+	d, _ := tb.take(time.Now(), count, infinityDuration)
+	return d
+}
+
+// TakeMaxDuration is like Take, except that
+// it will only take tokens from the bucket if the wait
+// time for the tokens is no greater than maxWait.
+//
+// If it would take longer than maxWait for the tokens
+// to become available, it does nothing and reports false,
+// otherwise it returns the time that the caller should
+// wait until the tokens are actually available, and reports
+// true.
+func (tb *Bucket) TakeMaxDuration(count int64, maxWait time.Duration) (time.Duration, bool) {
+	return tb.take(time.Now(), count, maxWait)
+}
+
+// TakeAvailable takes up to count immediately available tokens from the
+// bucket. It returns the number of tokens removed, or zero if there are
+// no available tokens. It does not block.
+func (tb *Bucket) TakeAvailable(count int64) int64 {
+	return tb.takeAvailable(time.Now(), count)
+}
+
+// takeAvailable is the internal version of TakeAvailable - it takes the
+// current time as an argument to enable easy testing.
+func (tb *Bucket) takeAvailable(now time.Time, count int64) int64 {
+	if count <= 0 {
+		return 0
+	}
+	tb.mu.Lock()
+	defer tb.mu.Unlock()
+
+	tb.adjust(now)
+	if tb.avail <= 0 {
+		return 0
+	}
+	if count > tb.avail {
+		count = tb.avail
+	}
+	tb.avail -= count
+	return count
+}
+
+// Rate returns the fill rate of the bucket, in tokens per second.
+func (tb *Bucket) Rate() float64 {
+	return 1e9 * float64(tb.quantum) / float64(tb.fillInterval)
+}
+
+// take is the internal version of Take - it takes the current time as
+// an argument to enable easy testing.
+func (tb *Bucket) take(now time.Time, count int64, maxWait time.Duration) (time.Duration, bool) {
+	if count <= 0 {
+		return 0, true
+	}
+	tb.mu.Lock()
+	defer tb.mu.Unlock()
+
+	currentTick := tb.adjust(now)
+	avail := tb.avail - count
+	if avail >= 0 {
+		tb.avail = avail
+		return 0, true
+	}
+	// Round up the missing tokens to the nearest multiple
+	// of quantum - the tokens won't be available until
+	// that tick.
+	endTick := currentTick + (-avail+tb.quantum-1)/tb.quantum
+	endTime := tb.startTime.Add(time.Duration(endTick) * tb.fillInterval)
+	waitTime := endTime.Sub(now)
+	if waitTime > maxWait {
+		return 0, false
+	}
+	tb.avail = avail
+	return waitTime, true
+}
+
+// adjust adjusts the current bucket capacity based on the current time.
+// It returns the current tick.
+func (tb *Bucket) adjust(now time.Time) (currentTick int64) {
+	currentTick = int64(now.Sub(tb.startTime) / tb.fillInterval)
+
+	if tb.avail >= tb.capacity {
+		return
+	}
+	tb.avail += (currentTick - tb.availTick) * tb.quantum
+	if tb.avail > tb.capacity {
+		tb.avail = tb.capacity
+	}
+	tb.availTick = currentTick
+	return
+}

Godeps/_workspace/src/github.com/juju/ratelimit/ratelimit_test.go

@@ -0,0 +1,328 @@
+// Copyright 2014 Canonical Ltd.
+// Licensed under the LGPLv3 with static-linking exception.
+// See LICENCE file for details.
+
+package ratelimit
+
+import (
+	gc "launchpad.net/gocheck"
+
+	"testing"
+	"time"
+)
+
+func TestPackage(t *testing.T) {
+	gc.TestingT(t)
+}
+
+type rateLimitSuite struct{}
+
+var _ = gc.Suite(rateLimitSuite{})
+
+type takeReq struct {
+	time       time.Duration
+	count      int64
+	expectWait time.Duration
+}
+
+var takeTests = []struct {
+	about        string
+	fillInterval time.Duration
+	capacity     int64
+	reqs         []takeReq
+}{{
+	about:        "serial requests",
+	fillInterval: 250 * time.Millisecond,
+	capacity:     10,
+	reqs: []takeReq{{
+		time:       0,
+		count:      0,
+		expectWait: 0,
+	}, {
+		time:       0,
+		count:      10,
+		expectWait: 0,
+	}, {
+		time:       0,
+		count:      1,
+		expectWait: 250 * time.Millisecond,
+	}, {
+		time:       250 * time.Millisecond,
+		count:      1,
+		expectWait: 250 * time.Millisecond,
+	}},
+}, {
+	about:        "concurrent requests",
+	fillInterval: 250 * time.Millisecond,
+	capacity:     10,
+	reqs: []takeReq{{
+		time:       0,
+		count:      10,
+		expectWait: 0,
+	}, {
+		time:       0,
+		count:      2,
+		expectWait: 500 * time.Millisecond,
+	}, {
+		time:       0,
+		count:      2,
+		expectWait: 1000 * time.Millisecond,
+	}, {
+		time:       0,
+		count:      1,
+		expectWait: 1250 * time.Millisecond,
+	}},
+}, {
+	about:        "more than capacity",
+	fillInterval: 1 * time.Millisecond,
+	capacity:     10,
+	reqs: []takeReq{{
+		time:       0,
+		count:      10,
+		expectWait: 0,
+	}, {
+		time:       20 * time.Millisecond,
+		count:      15,
+		expectWait: 5 * time.Millisecond,
+	}},
+}, {
+	about:        "sub-quantum time",
+	fillInterval: 10 * time.Millisecond,
+	capacity:     10,
+	reqs: []takeReq{{
+		time:       0,
+		count:      10,
+		expectWait: 0,
+	}, {
+		time:       7 * time.Millisecond,
+		count:      1,
+		expectWait: 3 * time.Millisecond,
+	}, {
+		time:       8 * time.Millisecond,
+		count:      1,
+		expectWait: 12 * time.Millisecond,
+	}},
+}, {
+	about:        "within capacity",
+	fillInterval: 10 * time.Millisecond,
+	capacity:     5,
+	reqs: []takeReq{{
+		time:       0,
+		count:      5,
+		expectWait: 0,
+	}, {
+		time:       60 * time.Millisecond,
+		count:      5,
+		expectWait: 0,
+	}, {
+		time:       60 * time.Millisecond,
+		count:      1,
+		expectWait: 10 * time.Millisecond,
+	}, {
+		time:       80 * time.Millisecond,
+		count:      2,
+		expectWait: 10 * time.Millisecond,
+	}},
+}}
+
+func (rateLimitSuite) TestTake(c *gc.C) {
+	for i, test := range takeTests {
+		tb := NewBucket(test.fillInterval, test.capacity)
+		for j, req := range test.reqs {
+			d, ok := tb.take(tb.startTime.Add(req.time), req.count, infinityDuration)
+			c.Assert(ok, gc.Equals, true)
+			if d != req.expectWait {
+				c.Fatalf("test %d.%d, %s, got %v want %v", i, j, test.about, d, req.expectWait)
+			}
+		}
+	}
+}
+
+func (rateLimitSuite) TestTakeMaxDuration(c *gc.C) {
+	for i, test := range takeTests {
+		tb := NewBucket(test.fillInterval, test.capacity)
+		for j, req := range test.reqs {
+			if req.expectWait > 0 {
+				d, ok := tb.take(tb.startTime.Add(req.time), req.count, req.expectWait-1)
+				c.Assert(ok, gc.Equals, false)
+				c.Assert(d, gc.Equals, time.Duration(0))
+			}
+			d, ok := tb.take(tb.startTime.Add(req.time), req.count, req.expectWait)
+			c.Assert(ok, gc.Equals, true)
+			if d != req.expectWait {
+				c.Fatalf("test %d.%d, %s, got %v want %v", i, j, test.about, d, req.expectWait)
+			}
+		}
+	}
+}
+
+type takeAvailableReq struct {
+	time   time.Duration
+	count  int64
+	expect int64
+}
+
+var takeAvailableTests = []struct {
+	about        string
+	fillInterval time.Duration
+	capacity     int64
+	reqs         []takeAvailableReq
+}{{
+	about:        "serial requests",
+	fillInterval: 250 * time.Millisecond,
+	capacity:     10,
+	reqs: []takeAvailableReq{{
+		time:   0,
+		count:  0,
+		expect: 0,
+	}, {
+		time:   0,
+		count:  10,
+		expect: 10,
+	}, {
+		time:   0,
+		count:  1,
+		expect: 0,
+	}, {
+		time:   250 * time.Millisecond,
+		count:  1,
+		expect: 1,
+	}},
+}, {
+	about:        "concurrent requests",
+	fillInterval: 250 * time.Millisecond,
+	capacity:     10,
+	reqs: []takeAvailableReq{{
+		time:   0,
+		count:  5,
+		expect: 5,
+	}, {
+		time:   0,
+		count:  2,
+		expect: 2,
+	}, {
+		time:   0,
+		count:  5,
+		expect: 3,
+	}, {
+		time:   0,
+		count:  1,
+		expect: 0,
+	}},
+}, {
+	about:        "more than capacity",
+	fillInterval: 1 * time.Millisecond,
+	capacity:     10,
+	reqs: []takeAvailableReq{{
+		time:   0,
+		count:  10,
+		expect: 10,
+	}, {
+		time:   20 * time.Millisecond,
+		count:  15,
+		expect: 10,
+	}},
+}, {
+	about:        "within capacity",
+	fillInterval: 10 * time.Millisecond,
+	capacity:     5,
+	reqs: []takeAvailableReq{{
+		time:   0,
+		count:  5,
+		expect: 5,
+	}, {
+		time:   60 * time.Millisecond,
+		count:  5,
+		expect: 5,
+	}, {
+		time:   70 * time.Millisecond,
+		count:  1,
+		expect: 1,
+	}},
+}}
+
+func (rateLimitSuite) TestTakeAvailable(c *gc.C) {
+	for i, test := range takeAvailableTests {
+		tb := NewBucket(test.fillInterval, test.capacity)
+		for j, req := range test.reqs {
+			d := tb.takeAvailable(tb.startTime.Add(req.time), req.count)
+			if d != req.expect {
+				c.Fatalf("test %d.%d, %s, got %v want %v", i, j, test.about, d, req.expect)
+			}
+		}
+	}
+}
+
+func (rateLimitSuite) TestPanics(c *gc.C) {
+	c.Assert(func() { NewBucket(0, 1) }, gc.PanicMatches, "token bucket fill interval is not > 0")
+	c.Assert(func() { NewBucket(-2, 1) }, gc.PanicMatches, "token bucket fill interval is not > 0")
+	c.Assert(func() { NewBucket(1, 0) }, gc.PanicMatches, "token bucket capacity is not > 0")
+	c.Assert(func() { NewBucket(1, -2) }, gc.PanicMatches, "token bucket capacity is not > 0")
+}
+
+func isCloseTo(x, y, tolerance float64) bool {
+	return abs(x-y)/y < tolerance
+}
+
+func (rateLimitSuite) TestRate(c *gc.C) {
+	tb := NewBucket(1, 1)
+	if !isCloseTo(tb.Rate(), 1e9, 0.00001) {
+		c.Fatalf("got %v want 1e9", tb.Rate())
+	}
+	tb = NewBucket(2*time.Second, 1)
+	if !isCloseTo(tb.Rate(), 0.5, 0.00001) {
+		c.Fatalf("got %v want 0.5", tb.Rate())
+	}
+	tb = NewBucketWithQuantum(100*time.Millisecond, 1, 5)
+	if !isCloseTo(tb.Rate(), 50, 0.00001) {
+		c.Fatalf("got %v want 50", tb.Rate())
+	}
+}
+
+func checkRate(c *gc.C, rate float64) {
+	tb := NewBucketWithRate(rate, 1<<62)
+	if !isCloseTo(tb.Rate(), rate, rateMargin) {
+		c.Fatalf("got %g want %v", tb.Rate(), rate)
+	}
+	d, ok := tb.take(tb.startTime, 1<<62, infinityDuration)
+	c.Assert(ok, gc.Equals, true)
+	c.Assert(d, gc.Equals, time.Duration(0))
+
+	// Check that the actual rate is as expected by
+	// asking for a not-quite multiple of the bucket's
+	// quantum and checking that the wait time
+	// correct.
+	d, ok = tb.take(tb.startTime, tb.quantum*2-tb.quantum/2, infinityDuration)
+	c.Assert(ok, gc.Equals, true)
+	expectTime := 1e9 * float64(tb.quantum) * 2 / rate
+	if !isCloseTo(float64(d), expectTime, rateMargin) {
+		c.Fatalf("rate %g: got %g want %v", rate, float64(d), expectTime)
+	}
+}
+
+func (rateLimitSuite) TestNewWithRate(c *gc.C) {
+	for rate := float64(1); rate < 1e6; rate += 7 {
+		checkRate(c, rate)
+	}
+	for _, rate := range []float64{
+		1024 * 1024 * 1024,
+		1e-5,
+		0.9e-5,
+		0.5,
+		0.9,
+		0.9e8,
+		3e12,
+		4e18,
+	} {
+		checkRate(c, rate)
+		checkRate(c, rate/3)
+		checkRate(c, rate*1.3)
+	}
+}
+
+func BenchmarkWait(b *testing.B) {
+	tb := NewBucket(1, 16*1024)
+	for i := b.N - 1; i >= 0; i-- {
+		tb.Wait(1)
+	}
+}

Godeps/_workspace/src/github.com/juju/ratelimit/reader.go

@@ -0,0 +1,51 @@
+// Copyright 2014 Canonical Ltd.
+// Licensed under the LGPLv3 with static-linking exception.
+// See LICENCE file for details.
+
+package ratelimit
+
+import "io"
+
+type reader struct {
+	r      io.Reader
+	bucket *Bucket
+}
+
+// Reader returns a reader that is rate limited by
+// the given token bucket. Each token in the bucket
+// represents one byte.
+func Reader(r io.Reader, bucket *Bucket) io.Reader {
+	return &reader{
+		r:      r,
+		bucket: bucket,
+	}
+}
+
+func (r *reader) Read(buf []byte) (int, error) {
+	n, err := r.r.Read(buf)
+	if n <= 0 {
+		return n, err
+	}
+	r.bucket.Wait(int64(n))
+	return n, err
+}
+
+type writer struct {
+	w      io.Writer
+	bucket *Bucket
+}
+
+// Writer returns a reader that is rate limited by
+// the given token bucket. Each token in the bucket
+// represents one byte.
+func Writer(w io.Writer, bucket *Bucket) io.Writer {
+	return &writer{
+		w:      w,
+		bucket: bucket,
+	}
+}
+
+func (w *writer) Write(buf []byte) (int, error) {
+	w.bucket.Wait(int64(len(buf)))
+	return w.w.Write(buf)
+}

Godeps/_workspace/src/github.com/kardianos/osext/osext.go

@@ -3,22 +3,16 @@
 // license that can be found in the LICENSE file.
 
 // Extensions to the standard "os" package.
-package osext // import "github.com/kardianos/osext"
+package osext
 
 import "path/filepath"
 
-var cx, ce = executableClean()
-
-func executableClean() (string, error) {
-	p, err := executable()
-	return filepath.Clean(p), err
-}
-
 // Executable returns an absolute path that can be used to
 // re-invoke the current program.
 // It may not be valid after the current program exits.
 func Executable() (string, error) {
-	return cx, ce
+	p, err := executable()
+	return filepath.Clean(p), err
 }
 
 // Returns same path as Executable, returns just the folder

Godeps/_workspace/src/github.com/kardianos/osext/osext_procfs.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build linux netbsd solaris dragonfly
+// +build linux netbsd openbsd solaris dragonfly
 
 package osext
 
@@ -27,7 +27,7 @@ func executable() (string, error) {
 		return execpath, nil
 	case "netbsd":
 		return os.Readlink("/proc/curproc/exe")
-	case "dragonfly":
+	case "openbsd", "dragonfly":
 		return os.Readlink("/proc/curproc/file")
 	case "solaris":
 		return os.Readlink(fmt.Sprintf("/proc/%d/path/a.out", os.Getpid()))

Godeps/_workspace/src/github.com/kardianos/osext/osext_sysctl.go

@@ -2,13 +2,12 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build darwin freebsd openbsd
+// +build darwin freebsd
 
 package osext
 
 import (
 	"os"
-	"os/exec"
 	"path/filepath"
 	"runtime"
 	"syscall"
@@ -24,8 +23,6 @@ func executable() (string, error) {
 		mib = [4]int32{1 /* CTL_KERN */, 14 /* KERN_PROC */, 12 /* KERN_PROC_PATHNAME */, -1}
 	case "darwin":
 		mib = [4]int32{1 /* CTL_KERN */, 38 /* KERN_PROCARGS */, int32(os.Getpid()), -1}
-	case "openbsd":
-		mib = [4]int32{1 /* CTL_KERN */, 55 /* KERN_PROC_ARGS */, int32(os.Getpid()), 1 /* KERN_PROC_ARGV */}
 	}
 
 	n := uintptr(0)
@@ -45,58 +42,14 @@ func executable() (string, error) {
 	if n == 0 { // This shouldn't happen.
 		return "", nil
 	}
-
-	var execPath string
-	switch runtime.GOOS {
-	case "openbsd":
-		// buf now contains **argv, with pointers to each of the C-style
-		// NULL terminated arguments.
-		var args []string
-		argv := uintptr(unsafe.Pointer(&buf[0]))
-	Loop:
-		for {
-			argp := *(**[1 << 20]byte)(unsafe.Pointer(argv))
-			if argp == nil {
-				break
-			}
-			for i := 0; uintptr(i) < n; i++ {
-				// we don't want the full arguments list
-				if string(argp[i]) == " " {
-					break Loop
-				}
-				if argp[i] != 0 {
-					continue
-				}
-				args = append(args, string(argp[:i]))
-				n -= uintptr(i)
-				break
-			}
-			if n < unsafe.Sizeof(argv) {
-				break
-			}
-			argv += unsafe.Sizeof(argv)
-			n -= unsafe.Sizeof(argv)
+	for i, v := range buf {
+		if v == 0 {
+			buf = buf[:i]
+			break
 		}
-		execPath = args[0]
-		// There is no canonical way to get an executable path on
-		// OpenBSD, so check PATH in case we are called directly
-		if execPath[0] != '/' && execPath[0] != '.' {
-			execIsInPath, err := exec.LookPath(execPath)
-			if err == nil {
-				execPath = execIsInPath
-			}
-		}
-	default:
-		for i, v := range buf {
-			if v == 0 {
-				buf = buf[:i]
-				break
-			}
-		}
-		execPath = string(buf)
 	}
-
 	var err error
+	execPath := string(buf)
 	// execPath will not be empty due to above checks.
 	// Try to get the absolute path if the execPath is not rooted.
 	if execPath[0] != '/' {

Godeps/_workspace/src/github.com/kardianos/osext/osext_test.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build darwin linux freebsd netbsd windows openbsd
+// +build darwin linux freebsd netbsd windows
 
 package osext
 

Godeps/_workspace/src/github.com/syncthing/protocol/AUTHORS

@@ -0,0 +1,4 @@
+# This is the official list of Protocol Authors for copyright purposes.
+
+Audrius Butkevicius <audrius.butkevicius@gmail.com>
+Jakob Borg <jakob@nym.se>

Godeps/_workspace/src/github.com/syncthing/protocol/CONTRIBUTING.md

@@ -0,0 +1,76 @@
+## Reporting Bugs
+
+Please file bugs in the [Github Issue
+Tracker](https://github.com/syncthing/protocol/issues).
+
+## Contributing Code
+
+Every contribution is welcome. Following the points below will make this
+a smoother process.
+
+Individuals making significant and valuable contributions are given
+commit-access to the project. If you make a significant contribution and
+are not considered for commit-access, please contact any of the
+Syncthing core team members.
+
+All nontrivial contributions should go through the pull request
+mechanism for internal review. Determining what is "nontrivial" is left
+at the discretion of the contributor.
+
+### Authorship
+
+All code authors are listed in the AUTHORS file. Commits must be made
+with the same name and email as listed in the AUTHORS file. To
+accomplish this, ensure that your git configuration is set correctly
+prior to making your first commit;
+
+    $ git config --global user.name "Jane Doe"
+    $ git config --global user.email janedoe@example.com
+
+You must be reachable on the given email address. If you do not wish to
+use your real name for whatever reason, using a nickname or pseudonym is
+perfectly acceptable.
+
+## Coding Style
+
+- Follow the conventions laid out in [Effective Go](https://golang.org/doc/effective_go.html)
+  as much as makes sense.
+
+- All text files use Unix line endings.
+
+- Each commit should be `go fmt` clean.
+
+- The commit message subject should be a single short sentence
+  describing the change, starting with a capital letter.
+
+- Commits that resolve an existing issue must include the issue number
+  as `(fixes #123)` at the end of the commit message subject.
+
+- Imports are grouped per `goimports` standard; that is, standard
+  library first, then third party libraries after a blank line.
+
+- A contribution solving a single issue or introducing a single new
+  feature should probably be a single commit based on the current
+  `master` branch. You may be asked to "rebase" or "squash" your pull
+  request to make sure this is the case, especially if there have been
+  amendments during review.
+
+## Licensing
+
+All contributions are made under the same MIT license as the rest of the
+project, except documentation, user interface text and translation
+strings which are licensed under the Creative Commons Attribution 4.0
+International License. You retain the copyright to code you have
+written.
+
+When accepting your first contribution, the maintainer of the project
+will ensure that you are added to the AUTHORS file. You are welcome to
+add yourself as a separate commit in your first pull request.
+
+## Tests
+
+Yes please!
+
+## License
+
+MIT

Godeps/_workspace/src/github.com/syncthing/protocol/README.md

@@ -0,0 +1,13 @@
+The BEPv1 Protocol
+==================
+
+[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/protocol.svg?style=flat-square)](http://build.syncthing.net/job/protocol/lastBuild/)
+[![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/protocol)
+[![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat-square)](http://opensource.org/licenses/MIT)
+
+This is the protocol implementation used by Syncthing.
+
+License
+=======
+
+MIT

Godeps/_workspace/src/github.com/syncthing/protocol/common_test.go

@@ -0,0 +1,81 @@
+// Copyright (C) 2014 The Protocol Authors.
+
+package protocol
+
+import (
+	"io"
+	"time"
+)
+
+type TestModel struct {
+	data     []byte
+	folder   string
+	name     string
+	offset   int64
+	size     int
+	hash     []byte
+	flags    uint32
+	options  []Option
+	closedCh chan bool
+}
+
+func newTestModel() *TestModel {
+	return &TestModel{
+		closedCh: make(chan bool),
+	}
+}
+
+func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+}
+
+func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+}
+
+func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
+	t.folder = folder
+	t.name = name
+	t.offset = offset
+	t.size = len(buf)
+	t.hash = hash
+	t.flags = flags
+	t.options = options
+	copy(buf, t.data)
+	return nil
+}
+
+func (t *TestModel) Close(deviceID DeviceID, err error) {
+	close(t.closedCh)
+}
+
+func (t *TestModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
+}
+
+func (t *TestModel) isClosed() bool {
+	select {
+	case <-t.closedCh:
+		return true
+	case <-time.After(1 * time.Second):
+		return false // Timeout
+	}
+}
+
+type ErrPipe struct {
+	io.PipeWriter
+	written int
+	max     int
+	err     error
+	closed  bool
+}
+
+func (e *ErrPipe) Write(data []byte) (int, error) {
+	if e.closed {
+		return 0, e.err
+	}
+	if e.written+len(data) > e.max {
+		n, _ := e.PipeWriter.Write(data[:e.max-e.written])
+		e.PipeWriter.CloseWithError(e.err)
+		e.closed = true
+		return n, e.err
+	}
+	return e.PipeWriter.Write(data)
+}

Godeps/_workspace/src/github.com/syncthing/protocol/compression.go

@@ -4,7 +4,13 @@ package protocol
 
 import "fmt"
 
+type Compression int
+
 const (
+	CompressMetadata Compression = iota // zero value is the default, default should be "metadata"
+	CompressNever
+	CompressAlways
+
 	compressionThreshold = 128 // don't bother compressing messages smaller than this many bytes
 )
 
@@ -25,6 +31,14 @@ var compressionUnmarshal = map[string]Compression{
 	"always":   CompressAlways,
 }
 
+func (c Compression) String() string {
+	s, ok := compressionMarshal[c]
+	if !ok {
+		return fmt.Sprintf("unknown:%d", c)
+	}
+	return s
+}
+
 func (c Compression) GoString() string {
 	return fmt.Sprintf("%q", c.String())
 }

Godeps/_workspace/src/github.com/syncthing/protocol/conflict_test.go

@@ -7,9 +7,9 @@ import "testing"
 func TestWinsConflict(t *testing.T) {
 	testcases := [][2]FileInfo{
 		// The first should always win over the second
-		{{ModifiedS: 42}, {ModifiedS: 41}},
-		{{ModifiedS: 41}, {ModifiedS: 42, Deleted: true}},
-		{{ModifiedS: 41, Version: Vector{[]Counter{{42, 2}, {43, 1}}}}, {ModifiedS: 41, Version: Vector{[]Counter{{42, 1}, {43, 2}}}}},
+		{{Modified: 42}, {Modified: 41}},
+		{{Modified: 41}, {Modified: 42, Flags: FlagDeleted}},
+		{{Modified: 41, Version: Vector{{42, 2}, {43, 1}}}, {Modified: 41, Version: Vector{{42, 1}, {43, 2}}}},
 	}
 
 	for _, tc := range testcases {

Godeps/_workspace/src/github.com/syncthing/protocol/debug.go

@@ -0,0 +1,15 @@
+// Copyright (C) 2014 The Protocol Authors.
+
+package protocol
+
+import (
+	"os"
+	"strings"
+
+	"github.com/calmh/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "protocol") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
+)

Godeps/_workspace/src/github.com/syncthing/protocol/deviceid.go

@@ -4,6 +4,7 @@ package protocol
 
 import (
 	"bytes"
+	"crypto/sha256"
 	"encoding/base32"
 	"encoding/binary"
 	"errors"
@@ -11,20 +12,12 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/syncthing/syncthing/lib/sha256"
-
 	"github.com/calmh/luhn"
 )
 
-const DeviceIDLength = 32
+type DeviceID [32]byte
 
-type DeviceID [DeviceIDLength]byte
-type ShortID uint64
-
-var (
-	LocalDeviceID = DeviceID{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
-	EmptyDeviceID = DeviceID{ /* all zeroes */ }
-)
+var LocalDeviceID = DeviceID{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
 
 // NewDeviceID generates a new device ID from the raw bytes of a certificate
 func NewDeviceID(rawCert []byte) DeviceID {
@@ -52,9 +45,6 @@ func DeviceIDFromBytes(bs []byte) DeviceID {
 
 // String returns the canonical string representation of the device ID
 func (n DeviceID) String() string {
-	if n == EmptyDeviceID {
-		return ""
-	}
 	id := base32.StdEncoding.EncodeToString(n[:])
 	id = strings.Trim(id, "=")
 	id, err := luhnify(id)
@@ -75,27 +65,18 @@ func (n DeviceID) Compare(other DeviceID) int {
 }
 
 func (n DeviceID) Equals(other DeviceID) bool {
-	return bytes.Equal(n[:], other[:])
+	return bytes.Compare(n[:], other[:]) == 0
 }
 
 // Short returns an integer representing bits 0-63 of the device ID.
-func (n DeviceID) Short() ShortID {
-	return ShortID(binary.BigEndian.Uint64(n[:]))
+func (n DeviceID) Short() uint64 {
+	return binary.BigEndian.Uint64(n[:])
 }
 
 func (n *DeviceID) MarshalText() ([]byte, error) {
 	return []byte(n.String()), nil
 }
 
-func (s ShortID) String() string {
-	if s == 0 {
-		return ""
-	}
-	var bs [8]byte
-	binary.BigEndian.PutUint64(bs[:], uint64(s))
-	return base32.StdEncoding.EncodeToString(bs[:])[:7]
-}
-
 func (n *DeviceID) UnmarshalText(bs []byte) error {
 	id := string(bs)
 	id = strings.Trim(id, "=")
@@ -105,9 +86,6 @@ func (n *DeviceID) UnmarshalText(bs []byte) error {
 
 	var err error
 	switch len(id) {
-	case 0:
-		*n = EmptyDeviceID
-		return nil
 	case 56:
 		// New style, with check digits
 		id, err = unluhnify(id)
@@ -128,29 +106,6 @@ func (n *DeviceID) UnmarshalText(bs []byte) error {
 	}
 }
 
-func (n *DeviceID) ProtoSize() int {
-	// Used by protobuf marshaller.
-	return DeviceIDLength
-}
-
-func (n *DeviceID) MarshalTo(bs []byte) (int, error) {
-	// Used by protobuf marshaller.
-	if len(bs) < DeviceIDLength {
-		return 0, errors.New("destination too short")
-	}
-	copy(bs, (*n)[:])
-	return DeviceIDLength, nil
-}
-
-func (n *DeviceID) Unmarshal(bs []byte) error {
-	// Used by protobuf marshaller.
-	if len(bs) < DeviceIDLength {
-		return errors.New("not enough data")
-	}
-	copy((*n)[:], bs)
-	return nil
-}
-
 func luhnify(s string) (string, error) {
 	if len(s) != 52 {
 		panic("unsupported string length")
@@ -206,18 +161,3 @@ func untypeoify(s string) string {
 	s = strings.Replace(s, "8", "B", -1)
 	return s
 }
-
-// DeviceIDs is a sortable slice of DeviceID
-type DeviceIDs []DeviceID
-
-func (l DeviceIDs) Len() int {
-	return len(l)
-}
-
-func (l DeviceIDs) Less(a, b int) bool {
-	return l[a].Compare(l[b]) == -1
-}
-
-func (l DeviceIDs) Swap(a, b int) {
-	l[a], l[b] = l[b], l[a]
-}

Godeps/_workspace/src/github.com/syncthing/protocol/deviceid_test.go

@@ -1,8 +1,5 @@
 // Copyright (C) 2014 The Protocol Authors.
 
-//go:generate go run ../../script/protofmt.go deviceid_test.proto
-//go:generate protoc -I ../../vendor/ -I ../../vendor/github.com/gogo/protobuf/protobuf -I . --gogofast_out=. deviceid_test.proto
-
 package protocol
 
 import "testing"
@@ -37,8 +34,7 @@ var validateCases = []struct {
 	s  string
 	ok bool
 }{
-	{"", true}, // Empty device ID, all zeroes
-	{"a", false},
+	{"", false},
 	{"P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2", true},
 	{"P56IOI7-MZJNU2-IQGDREY-DM2MGT-MGL3BXN-PQ6W5B-TBBZ4TJ-XZWICQ", true},
 	{"P56IOI7 MZJNU2I QGDREYD M2MGTMGL 3BXNPQ6W 5BTB BZ4T JXZWICQ", true},
@@ -78,75 +74,3 @@ func TestMarshallingDeviceID(t *testing.T) {
 		t.Error("Compare error")
 	}
 }
-
-func TestShortIDString(t *testing.T) {
-	id, _ := DeviceIDFromString(formatted)
-
-	sid := id.Short().String()
-	if len(sid) != 7 {
-		t.Errorf("Wrong length for short ID: got %d, want 7", len(sid))
-	}
-
-	want := formatted[:len(sid)]
-	if sid != want {
-		t.Errorf("Wrong short ID: got %q, want %q", sid, want)
-	}
-}
-
-func TestDeviceIDFromBytes(t *testing.T) {
-	id0, _ := DeviceIDFromString(formatted)
-	id1 := DeviceIDFromBytes(id0[:])
-	if id1.String() != formatted {
-		t.Errorf("Wrong device ID, got %q, want %q", id1, formatted)
-	}
-}
-
-func TestNewDeviceIDMarshalling(t *testing.T) {
-	// The new DeviceID.Unmarshal / DeviceID.MarshalTo serialization should
-	// be message compatible with how we used to serialize DeviceIDs.
-
-	// Create a message with a device ID in old style bytes format
-
-	id0, _ := DeviceIDFromString(formatted)
-	msg0 := TestOldDeviceID{id0[:]}
-
-	//  Marshal it
-
-	bs, err := msg0.Marshal()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Unmarshal using the new DeviceID.Unmarshal
-
-	var msg1 TestNewDeviceID
-	if err := msg1.Unmarshal(bs); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify it's the same
-
-	if msg1.Test != id0 {
-		t.Error("Mismatch in old -> new direction")
-	}
-
-	// Marshal using the new DeviceID.MarshalTo
-
-	bs, err = msg1.Marshal()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Create an old style message and attempt unmarshal
-
-	var msg2 TestOldDeviceID
-	if err := msg2.Unmarshal(bs); err != nil {
-		t.Fatal(err)
-	}
-
-	// Verify it's the same
-
-	if DeviceIDFromBytes(msg2.Test) != id0 {
-		t.Error("Mismatch in old -> new direction")
-	}
-}

Godeps/_workspace/src/github.com/syncthing/protocol/errors.go

@@ -0,0 +1,51 @@
+// Copyright (C) 2014 The Protocol Authors.
+
+package protocol
+
+import (
+	"errors"
+)
+
+const (
+	ecNoError int32 = iota
+	ecGeneric
+	ecNoSuchFile
+	ecInvalid
+)
+
+var (
+	ErrNoError    error = nil
+	ErrGeneric          = errors.New("generic error")
+	ErrNoSuchFile       = errors.New("no such file")
+	ErrInvalid          = errors.New("file is invalid")
+)
+
+var lookupError = map[int32]error{
+	ecNoError:    ErrNoError,
+	ecGeneric:    ErrGeneric,
+	ecNoSuchFile: ErrNoSuchFile,
+	ecInvalid:    ErrInvalid,
+}
+
+var lookupCode = map[error]int32{
+	ErrNoError:    ecNoError,
+	ErrGeneric:    ecGeneric,
+	ErrNoSuchFile: ecNoSuchFile,
+	ErrInvalid:    ecInvalid,
+}
+
+func codeToError(errcode int32) error {
+	err, ok := lookupError[errcode]
+	if !ok {
+		return ErrGeneric
+	}
+	return err
+}
+
+func errorToCode(err error) int32 {
+	code, ok := lookupCode[err]
+	if !ok {
+		return ecGeneric
+	}
+	return code
+}

Godeps/_workspace/src/github.com/syncthing/protocol/fuzz.go

@@ -0,0 +1,70 @@
+// Copyright (C) 2015 The Protocol Authors.
+
+// +build gofuzz
+
+package protocol
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"reflect"
+	"sync"
+)
+
+func Fuzz(data []byte) int {
+	// Regenerate the length, or we'll most commonly exit quickly due to an
+	// unexpected eof which is unintestering.
+	if len(data) > 8 {
+		binary.BigEndian.PutUint32(data[4:], uint32(len(data))-8)
+	}
+
+	// Setup a rawConnection we'll use to parse the message.
+	c := rawConnection{
+		cr:     &countingReader{Reader: bytes.NewReader(data)},
+		closed: make(chan struct{}),
+		pool: sync.Pool{
+			New: func() interface{} {
+				return make([]byte, BlockSize)
+			},
+		},
+	}
+
+	// Attempt to parse the message.
+	hdr, msg, err := c.readMessage()
+	if err != nil {
+		return 0
+	}
+
+	// If parsing worked, attempt to encode it again.
+	newBs, err := msg.AppendXDR(nil)
+	if err != nil {
+		panic("not encodable")
+	}
+
+	// Create an appriate header for the re-encoding.
+	newMsg := make([]byte, 8)
+	binary.BigEndian.PutUint32(newMsg, encodeHeader(hdr))
+	binary.BigEndian.PutUint32(newMsg[4:], uint32(len(newBs)))
+	newMsg = append(newMsg, newBs...)
+
+	// Use the rawConnection to parse the re-encoding.
+	c.cr = &countingReader{Reader: bytes.NewReader(newMsg)}
+	hdr2, msg2, err := c.readMessage()
+	if err != nil {
+		fmt.Println("Initial:\n" + hex.Dump(data))
+		fmt.Println("New:\n" + hex.Dump(newMsg))
+		panic("not parseable after re-encode: " + err.Error())
+	}
+
+	// Make sure the data is the same as it was before.
+	if hdr != hdr2 {
+		panic("headers differ")
+	}
+	if !reflect.DeepEqual(msg, msg2) {
+		panic("contents differ")
+	}
+
+	return 1
+}

Godeps/_workspace/src/github.com/syncthing/protocol/fuzz_test.go

@@ -0,0 +1,89 @@
+// Copyright (C) 2015 The Protocol Authors.
+
+// +build gofuzz
+
+package protocol
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+	"testing/quick"
+)
+
+// This can be used to generate a corpus of valid messages as a starting point
+// for the fuzzer.
+func TestGenerateCorpus(t *testing.T) {
+	t.Skip("Use to generate initial corpus only")
+
+	n := 0
+	check := func(idx IndexMessage) bool {
+		for i := range idx.Options {
+			if len(idx.Options[i].Key) > 64 {
+				idx.Options[i].Key = idx.Options[i].Key[:64]
+			}
+		}
+		hdr := header{
+			version:     0,
+			msgID:       42,
+			msgType:     messageTypeIndex,
+			compression: false,
+		}
+
+		msgBs := idx.MustMarshalXDR()
+
+		buf := make([]byte, 8)
+		binary.BigEndian.PutUint32(buf, encodeHeader(hdr))
+		binary.BigEndian.PutUint32(buf[4:], uint32(len(msgBs)))
+		buf = append(buf, msgBs...)
+
+		ioutil.WriteFile(fmt.Sprintf("testdata/corpus/test-%03d.xdr", n), buf, 0644)
+		n++
+		return true
+	}
+
+	if err := quick.Check(check, &quick.Config{MaxCount: 1000}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tests any crashers found by the fuzzer, for closer investigation.
+func TestCrashers(t *testing.T) {
+	testFiles(t, "testdata/crashers")
+}
+
+// Tests the entire corpus, which should PASS before the fuzzer starts
+// fuzzing.
+func TestCorpus(t *testing.T) {
+	testFiles(t, "testdata/corpus")
+}
+
+func testFiles(t *testing.T, dir string) {
+	fd, err := os.Open(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	crashers, err := fd.Readdirnames(-1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, name := range crashers {
+		if strings.HasSuffix(name, ".output") {
+			continue
+		}
+		if strings.HasSuffix(name, ".quoted") {
+			continue
+		}
+
+		t.Log(name)
+		crasher, err := ioutil.ReadFile(dir + "/" + name)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		Fuzz(crasher)
+	}
+}

Godeps/_workspace/src/github.com/syncthing/protocol/header.go

@@ -0,0 +1,43 @@
+// Copyright (C) 2014 The Protocol Authors.
+
+package protocol
+
+import "github.com/calmh/xdr"
+
+type header struct {
+	version     int
+	msgID       int
+	msgType     int
+	compression bool
+}
+
+func (h header) encodeXDR(xw *xdr.Writer) (int, error) {
+	u := encodeHeader(h)
+	return xw.WriteUint32(u)
+}
+
+func (h *header) decodeXDR(xr *xdr.Reader) error {
+	u := xr.ReadUint32()
+	*h = decodeHeader(u)
+	return xr.Error()
+}
+
+func encodeHeader(h header) uint32 {
+	var isComp uint32
+	if h.compression {
+		isComp = 1 << 0 // the zeroth bit is the compression bit
+	}
+	return uint32(h.version&0xf)<<28 +
+		uint32(h.msgID&0xfff)<<16 +
+		uint32(h.msgType&0xff)<<8 +
+		isComp
+}
+
+func decodeHeader(u uint32) header {
+	return header{
+		version:     int(u>>28) & 0xf,
+		msgID:       int(u>>16) & 0xfff,
+		msgType:     int(u>>8) & 0xff,
+		compression: u&1 == 1,
+	}
+}

Godeps/_workspace/src/github.com/syncthing/protocol/message.go

@@ -0,0 +1,151 @@
+// Copyright (C) 2014 The Protocol Authors.
+
+//go:generate -command genxdr go run ../syncthing/Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go
+//go:generate genxdr -o message_xdr.go message.go
+
+package protocol
+
+import "fmt"
+
+type IndexMessage struct {
+	Folder  string
+	Files   []FileInfo // max:1000000
+	Flags   uint32
+	Options []Option // max:64
+}
+
+type FileInfo struct {
+	Name         string // max:8192
+	Flags        uint32
+	Modified     int64
+	Version      Vector
+	LocalVersion int64
+	Blocks       []BlockInfo // max:1000000
+}
+
+func (f FileInfo) String() string {
+	return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%v, Size:%d, Blocks:%v}",
+		f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.Blocks)
+}
+
+func (f FileInfo) Size() (bytes int64) {
+	if f.IsDeleted() || f.IsDirectory() {
+		return 128
+	}
+	for _, b := range f.Blocks {
+		bytes += int64(b.Size)
+	}
+	return
+}
+
+func (f FileInfo) IsDeleted() bool {
+	return f.Flags&FlagDeleted != 0
+}
+
+func (f FileInfo) IsInvalid() bool {
+	return f.Flags&FlagInvalid != 0
+}
+
+func (f FileInfo) IsDirectory() bool {
+	return f.Flags&FlagDirectory != 0
+}
+
+func (f FileInfo) IsSymlink() bool {
+	return f.Flags&FlagSymlink != 0
+}
+
+func (f FileInfo) HasPermissionBits() bool {
+	return f.Flags&FlagNoPermBits == 0
+}
+
+// WinsConflict returns true if "f" is the one to choose when it is in
+// conflict with "other".
+func (f FileInfo) WinsConflict(other FileInfo) bool {
+	// If a modification is in conflict with a delete, we pick the
+	// modification.
+	if !f.IsDeleted() && other.IsDeleted() {
+		return true
+	}
+	if f.IsDeleted() && !other.IsDeleted() {
+		return false
+	}
+
+	// The one with the newer modification time wins.
+	if f.Modified > other.Modified {
+		return true
+	}
+	if f.Modified < other.Modified {
+		return false
+	}
+
+	// The modification times were equal. Use the device ID in the version
+	// vector as tie breaker.
+	return f.Version.Compare(other.Version) == ConcurrentGreater
+}
+
+type BlockInfo struct {
+	Offset int64 // noencode (cache only)
+	Size   int32
+	Hash   []byte // max:64
+}
+
+func (b BlockInfo) String() string {
+	return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash)
+}
+
+type RequestMessage struct {
+	Folder  string // max:64
+	Name    string // max:8192
+	Offset  int64
+	Size    int32
+	Hash    []byte // max:64
+	Flags   uint32
+	Options []Option // max:64
+}
+
+type ResponseMessage struct {
+	Data []byte
+	Code int32
+}
+
+type ClusterConfigMessage struct {
+	ClientName    string   // max:64
+	ClientVersion string   // max:64
+	Folders       []Folder // max:1000000
+	Options       []Option // max:64
+}
+
+func (o *ClusterConfigMessage) GetOption(key string) string {
+	for _, option := range o.Options {
+		if option.Key == key {
+			return option.Value
+		}
+	}
+	return ""
+}
+
+type Folder struct {
+	ID      string   // max:64
+	Devices []Device // max:1000000
+	Flags   uint32
+	Options []Option // max:64
+}
+
+type Device struct {
+	ID              []byte // max:32
+	MaxLocalVersion int64
+	Flags           uint32
+	Options         []Option // max:64
+}
+
+type Option struct {
+	Key   string // max:64
+	Value string // max:1024
+}
+
+type CloseMessage struct {
+	Reason string // max:1024
+	Code   int32
+}
+
+type EmptyMessage struct{}

Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_darwin.go

@@ -0,0 +1,40 @@
+// Copyright (C) 2014 The Protocol Authors.
+
+// +build darwin
+
+package protocol
+
+// Darwin uses NFD normalization
+
+import "golang.org/x/text/unicode/norm"
+
+type nativeModel struct {
+	next Model
+}
+
+func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+	for i := range files {
+		files[i].Name = norm.NFD.String(files[i].Name)
+	}
+	m.next.Index(deviceID, folder, files, flags, options)
+}
+
+func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+	for i := range files {
+		files[i].Name = norm.NFD.String(files[i].Name)
+	}
+	m.next.IndexUpdate(deviceID, folder, files, flags, options)
+}
+
+func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
+	name = norm.NFD.String(name)
+	return m.next.Request(deviceID, folder, name, offset, hash, flags, options, buf)
+}
+
+func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
+	m.next.ClusterConfig(deviceID, config)
+}
+
+func (m nativeModel) Close(deviceID DeviceID, err error) {
+	m.next.Close(deviceID, err)
+}

Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_unix.go

@@ -0,0 +1,31 @@
+// Copyright (C) 2014 The Protocol Authors.
+
+// +build !windows,!darwin
+
+package protocol
+
+// Normal Unixes uses NFC and slashes, which is the wire format.
+
+type nativeModel struct {
+	next Model
+}
+
+func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+	m.next.Index(deviceID, folder, files, flags, options)
+}
+
+func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+	m.next.IndexUpdate(deviceID, folder, files, flags, options)
+}
+
+func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
+	return m.next.Request(deviceID, folder, name, offset, hash, flags, options, buf)
+}
+
+func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
+	m.next.ClusterConfig(deviceID, config)
+}
+
+func (m nativeModel) Close(deviceID DeviceID, err error) {
+	m.next.Close(deviceID, err)
+}

Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_windows.go

@@ -0,0 +1,63 @@
+// Copyright (C) 2014 The Protocol Authors.
+
+// +build windows
+
+package protocol
+
+// Windows uses backslashes as file separator and disallows a bunch of
+// characters in the filename
+
+import (
+	"path/filepath"
+	"strings"
+)
+
+var disallowedCharacters = string([]rune{
+	'<', '>', ':', '"', '|', '?', '*',
+	0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
+	11, 12, 13, 14, 15, 16, 17, 18, 19, 20,
+	21, 22, 23, 24, 25, 26, 27, 28, 29, 30,
+	31,
+})
+
+type nativeModel struct {
+	next Model
+}
+
+func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+	fixupFiles(folder, files)
+	m.next.Index(deviceID, folder, files, flags, options)
+}
+
+func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+	fixupFiles(folder, files)
+	m.next.IndexUpdate(deviceID, folder, files, flags, options)
+}
+
+func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
+	name = filepath.FromSlash(name)
+	return m.next.Request(deviceID, folder, name, offset, hash, flags, options, buf)
+}
+
+func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
+	m.next.ClusterConfig(deviceID, config)
+}
+
+func (m nativeModel) Close(deviceID DeviceID, err error) {
+	m.next.Close(deviceID, err)
+}
+
+func fixupFiles(folder string, files []FileInfo) {
+	for i, f := range files {
+		if strings.ContainsAny(f.Name, disallowedCharacters) {
+			if f.IsDeleted() {
+				// Don't complain if the file is marked as deleted, since it
+				// can't possibly exist here anyway.
+				continue
+			}
+			files[i].Flags |= FlagInvalid
+			l.Warnf("File name %q (folder %q) contains invalid characters; marked as invalid.", f.Name, folder)
+		}
+		files[i].Name = filepath.FromSlash(files[i].Name)
+	}
+}

Godeps/_workspace/src/github.com/syncthing/protocol/protocol.go

@@ -0,0 +1,788 @@
+// Copyright (C) 2014 The Protocol Authors.
+
+package protocol
+
+import (
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"sync"
+	"time"
+
+	lz4 "github.com/bkaradzic/go-lz4"
+)
+
+const (
+	// Data block size (128 KiB)
+	BlockSize = 128 << 10
+
+	// We reject messages larger than this when encountered on the wire. (64 MiB)
+	MaxMessageLen = 64 << 20
+)
+
+const (
+	messageTypeClusterConfig = 0
+	messageTypeIndex         = 1
+	messageTypeRequest       = 2
+	messageTypeResponse      = 3
+	messageTypePing          = 4
+	messageTypePong          = 5
+	messageTypeIndexUpdate   = 6
+	messageTypeClose         = 7
+)
+
+const (
+	stateInitial = iota
+	stateReady
+)
+
+// FileInfo flags
+const (
+	FlagDeleted              uint32 = 1 << 12
+	FlagInvalid                     = 1 << 13
+	FlagDirectory                   = 1 << 14
+	FlagNoPermBits                  = 1 << 15
+	FlagSymlink                     = 1 << 16
+	FlagSymlinkMissingTarget        = 1 << 17
+
+	FlagsAll = (1 << 18) - 1
+
+	SymlinkTypeMask = FlagDirectory | FlagSymlinkMissingTarget
+)
+
+// IndexMessage message flags (for IndexUpdate)
+const (
+	FlagIndexTemporary uint32 = 1 << iota
+)
+
+// Request message flags
+const (
+	FlagRequestTemporary uint32 = 1 << iota
+)
+
+// ClusterConfigMessage.Folders.Devices flags
+const (
+	FlagShareTrusted  uint32 = 1 << 0
+	FlagShareReadOnly        = 1 << 1
+	FlagIntroducer           = 1 << 2
+	FlagShareBits            = 0x000000ff
+)
+
+var (
+	ErrClusterHash = fmt.Errorf("configuration error: mismatched cluster hash")
+	ErrClosed      = errors.New("connection closed")
+)
+
+// Specific variants of empty messages...
+type pingMessage struct{ EmptyMessage }
+type pongMessage struct{ EmptyMessage }
+
+type Model interface {
+	// An index was received from the peer device
+	Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option)
+	// An index update was received from the peer device
+	IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option)
+	// A request was made by the peer device
+	Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error
+	// A cluster configuration message was received
+	ClusterConfig(deviceID DeviceID, config ClusterConfigMessage)
+	// The peer device closed the connection
+	Close(deviceID DeviceID, err error)
+}
+
+type Connection interface {
+	Start()
+	ID() DeviceID
+	Name() string
+	Index(folder string, files []FileInfo, flags uint32, options []Option) error
+	IndexUpdate(folder string, files []FileInfo, flags uint32, options []Option) error
+	Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error)
+	ClusterConfig(config ClusterConfigMessage)
+	Statistics() Statistics
+}
+
+type rawConnection struct {
+	id       DeviceID
+	name     string
+	receiver Model
+
+	cr *countingReader
+	cw *countingWriter
+
+	awaiting    [4096]chan asyncResult
+	awaitingMut sync.Mutex
+
+	idxMut sync.Mutex // ensures serialization of Index calls
+
+	nextID      chan int
+	outbox      chan hdrMsg
+	closed      chan struct{}
+	once        sync.Once
+	pool        sync.Pool
+	compression Compression
+
+	rdbuf0 []byte // used & reused by readMessage
+	rdbuf1 []byte // used & reused by readMessage
+}
+
+type asyncResult struct {
+	val []byte
+	err error
+}
+
+type hdrMsg struct {
+	hdr  header
+	msg  encodable
+	done chan struct{}
+}
+
+type encodable interface {
+	AppendXDR([]byte) ([]byte, error)
+}
+
+type isEofer interface {
+	IsEOF() bool
+}
+
+var (
+	PingTimeout  = 30 * time.Second
+	PingIdleTime = 60 * time.Second
+)
+
+func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiver Model, name string, compress Compression) Connection {
+	cr := &countingReader{Reader: reader}
+	cw := &countingWriter{Writer: writer}
+
+	c := rawConnection{
+		id:       deviceID,
+		name:     name,
+		receiver: nativeModel{receiver},
+		cr:       cr,
+		cw:       cw,
+		outbox:   make(chan hdrMsg),
+		nextID:   make(chan int),
+		closed:   make(chan struct{}),
+		pool: sync.Pool{
+			New: func() interface{} {
+				return make([]byte, BlockSize)
+			},
+		},
+		compression: compress,
+	}
+
+	return wireFormatConnection{&c}
+}
+
+// Start creates the goroutines for sending and receiving of messages. It must
+// be called exactly once after creating a connection.
+func (c *rawConnection) Start() {
+	go c.readerLoop()
+	go c.writerLoop()
+	go c.pingerLoop()
+	go c.idGenerator()
+}
+
+func (c *rawConnection) ID() DeviceID {
+	return c.id
+}
+
+func (c *rawConnection) Name() string {
+	return c.name
+}
+
+// Index writes the list of file information to the connected peer device
+func (c *rawConnection) Index(folder string, idx []FileInfo, flags uint32, options []Option) error {
+	select {
+	case <-c.closed:
+		return ErrClosed
+	default:
+	}
+	c.idxMut.Lock()
+	c.send(-1, messageTypeIndex, IndexMessage{
+		Folder:  folder,
+		Files:   idx,
+		Flags:   flags,
+		Options: options,
+	}, nil)
+	c.idxMut.Unlock()
+	return nil
+}
+
+// IndexUpdate writes the list of file information to the connected peer device as an update
+func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo, flags uint32, options []Option) error {
+	select {
+	case <-c.closed:
+		return ErrClosed
+	default:
+	}
+	c.idxMut.Lock()
+	c.send(-1, messageTypeIndexUpdate, IndexMessage{
+		Folder:  folder,
+		Files:   idx,
+		Flags:   flags,
+		Options: options,
+	}, nil)
+	c.idxMut.Unlock()
+	return nil
+}
+
+// Request returns the bytes for the specified block after fetching them from the connected peer.
+func (c *rawConnection) Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
+	var id int
+	select {
+	case id = <-c.nextID:
+	case <-c.closed:
+		return nil, ErrClosed
+	}
+
+	c.awaitingMut.Lock()
+	if ch := c.awaiting[id]; ch != nil {
+		panic("id taken")
+	}
+	rc := make(chan asyncResult, 1)
+	c.awaiting[id] = rc
+	c.awaitingMut.Unlock()
+
+	ok := c.send(id, messageTypeRequest, RequestMessage{
+		Folder:  folder,
+		Name:    name,
+		Offset:  offset,
+		Size:    int32(size),
+		Hash:    hash,
+		Flags:   flags,
+		Options: options,
+	}, nil)
+	if !ok {
+		return nil, ErrClosed
+	}
+
+	res, ok := <-rc
+	if !ok {
+		return nil, ErrClosed
+	}
+	return res.val, res.err
+}
+
+// ClusterConfig send the cluster configuration message to the peer and returns any error
+func (c *rawConnection) ClusterConfig(config ClusterConfigMessage) {
+	c.send(-1, messageTypeClusterConfig, config, nil)
+}
+
+func (c *rawConnection) ping() bool {
+	var id int
+	select {
+	case id = <-c.nextID:
+	case <-c.closed:
+		return false
+	}
+
+	rc := make(chan asyncResult, 1)
+	c.awaitingMut.Lock()
+	c.awaiting[id] = rc
+	c.awaitingMut.Unlock()
+
+	ok := c.send(id, messageTypePing, nil, nil)
+	if !ok {
+		return false
+	}
+
+	res, ok := <-rc
+	return ok && res.err == nil
+}
+
+func (c *rawConnection) readerLoop() (err error) {
+	defer func() {
+		c.close(err)
+	}()
+
+	state := stateInitial
+	for {
+		select {
+		case <-c.closed:
+			return ErrClosed
+		default:
+		}
+
+		hdr, msg, err := c.readMessage()
+		if err != nil {
+			return err
+		}
+
+		switch msg := msg.(type) {
+		case ClusterConfigMessage:
+			if state != stateInitial {
+				return fmt.Errorf("protocol error: cluster config message in state %d", state)
+			}
+			go c.receiver.ClusterConfig(c.id, msg)
+			state = stateReady
+
+		case IndexMessage:
+			switch hdr.msgType {
+			case messageTypeIndex:
+				if state != stateReady {
+					return fmt.Errorf("protocol error: index message in state %d", state)
+				}
+				c.handleIndex(msg)
+				state = stateReady
+
+			case messageTypeIndexUpdate:
+				if state != stateReady {
+					return fmt.Errorf("protocol error: index update message in state %d", state)
+				}
+				c.handleIndexUpdate(msg)
+				state = stateReady
+			}
+
+		case RequestMessage:
+			if state != stateReady {
+				return fmt.Errorf("protocol error: request message in state %d", state)
+			}
+			// Requests are handled asynchronously
+			go c.handleRequest(hdr.msgID, msg)
+
+		case ResponseMessage:
+			if state != stateReady {
+				return fmt.Errorf("protocol error: response message in state %d", state)
+			}
+			c.handleResponse(hdr.msgID, msg)
+
+		case pingMessage:
+			if state != stateReady {
+				return fmt.Errorf("protocol error: ping message in state %d", state)
+			}
+			c.send(hdr.msgID, messageTypePong, pongMessage{}, nil)
+
+		case pongMessage:
+			if state != stateReady {
+				return fmt.Errorf("protocol error: pong message in state %d", state)
+			}
+			c.handlePong(hdr.msgID)
+
+		case CloseMessage:
+			return errors.New(msg.Reason)
+
+		default:
+			return fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType)
+		}
+	}
+}
+
+func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) {
+	if cap(c.rdbuf0) < 8 {
+		c.rdbuf0 = make([]byte, 8)
+	} else {
+		c.rdbuf0 = c.rdbuf0[:8]
+	}
+	_, err = io.ReadFull(c.cr, c.rdbuf0)
+	if err != nil {
+		return
+	}
+
+	hdr = decodeHeader(binary.BigEndian.Uint32(c.rdbuf0[0:4]))
+	msglen := int(binary.BigEndian.Uint32(c.rdbuf0[4:8]))
+
+	if debug {
+		l.Debugf("read header %v (msglen=%d)", hdr, msglen)
+	}
+
+	if msglen > MaxMessageLen {
+		err = fmt.Errorf("message length %d exceeds maximum %d", msglen, MaxMessageLen)
+		return
+	}
+
+	if hdr.version != 0 {
+		err = fmt.Errorf("unknown protocol version 0x%x", hdr.version)
+		return
+	}
+
+	if cap(c.rdbuf0) < msglen {
+		c.rdbuf0 = make([]byte, msglen)
+	} else {
+		c.rdbuf0 = c.rdbuf0[:msglen]
+	}
+	_, err = io.ReadFull(c.cr, c.rdbuf0)
+	if err != nil {
+		return
+	}
+
+	if debug {
+		l.Debugf("read %d bytes", len(c.rdbuf0))
+	}
+
+	msgBuf := c.rdbuf0
+	if hdr.compression && msglen > 0 {
+		c.rdbuf1 = c.rdbuf1[:cap(c.rdbuf1)]
+		c.rdbuf1, err = lz4.Decode(c.rdbuf1, c.rdbuf0)
+		if err != nil {
+			return
+		}
+		msgBuf = c.rdbuf1
+		if debug {
+			l.Debugf("decompressed to %d bytes", len(msgBuf))
+		}
+	}
+
+	if debug {
+		if len(msgBuf) > 1024 {
+			l.Debugf("message data:\n%s", hex.Dump(msgBuf[:1024]))
+		} else {
+			l.Debugf("message data:\n%s", hex.Dump(msgBuf))
+		}
+	}
+
+	// We check each returned error for the XDRError.IsEOF() method.
+	// IsEOF()==true here means that the message contained fewer fields than
+	// expected. It does not signify an EOF on the socket, because we've
+	// successfully read a size value and that many bytes already. New fields
+	// we expected but the other peer didn't send should be interpreted as
+	// zero/nil, and if that's not valid we'll verify it somewhere else.
+
+	switch hdr.msgType {
+	case messageTypeIndex, messageTypeIndexUpdate:
+		var idx IndexMessage
+		err = idx.UnmarshalXDR(msgBuf)
+		if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() {
+			err = nil
+		}
+		msg = idx
+
+	case messageTypeRequest:
+		var req RequestMessage
+		err = req.UnmarshalXDR(msgBuf)
+		if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() {
+			err = nil
+		}
+		msg = req
+
+	case messageTypeResponse:
+		var resp ResponseMessage
+		err = resp.UnmarshalXDR(msgBuf)
+		if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() {
+			err = nil
+		}
+		msg = resp
+
+	case messageTypePing:
+		msg = pingMessage{}
+
+	case messageTypePong:
+		msg = pongMessage{}
+
+	case messageTypeClusterConfig:
+		var cc ClusterConfigMessage
+		err = cc.UnmarshalXDR(msgBuf)
+		if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() {
+			err = nil
+		}
+		msg = cc
+
+	case messageTypeClose:
+		var cm CloseMessage
+		err = cm.UnmarshalXDR(msgBuf)
+		if xdrErr, ok := err.(isEofer); ok && xdrErr.IsEOF() {
+			err = nil
+		}
+		msg = cm
+
+	default:
+		err = fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType)
+	}
+
+	return
+}
+
+func (c *rawConnection) handleIndex(im IndexMessage) {
+	if debug {
+		l.Debugf("Index(%v, %v, %d file, flags %x, opts: %s)", c.id, im.Folder, len(im.Files), im.Flags, im.Options)
+	}
+	c.receiver.Index(c.id, im.Folder, filterIndexMessageFiles(im.Files), im.Flags, im.Options)
+}
+
+func (c *rawConnection) handleIndexUpdate(im IndexMessage) {
+	if debug {
+		l.Debugf("queueing IndexUpdate(%v, %v, %d files, flags %x, opts: %s)", c.id, im.Folder, len(im.Files), im.Flags, im.Options)
+	}
+	c.receiver.IndexUpdate(c.id, im.Folder, filterIndexMessageFiles(im.Files), im.Flags, im.Options)
+}
+
+func filterIndexMessageFiles(fs []FileInfo) []FileInfo {
+	var out []FileInfo
+	for i, f := range fs {
+		switch f.Name {
+		case "", ".", "..", "/": // A few obviously invalid filenames
+			l.Infof("Dropping invalid filename %q from incoming index", f.Name)
+			if out == nil {
+				// Most incoming updates won't contain anything invalid, so we
+				// delay the allocation and copy to output slice until we
+				// really need to do it, then copy all the so var valid files
+				// to it.
+				out = make([]FileInfo, i, len(fs)-1)
+				copy(out, fs)
+			}
+		default:
+			if out != nil {
+				out = append(out, f)
+			}
+		}
+	}
+	if out != nil {
+		return out
+	}
+	return fs
+}
+
+func (c *rawConnection) handleRequest(msgID int, req RequestMessage) {
+	size := int(req.Size)
+	usePool := size <= BlockSize
+
+	var buf []byte
+	var done chan struct{}
+
+	if usePool {
+		buf = c.pool.Get().([]byte)[:size]
+		done = make(chan struct{})
+	} else {
+		buf = make([]byte, size)
+	}
+
+	err := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), req.Hash, req.Flags, req.Options, buf)
+	if err != nil {
+		c.send(msgID, messageTypeResponse, ResponseMessage{
+			Data: nil,
+			Code: errorToCode(err),
+		}, done)
+	} else {
+		c.send(msgID, messageTypeResponse, ResponseMessage{
+			Data: buf,
+			Code: errorToCode(err),
+		}, done)
+	}
+
+	if usePool {
+		<-done
+		c.pool.Put(buf)
+	}
+}
+
+func (c *rawConnection) handleResponse(msgID int, resp ResponseMessage) {
+	c.awaitingMut.Lock()
+	if rc := c.awaiting[msgID]; rc != nil {
+		c.awaiting[msgID] = nil
+		rc <- asyncResult{resp.Data, codeToError(resp.Code)}
+		close(rc)
+	}
+	c.awaitingMut.Unlock()
+}
+
+func (c *rawConnection) handlePong(msgID int) {
+	c.awaitingMut.Lock()
+	if rc := c.awaiting[msgID]; rc != nil {
+		c.awaiting[msgID] = nil
+		rc <- asyncResult{}
+		close(rc)
+	}
+	c.awaitingMut.Unlock()
+}
+
+func (c *rawConnection) send(msgID int, msgType int, msg encodable, done chan struct{}) bool {
+	if msgID < 0 {
+		select {
+		case id := <-c.nextID:
+			msgID = id
+		case <-c.closed:
+			return false
+		}
+	}
+
+	hdr := header{
+		version: 0,
+		msgID:   msgID,
+		msgType: msgType,
+	}
+
+	select {
+	case c.outbox <- hdrMsg{hdr, msg, done}:
+		return true
+	case <-c.closed:
+		return false
+	}
+}
+
+func (c *rawConnection) writerLoop() {
+	var msgBuf = make([]byte, 8) // buffer for wire format message, kept and reused
+	var uncBuf []byte            // buffer for uncompressed message, kept and reused
+	for {
+		var tempBuf []byte
+		var err error
+
+		select {
+		case hm := <-c.outbox:
+			if hm.msg != nil {
+				// Uncompressed message in uncBuf
+				uncBuf, err = hm.msg.AppendXDR(uncBuf[:0])
+				if hm.done != nil {
+					close(hm.done)
+				}
+				if err != nil {
+					c.close(err)
+					return
+				}
+
+				compress := false
+				switch c.compression {
+				case CompressAlways:
+					compress = true
+				case CompressMetadata:
+					compress = hm.hdr.msgType != messageTypeResponse
+				}
+
+				if compress && len(uncBuf) >= compressionThreshold {
+					// Use compression for large messages
+					hm.hdr.compression = true
+
+					// Make sure we have enough space for the compressed message plus header in msgBug
+					msgBuf = msgBuf[:cap(msgBuf)]
+					if maxLen := lz4.CompressBound(len(uncBuf)) + 8; maxLen > len(msgBuf) {
+						msgBuf = make([]byte, maxLen)
+					}
+
+					// Compressed is written to msgBuf, we keep tb for the length only
+					tempBuf, err = lz4.Encode(msgBuf[8:], uncBuf)
+					binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(tempBuf)))
+					msgBuf = msgBuf[0 : len(tempBuf)+8]
+
+					if debug {
+						l.Debugf("write compressed message; %v (len=%d)", hm.hdr, len(tempBuf))
+					}
+				} else {
+					// No point in compressing very short messages
+					hm.hdr.compression = false
+
+					msgBuf = msgBuf[:cap(msgBuf)]
+					if l := len(uncBuf) + 8; l > len(msgBuf) {
+						msgBuf = make([]byte, l)
+					}
+
+					binary.BigEndian.PutUint32(msgBuf[4:8], uint32(len(uncBuf)))
+					msgBuf = msgBuf[0 : len(uncBuf)+8]
+					copy(msgBuf[8:], uncBuf)
+
+					if debug {
+						l.Debugf("write uncompressed message; %v (len=%d)", hm.hdr, len(uncBuf))
+					}
+				}
+			} else {
+				if debug {
+					l.Debugf("write empty message; %v", hm.hdr)
+				}
+				binary.BigEndian.PutUint32(msgBuf[4:8], 0)
+				msgBuf = msgBuf[:8]
+			}
+
+			binary.BigEndian.PutUint32(msgBuf[0:4], encodeHeader(hm.hdr))
+
+			if err == nil {
+				var n int
+				n, err = c.cw.Write(msgBuf)
+				if debug {
+					l.Debugf("wrote %d bytes on the wire", n)
+				}
+			}
+			if err != nil {
+				c.close(err)
+				return
+			}
+		case <-c.closed:
+			return
+		}
+	}
+}
+
+func (c *rawConnection) close(err error) {
+	c.once.Do(func() {
+		close(c.closed)
+
+		c.awaitingMut.Lock()
+		for i, ch := range c.awaiting {
+			if ch != nil {
+				close(ch)
+				c.awaiting[i] = nil
+			}
+		}
+		c.awaitingMut.Unlock()
+
+		go c.receiver.Close(c.id, err)
+	})
+}
+
+func (c *rawConnection) idGenerator() {
+	nextID := 0
+	for {
+		nextID = (nextID + 1) & 0xfff
+		select {
+		case c.nextID <- nextID:
+		case <-c.closed:
+			return
+		}
+	}
+}
+
+func (c *rawConnection) pingerLoop() {
+	var rc = make(chan bool, 1)
+	ticker := time.Tick(PingIdleTime / 2)
+	for {
+		select {
+		case <-ticker:
+			if d := time.Since(c.cr.Last()); d < PingIdleTime {
+				if debug {
+					l.Debugln(c.id, "ping skipped after rd", d)
+				}
+				continue
+			}
+			if d := time.Since(c.cw.Last()); d < PingIdleTime {
+				if debug {
+					l.Debugln(c.id, "ping skipped after wr", d)
+				}
+				continue
+			}
+			go func() {
+				if debug {
+					l.Debugln(c.id, "ping ->")
+				}
+				rc <- c.ping()
+			}()
+			select {
+			case ok := <-rc:
+				if debug {
+					l.Debugln(c.id, "<- pong")
+				}
+				if !ok {
+					c.close(fmt.Errorf("ping failure"))
+				}
+			case <-time.After(PingTimeout):
+				c.close(fmt.Errorf("ping timeout"))
+			case <-c.closed:
+				return
+			}
+
+		case <-c.closed:
+			return
+		}
+	}
+}
+
+type Statistics struct {
+	At            time.Time
+	InBytesTotal  int64
+	OutBytesTotal int64
+}
+
+func (c *rawConnection) Statistics() Statistics {
+	return Statistics{
+		At:            time.Now(),
+		InBytesTotal:  c.cr.Tot(),
+		OutBytesTotal: c.cw.Tot(),
+	}
+}

Godeps/_workspace/src/github.com/syncthing/protocol/protocol_test.go

@@ -0,0 +1,404 @@
+// Copyright (C) 2014 The Protocol Authors.
+
+package protocol
+
+import (
+	"bytes"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+	"testing/quick"
+
+	"github.com/calmh/xdr"
+)
+
+var (
+	c0ID = NewDeviceID([]byte{1})
+	c1ID = NewDeviceID([]byte{2})
+)
+
+func TestHeaderFunctions(t *testing.T) {
+	f := func(ver, id, typ int) bool {
+		ver = int(uint(ver) % 16)
+		id = int(uint(id) % 4096)
+		typ = int(uint(typ) % 256)
+		h0 := header{version: ver, msgID: id, msgType: typ}
+		h1 := decodeHeader(encodeHeader(h0))
+		return h0 == h1
+	}
+	if err := quick.Check(f, nil); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestHeaderLayout(t *testing.T) {
+	var e, a uint32
+
+	// Version are the first four bits
+	e = 0xf0000000
+	a = encodeHeader(header{version: 0xf})
+	if a != e {
+		t.Errorf("Header layout incorrect; %08x != %08x", a, e)
+	}
+
+	// Message ID are the following 12 bits
+	e = 0x0fff0000
+	a = encodeHeader(header{msgID: 0xfff})
+	if a != e {
+		t.Errorf("Header layout incorrect; %08x != %08x", a, e)
+	}
+
+	// Type are the last 8 bits before reserved
+	e = 0x0000ff00
+	a = encodeHeader(header{msgType: 0xff})
+	if a != e {
+		t.Errorf("Header layout incorrect; %08x != %08x", a, e)
+	}
+}
+
+func TestPing(t *testing.T) {
+	ar, aw := io.Pipe()
+	br, bw := io.Pipe()
+
+	c0 := NewConnection(c0ID, ar, bw, newTestModel(), "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
+	c0.Start()
+	c1 := NewConnection(c1ID, br, aw, newTestModel(), "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
+	c1.Start()
+	c0.ClusterConfig(ClusterConfigMessage{})
+	c1.ClusterConfig(ClusterConfigMessage{})
+
+	if ok := c0.ping(); !ok {
+		t.Error("c0 ping failed")
+	}
+	if ok := c1.ping(); !ok {
+		t.Error("c1 ping failed")
+	}
+}
+
+func TestPingErr(t *testing.T) {
+	e := errors.New("something broke")
+
+	for i := 0; i < 32; i++ {
+		for j := 0; j < 32; j++ {
+			m0 := newTestModel()
+			m1 := newTestModel()
+
+			ar, aw := io.Pipe()
+			br, bw := io.Pipe()
+			eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e}
+			ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e}
+
+			c0 := NewConnection(c0ID, ar, ebw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
+			c0.Start()
+			c1 := NewConnection(c1ID, br, eaw, m1, "name", CompressAlways)
+			c1.Start()
+			c0.ClusterConfig(ClusterConfigMessage{})
+			c1.ClusterConfig(ClusterConfigMessage{})
+
+			res := c0.ping()
+			if (i < 8 || j < 8) && res {
+				// This should have resulted in failure, as there is no way an empty ClusterConfig plus a Ping message fits in eight bytes.
+				t.Errorf("Unexpected ping success; i=%d, j=%d", i, j)
+			} else if (i >= 28 && j >= 28) && !res {
+				// This should have worked though, as 28 bytes is plenty for both.
+				t.Errorf("Unexpected ping fail; i=%d, j=%d", i, j)
+			}
+		}
+	}
+}
+
+// func TestRequestResponseErr(t *testing.T) {
+// 	e := errors.New("something broke")
+
+// 	var pass bool
+// 	for i := 0; i < 48; i++ {
+// 		for j := 0; j < 38; j++ {
+// 			m0 := newTestModel()
+// 			m0.data = []byte("response data")
+// 			m1 := newTestModel()
+
+// 			ar, aw := io.Pipe()
+// 			br, bw := io.Pipe()
+// 			eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e}
+// 			ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e}
+
+// 			NewConnection(c0ID, ar, ebw, m0, nil)
+// 			c1 := NewConnection(c1ID, br, eaw, m1, nil).(wireFormatConnection).next.(*rawConnection)
+
+// 			d, err := c1.Request("default", "tn", 1234, 5678)
+// 			if err == e || err == ErrClosed {
+// 				t.Logf("Error at %d+%d bytes", i, j)
+// 				if !m1.isClosed() {
+// 					t.Fatal("c1 not closed")
+// 				}
+// 				if !m0.isClosed() {
+// 					t.Fatal("c0 not closed")
+// 				}
+// 				continue
+// 			}
+// 			if err != nil {
+// 				t.Fatal(err)
+// 			}
+// 			if string(d) != "response data" {
+// 				t.Fatalf("Incorrect response data %q", string(d))
+// 			}
+// 			if m0.folder != "default" {
+// 				t.Fatalf("Incorrect folder %q", m0.folder)
+// 			}
+// 			if m0.name != "tn" {
+// 				t.Fatalf("Incorrect name %q", m0.name)
+// 			}
+// 			if m0.offset != 1234 {
+// 				t.Fatalf("Incorrect offset %d", m0.offset)
+// 			}
+// 			if m0.size != 5678 {
+// 				t.Fatalf("Incorrect size %d", m0.size)
+// 			}
+// 			t.Logf("Pass at %d+%d bytes", i, j)
+// 			pass = true
+// 		}
+// 	}
+// 	if !pass {
+// 		t.Fatal("Never passed")
+// 	}
+// }
+
+func TestVersionErr(t *testing.T) {
+	m0 := newTestModel()
+	m1 := newTestModel()
+
+	ar, aw := io.Pipe()
+	br, bw := io.Pipe()
+
+	c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
+	c0.Start()
+	c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
+	c1.Start()
+	c0.ClusterConfig(ClusterConfigMessage{})
+	c1.ClusterConfig(ClusterConfigMessage{})
+
+	w := xdr.NewWriter(c0.cw)
+	w.WriteUint32(encodeHeader(header{
+		version: 2,
+		msgID:   0,
+		msgType: 0,
+	}))
+	w.WriteUint32(0) // Avoids reader closing due to EOF
+
+	if !m1.isClosed() {
+		t.Error("Connection should close due to unknown version")
+	}
+}
+
+func TestTypeErr(t *testing.T) {
+	m0 := newTestModel()
+	m1 := newTestModel()
+
+	ar, aw := io.Pipe()
+	br, bw := io.Pipe()
+
+	c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
+	c0.Start()
+	c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
+	c1.Start()
+	c0.ClusterConfig(ClusterConfigMessage{})
+	c1.ClusterConfig(ClusterConfigMessage{})
+
+	w := xdr.NewWriter(c0.cw)
+	w.WriteUint32(encodeHeader(header{
+		version: 0,
+		msgID:   0,
+		msgType: 42,
+	}))
+	w.WriteUint32(0) // Avoids reader closing due to EOF
+
+	if !m1.isClosed() {
+		t.Error("Connection should close due to unknown message type")
+	}
+}
+
+func TestClose(t *testing.T) {
+	m0 := newTestModel()
+	m1 := newTestModel()
+
+	ar, aw := io.Pipe()
+	br, bw := io.Pipe()
+
+	c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
+	c0.Start()
+	c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
+	c1.Start()
+	c0.ClusterConfig(ClusterConfigMessage{})
+	c1.ClusterConfig(ClusterConfigMessage{})
+
+	c0.close(nil)
+
+	<-c0.closed
+	if !m0.isClosed() {
+		t.Fatal("Connection should be closed")
+	}
+
+	// None of these should panic, some should return an error
+
+	if c0.ping() {
+		t.Error("Ping should not return true")
+	}
+
+	c0.Index("default", nil, 0, nil)
+	c0.Index("default", nil, 0, nil)
+
+	if _, err := c0.Request("default", "foo", 0, 0, nil, 0, nil); err == nil {
+		t.Error("Request should return an error")
+	}
+}
+
+func TestElementSizeExceededNested(t *testing.T) {
+	m := ClusterConfigMessage{
+		Folders: []Folder{
+			{ID: "longstringlongstringlongstringinglongstringlongstringlonlongstringlongstringlon"},
+		},
+	}
+	_, err := m.EncodeXDR(ioutil.Discard)
+	if err == nil {
+		t.Errorf("ID length %d > max 64, but no error", len(m.Folders[0].ID))
+	}
+}
+
+func TestMarshalIndexMessage(t *testing.T) {
+	var quickCfg = &quick.Config{MaxCountScale: 10}
+	if testing.Short() {
+		quickCfg = nil
+	}
+
+	f := func(m1 IndexMessage) bool {
+		for _, f := range m1.Files {
+			for i := range f.Blocks {
+				f.Blocks[i].Offset = 0
+				if len(f.Blocks[i].Hash) == 0 {
+					f.Blocks[i].Hash = nil
+				}
+			}
+		}
+
+		return testMarshal(t, "index", &m1, &IndexMessage{})
+	}
+
+	if err := quick.Check(f, quickCfg); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestMarshalRequestMessage(t *testing.T) {
+	var quickCfg = &quick.Config{MaxCountScale: 10}
+	if testing.Short() {
+		quickCfg = nil
+	}
+
+	f := func(m1 RequestMessage) bool {
+		return testMarshal(t, "request", &m1, &RequestMessage{})
+	}
+
+	if err := quick.Check(f, quickCfg); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestMarshalResponseMessage(t *testing.T) {
+	var quickCfg = &quick.Config{MaxCountScale: 10}
+	if testing.Short() {
+		quickCfg = nil
+	}
+
+	f := func(m1 ResponseMessage) bool {
+		if len(m1.Data) == 0 {
+			m1.Data = nil
+		}
+		return testMarshal(t, "response", &m1, &ResponseMessage{})
+	}
+
+	if err := quick.Check(f, quickCfg); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestMarshalClusterConfigMessage(t *testing.T) {
+	var quickCfg = &quick.Config{MaxCountScale: 10}
+	if testing.Short() {
+		quickCfg = nil
+	}
+
+	f := func(m1 ClusterConfigMessage) bool {
+		return testMarshal(t, "clusterconfig", &m1, &ClusterConfigMessage{})
+	}
+
+	if err := quick.Check(f, quickCfg); err != nil {
+		t.Error(err)
+	}
+}
+
+func TestMarshalCloseMessage(t *testing.T) {
+	var quickCfg = &quick.Config{MaxCountScale: 10}
+	if testing.Short() {
+		quickCfg = nil
+	}
+
+	f := func(m1 CloseMessage) bool {
+		return testMarshal(t, "close", &m1, &CloseMessage{})
+	}
+
+	if err := quick.Check(f, quickCfg); err != nil {
+		t.Error(err)
+	}
+}
+
+type message interface {
+	EncodeXDR(io.Writer) (int, error)
+	DecodeXDR(io.Reader) error
+}
+
+func testMarshal(t *testing.T, prefix string, m1, m2 message) bool {
+	var buf bytes.Buffer
+
+	failed := func(bc []byte) {
+		bs, _ := json.MarshalIndent(m1, "", "  ")
+		ioutil.WriteFile(prefix+"-1.txt", bs, 0644)
+		bs, _ = json.MarshalIndent(m2, "", "  ")
+		ioutil.WriteFile(prefix+"-2.txt", bs, 0644)
+		if len(bc) > 0 {
+			f, _ := os.Create(prefix + "-data.txt")
+			fmt.Fprint(f, hex.Dump(bc))
+			f.Close()
+		}
+	}
+
+	_, err := m1.EncodeXDR(&buf)
+	if err != nil && strings.Contains(err.Error(), "exceeds size") {
+		return true
+	}
+	if err != nil {
+		failed(nil)
+		t.Fatal(err)
+	}
+
+	bc := make([]byte, len(buf.Bytes()))
+	copy(bc, buf.Bytes())
+
+	err = m2.DecodeXDR(&buf)
+	if err != nil {
+		failed(bc)
+		t.Fatal(err)
+	}
+
+	ok := reflect.DeepEqual(m1, m2)
+	if !ok {
+		failed(bc)
+	}
+	return ok
+}