Docs & translation update

Handle sparse files (fixes #245)

.gitattributes

@@ -0,0 +1,9 @@
+# Text files use LF line endings in this repository
+*   text=auto
+
+# Except the dependencies, which we leave alone
+Godeps/**   -text=auto
+
+# Diffs on these files are meaningless
+gui.files.go    -diff
+*.svg           -diff

.gitignore

@@ -1,15 +1,16 @@
 syncthing
+!gui/syncthing
+!Godeps/_workspace/src/github.com/syncthing
 syncthing.exe
 *.tar.gz
 *.zip
 *.asc
-*.sublime*
-.idea/
 .jshintrc
 coverage.out
 files/pidx
 bin
 perfstats*.csv
 coverage.xml
-!gui/scripts/syncthing
-.DS_Store
+syncthing.sig
+RELEASE
+deb

AUTHORS

@@ -1,34 +1,72 @@
 # This is the official list of Syncthing authors for copyright purposes.
 
 Aaron Bieber <qbit@deftly.net>
+Adam Piggott	<aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com>
 Alexander Graf <register-github@alex-graf.de>
 Andrew Dunham <andrew@du.nham.ca>
-Audrius Butkevicius <audrius.butkevicius@gmail.com>
+Antony Male <antony.male@gmail.com>
 Arthur Axel fREW Schmidt <frew@afoolishmanifesto.com> <frioux@gmail.com>
+Audrius Butkevicius <audrius.butkevicius@gmail.com>
+Bart De Vries <devriesb@gmail.com>
+Ben Curthoys <ben@bencurthoys.com>
 Ben Schulz <ueomkail@gmail.com> <uok@users.noreply.github.com>
 Ben Sidhom <bsidhom@gmail.com>
 Brandon Philips <brandon@ifup.org>
+Brendan Long <self@brendanlong.com>
+Brian R. Becker <brbecker@gmail.com>
 Caleb Callaway <enlightened.despot@gmail.com>
+Carsten Hagemann <moter8@gmail.com>
+Cathryne Linenweaver <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
+Chris Howie <me@chrishowie.com>
 Chris Joel <chris@scriptolo.gy>
+Colin Kennedy <moshen.colin@gmail.com>
+Daniel Bergmann <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
 Daniel Martí <mvdan@mvdan.cc>
+Denis A. <denisva@gmail.com>
 Dennis Wilson <dw@risu.io>
 Dominik Heidler <dominik@heidler.eu>
+Elias Jarlebring <jarlebring@gmail.com>
 Emil Hessman <emil@hessman.se>
+Erik Meitner <e.meitner@willystreet.coop>
+Federico Castagnini <federico.castagnini@gmail.com>
 Felix Ableitner <me@nutomic.com>
 Felix Unterpaintner <bigbear2nd@gmail.com>
+Francois-Xavier Gsell <fxgsell@gmail.com>
+Frank Isemann <frank@isemann.name>
 Gilli Sigurdsson <gilli@vx.is>
+Jacek Szafarkiewicz <szafar@linux.pl>
 Jakob Borg <jakob@nym.se>
+Jake Peterson <jake@acogdev.com>
 James Patterson <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
+Jaroslav Malec <dzardacz@gmail.com>
 Jens Diemer <github.com@jensdiemer.de> <git@jensdiemer.de>
 Jochen Voss <voss@seehuhn.de>
+Johan Vromans <jvromans@squirrel.nl>
+Karol Różycki <rozycki.karol@gmail.com>
+Ken'ichi Kamada <kamada@nanohz.org>
 Lode Hoste <zillode@zillode.be>
+Lord Landon Agahnim <lordlandon@gmail.com>
+Marc Laporte <marc@marclaporte.com> <marc@laporte.name>
+Marc Pujol <kilburn@la3.org>
 Marcin Dziadus <dziadus.marcin@gmail.com>
+Mateusz Naściszewski <matin1111@wp.pl>
+Matt Burke <mburke@amplify.com> <burkemw3@gmail.com>
+Michael Jephcote <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
 Michael Tilli <pyfisch@gmail.com>
+Pascal Jungblut <github@pascalj.com> <mail@pascal-jungblut.com>
+Peter Hoeg <peter@speartail.com>
 Philippe Schommers <philippe@schommers.be>
 Phill Luby <phill.luby@newredo.com>
 Piotr Bejda <piotrb10@gmail.com>
 Ryan Sullivan <kayoticsully@gmail.com>
+Sergey Mishin <ralder@yandex.ru>
+Stefan Tatschner <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
+Stefan Kuntz <stefan.github@gmail.com> <Stefan.github@gmail.com>
+Tim Abell <tim@timwise.co.uk>
+Tobias Nygren <tnn@nygren.pp.se>
 Tomas Cerveny <kozec@kozec.com>
 Tully Robinson <tully@tojr.org>
+Tyler Brazier <tyler@tylerbrazier.com>
 Veeti Paananen <veeti.paananen@rojekti.fi>
 Vil Brekin <vilbrekin@gmail.com>
+Yannic A. <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>

CONTRIBUTING.md

@@ -32,95 +32,21 @@ latest info on Transifex.
 ## Contributing Code
 
 Every contribution is welcome. If you want to contribute but are unsure
-where to start, any open issues are fair game! Be prepared for a
-[certain amount of review](https://discourse.syncthing.net/t/733); it's
-all in the name of quality. :) Following the points below will make this
-a smoother process.
+where to start, any open issues are fair game! See the [Contribution
+Guidelines](http://docs.syncthing.net/dev/contributing.html) for the full
+story on committing code.
 
-Individuals making significant and valuable contributions are given
-commit-access to the project. If you make a significant contribution and
-are not considered for commit-access, please contact any of the
-Syncthing core team members.
+## Contributing Documentation
 
-All nontrivial contributions should go through the pull request
-mechanism for internal review. Determining what is "nontrivial" is left
-at the discretion of the contributor.
-
-### Core Team
-
-The Syncthing core team currently consists of the following members;
-
- - Jakob Borg (@calmh)
- - Audrius Butkevicius (@AudriusButkevicius)
-
-## Coding Style
-
-- Follow the conventions laid out in [Effective Go](https://golang.org/doc/effective_go.html)
-  as much as makes sense.
-
-- All text files use Unix line endings.
-
-- Each commit should be `go fmt` clean.
-
-- The commit message subject should be a single short sentence
-  describing the change, starting with a capital letter.
-
-- Commits that resolve an existing issue must include the issue number
-  as `(fixes #123)` at the end of the commit message subject.
-
-- Imports are grouped per `goimports` standard; that is, standard
-  library first, then third party libraries after a blank line.
-
-- A contribution solving a single issue or introducing a single new
-  feature should probably be a single commit based on the current
-  `master` branch. You may be asked to "rebase" or "squash" your pull
-  request to make sure this is the case, especially if there have been
-  amendments during review.
+Updates to the [documentation site](http://docs.syncthing.net/) can be
+made as pull requests on the [documentation
+repository](https://github.com/syncthing/docs).
 
 ## Licensing
 
-All contributions are made under the same GPL license as the rest of the
-project, except documentation, user interface text and translation
+All contributions are made under the same MPLv2 license as the rest of
+the project, except documentation, user interface text and translation
 strings which are licensed under the Creative Commons Attribution 4.0
 International License. You retain the copyright to code you have
 written.
 
-When accepting your first contribution, the maintainer of the project
-will ensure that you are added to the AUTHORS file. You are welcome to
-add yourself as a separate commit in your first pull request.
-
-## Building
-
-[See the documentation](http://discourse.syncthing.net/t/44) on how to
-get started with a build environment.
-
-## Branches
-
- - `master` is the main branch containing good code that will end up in
-   the next release. You should base your work on it. It won't ever be
-   rebased or force-pushed to.
-
- - `vx.y` branches exist to make patch releases on otherwise obsolete
-   minor releases. Should only contain fixes cherry picked from master.
-   Don't base any work on them.
-
- - Other branches are probably topic branches and may be subject to
-   rebasing. Don't base any work on them unless you specifically know
-   otherwise.
-
-## Tags
-
-All releases are tagged semver style as `vx.y.z`. Release tags are
-signed by GPG key BCE524C7.
-
-## Tests
-
-Yes please!
-
-## Documentation
-
-[Over here!](http://discourse.syncthing.net/category/documentation)
-
-## License
-
-GPLv3

Godeps/Godeps.json

@@ -1,41 +1,50 @@
 {
 	"ImportPath": "github.com/syncthing/syncthing",
-	"GoVersion": "go1.4rc1",
+	"GoVersion": "go1.5.1",
 	"Packages": [
 		"./cmd/..."
 	],
 	"Deps": [
-		{
-			"ImportPath": "github.com/AudriusButkevicius/lfu-go",
-			"Rev": "164bcecceb92fd6037f4d18a8d97b495ec6ef669"
-		},
 		{
 			"ImportPath": "github.com/bkaradzic/go-lz4",
-			"Rev": "93a831dcee242be64a9cc9803dda84af25932de7"
+			"Rev": "74ddf82598bc4745b965729e9c6a463bedd33049"
 		},
 		{
-			"ImportPath": "github.com/calmh/logger",
-			"Rev": "f50d32b313bec2933a3e1049f7416a29f3413d29"
+			"ImportPath": "github.com/calmh/du",
+			"Rev": "3c0690cca16228b97741327b1b6781397afbdb24"
 		},
 		{
-			"ImportPath": "github.com/calmh/osext",
-			"Rev": "9bf61584e5f1f172e8766ddc9022d9c401faaa5e"
+			"ImportPath": "github.com/calmh/luhn",
+			"Rev": "0c8388ff95fa92d4094011e5a04fc99dea3d1632"
 		},
 		{
 			"ImportPath": "github.com/calmh/xdr",
-			"Rev": "45c46b7db7ff83b8b9ee09bbd95f36ab50043ece"
+			"Rev": "47c0042d09a827b81ee62497f99e5e0c7f0bd31c"
+		},
+		{
+			"ImportPath": "github.com/golang/snappy",
+			"Rev": "723cc1e459b8eea2dea4583200fd60757d40097a"
 		},
 		{
 			"ImportPath": "github.com/juju/ratelimit",
-			"Rev": "f9f36d11773655c0485207f0ad30dc2655f69d56"
+			"Rev": "772f5c38e468398c4511514f4f6aa9a4185bc0a0"
+		},
+		{
+			"ImportPath": "github.com/kardianos/osext",
+			"Rev": "6e7f843663477789fac7c02def0d0909e969b4e5"
+		},
+		{
+			"ImportPath": "github.com/rcrowley/go-metrics",
+			"Rev": "1ce93efbc8f9c568886b2ef85ce305b2217b3de3"
 		},
 		{
 			"ImportPath": "github.com/syndtr/goleveldb/leveldb",
-			"Rev": "97e257099d2ab9578151ba85e2641e2cd14d3ca8"
+			"Rev": "1a9d62f03ea92815b46fcaab357cfd4df264b1a0"
 		},
 		{
-			"ImportPath": "github.com/syndtr/gosnappy/snappy",
-			"Rev": "ce8acff4829e0c2458a67ead32390ac0a381c862"
+			"ImportPath": "github.com/thejerf/suture",
+			"Comment": "v1.0.1",
+			"Rev": "99c1f2d613756768fc4299acd9dc621e11ed3fd7"
 		},
 		{
 			"ImportPath": "github.com/vitrun/qart/coding",
@@ -51,23 +60,31 @@
 		},
 		{
 			"ImportPath": "golang.org/x/crypto/bcrypt",
-			"Comment": "null-236",
-			"Rev": "69e2a90ed92d03812364aeb947b7068dc42e561e"
+			"Rev": "575fdbe86e5dd89229707ebec0575ce7d088a4a6"
 		},
 		{
 			"ImportPath": "golang.org/x/crypto/blowfish",
-			"Comment": "null-236",
-			"Rev": "69e2a90ed92d03812364aeb947b7068dc42e561e"
+			"Rev": "575fdbe86e5dd89229707ebec0575ce7d088a4a6"
+		},
+		{
+			"ImportPath": "golang.org/x/net/internal/iana",
+			"Rev": "042ba42fa6633b34205efc66ba5719cd3afd8d38"
+		},
+		{
+			"ImportPath": "golang.org/x/net/ipv6",
+			"Rev": "042ba42fa6633b34205efc66ba5719cd3afd8d38"
+		},
+		{
+			"ImportPath": "golang.org/x/net/proxy",
+			"Rev": "042ba42fa6633b34205efc66ba5719cd3afd8d38"
 		},
 		{
 			"ImportPath": "golang.org/x/text/transform",
-			"Comment": "null-112",
-			"Rev": "2f707e0ad64637ca1318279be7201f5ed19c4050"
+			"Rev": "5eb8d4684c4796dd36c74f6452f2c0fa6c79597e"
 		},
 		{
 			"ImportPath": "golang.org/x/text/unicode/norm",
-			"Comment": "null-112",
-			"Rev": "2f707e0ad64637ca1318279be7201f5ed19c4050"
+			"Rev": "5eb8d4684c4796dd36c74f6452f2c0fa6c79597e"
 		}
 	]
 }

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/LICENSE

@@ -1,19 +0,0 @@
-Copyright (C) 2012 Dave Grijalva
-
-Permission is hereby granted, free of charge, to any person obtaining a 
-copy of this software and associated documentation files (the "Software"), 
-to deal in the Software without restriction, including without limitation 
-the rights to use, copy, modify, merge, publish, distribute, sublicense, 
-and/or sell copies of the Software, and to permit persons to whom the 
-Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included 
-in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
-OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
-THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 
-THE SOFTWARE.

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/README.md

@@ -1,19 +0,0 @@
-A simple LFU cache for golang.  Based on the paper [An O(1) algorithm for implementing the LFU cache eviction scheme](http://dhruvbird.com/lfu.pdf).
-
-Usage:
-
-```go
-import "github.com/dgrijalva/lfu-go"
-
-// Make a new thing
-c := lfu.New()
-
-// Set some values
-c.Set("myKey", myValue)
-
-// Retrieve some values
-myValue = c.Get("myKey")
-
-// Evict some values
-c.Evict(1)
-```

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/lfu.go

@@ -1,156 +0,0 @@
-package lfu
-
-import (
-	"container/list"
-	"sync"
-)
-
-type Eviction struct {
-	Key   string
-	Value interface{}
-}
-
-type Cache struct {
-	// If len > UpperBound, cache will automatically evict
-	// down to LowerBound.  If either value is 0, this behavior
-	// is disabled.
-	UpperBound      int
-	LowerBound      int
-	values          map[string]*cacheEntry
-	freqs           *list.List
-	len             int
-	lock            *sync.Mutex
-	EvictionChannel chan<- Eviction
-}
-
-type cacheEntry struct {
-	key      string
-	value    interface{}
-	freqNode *list.Element
-}
-
-type listEntry struct {
-	entries map[*cacheEntry]byte
-	freq    int
-}
-
-func New() *Cache {
-	c := new(Cache)
-	c.values = make(map[string]*cacheEntry)
-	c.freqs = list.New()
-	c.lock = new(sync.Mutex)
-	return c
-}
-
-func (c *Cache) Get(key string) interface{} {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-	if e, ok := c.values[key]; ok {
-		c.increment(e)
-		return e.value
-	}
-	return nil
-}
-
-func (c *Cache) Set(key string, value interface{}) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-	if e, ok := c.values[key]; ok {
-		// value already exists for key.  overwrite
-		e.value = value
-		c.increment(e)
-	} else {
-		// value doesn't exist.  insert
-		e := new(cacheEntry)
-		e.key = key
-		e.value = value
-		c.values[key] = e
-		c.increment(e)
-		c.len++
-		// bounds mgmt
-		if c.UpperBound > 0 && c.LowerBound > 0 {
-			if c.len > c.UpperBound {
-				c.evict(c.len - c.LowerBound)
-			}
-		}
-	}
-}
-
-func (c *Cache) Len() int {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-	return c.len
-}
-
-func (c *Cache) Evict(count int) int {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-	return c.evict(count)
-}
-
-func (c *Cache) evict(count int) int {
-	// No lock here so it can be called
-	// from within the lock (during Set)
-	var evicted int
-	for i := 0; i < count; {
-		if place := c.freqs.Front(); place != nil {
-			for entry, _ := range place.Value.(*listEntry).entries {
-				if i < count {
-					if c.EvictionChannel != nil {
-						c.EvictionChannel <- Eviction{
-							Key:   entry.key,
-							Value: entry.value,
-						}
-					}
-					delete(c.values, entry.key)
-					c.remEntry(place, entry)
-					evicted++
-					c.len--
-					i++
-				}
-			}
-		}
-	}
-	return evicted
-}
-
-func (c *Cache) increment(e *cacheEntry) {
-	currentPlace := e.freqNode
-	var nextFreq int
-	var nextPlace *list.Element
-	if currentPlace == nil {
-		// new entry
-		nextFreq = 1
-		nextPlace = c.freqs.Front()
-	} else {
-		// move up
-		nextFreq = currentPlace.Value.(*listEntry).freq + 1
-		nextPlace = currentPlace.Next()
-	}
-
-	if nextPlace == nil || nextPlace.Value.(*listEntry).freq != nextFreq {
-		// create a new list entry
-		li := new(listEntry)
-		li.freq = nextFreq
-		li.entries = make(map[*cacheEntry]byte)
-		if currentPlace != nil {
-			nextPlace = c.freqs.InsertAfter(li, currentPlace)
-		} else {
-			nextPlace = c.freqs.PushFront(li)
-		}
-	}
-	e.freqNode = nextPlace
-	nextPlace.Value.(*listEntry).entries[e] = 1
-	if currentPlace != nil {
-		// remove from current position
-		c.remEntry(currentPlace, e)
-	}
-}
-
-func (c *Cache) remEntry(place *list.Element, entry *cacheEntry) {
-	entries := place.Value.(*listEntry).entries
-	delete(entries, entry)
-	if len(entries) == 0 {
-		c.freqs.Remove(place)
-	}
-}

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/lfu_test.go

@@ -1,68 +0,0 @@
-package lfu
-
-import (
-	"fmt"
-	"testing"
-)
-
-func TestLFU(t *testing.T) {
-	c := New()
-	c.Set("a", "a")
-	if v := c.Get("a"); v != "a" {
-		t.Errorf("Value was not saved: %v != 'a'", v)
-	}
-	if l := c.Len(); l != 1 {
-		t.Errorf("Length was not updated: %v != 1", l)
-	}
-
-	c.Set("b", "b")
-	if v := c.Get("b"); v != "b" {
-		t.Errorf("Value was not saved: %v != 'b'", v)
-	}
-	if l := c.Len(); l != 2 {
-		t.Errorf("Length was not updated: %v != 2", l)
-	}
-
-	c.Get("a")
-	evicted := c.Evict(1)
-	if v := c.Get("a"); v != "a" {
-		t.Errorf("Value was improperly evicted: %v != 'a'", v)
-	}
-	if v := c.Get("b"); v != nil {
-		t.Errorf("Value was not evicted: %v", v)
-	}
-	if l := c.Len(); l != 1 {
-		t.Errorf("Length was not updated: %v != 1", l)
-	}
-	if evicted != 1 {
-		t.Errorf("Number of evicted items is wrong: %v != 1", evicted)
-	}
-}
-
-func TestBoundsMgmt(t *testing.T) {
-	c := New()
-	c.UpperBound = 10
-	c.LowerBound = 5
-
-	for i := 0; i < 100; i++ {
-		c.Set(fmt.Sprintf("%v", i), i)
-	}
-	if c.Len() > 10 {
-		t.Errorf("Bounds management failed to evict properly: %v", c.Len())
-	}
-}
-
-func TestEviction(t *testing.T) {
-	ch := make(chan Eviction, 1)
-
-	c := New()
-	c.EvictionChannel = ch
-	c.Set("a", "b")
-	c.Evict(1)
-
-	ev := <-ch
-
-	if ev.Key != "a" || ev.Value.(string) != "b" {
-		t.Error("Incorrect item")
-	}
-}

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/.travis.yml

@@ -4,4 +4,6 @@ go:
   - 1.1
   - 1.2
   - 1.3
+  - 1.4
+  - 1.5
   - tip

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/README.md

@@ -4,7 +4,7 @@ go-lz4
 go-lz4 is port of LZ4 lossless compression algorithm to Go. The original C code
 is located at:
 
-https://code.google.com/p/lz4/
+https://github.com/Cyan4973/lz4
 
 Status
 ------

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/fuzz.go

@@ -0,0 +1,23 @@
+// +build gofuzz
+
+package lz4
+
+import "encoding/binary"
+
+func Fuzz(data []byte) int {
+
+	if len(data) < 4 {
+		return 0
+	}
+
+	ln := binary.LittleEndian.Uint32(data)
+	if ln > (1 << 21) {
+		return 0
+	}
+
+	if _, err := Decode(nil, data); err != nil {
+		return 0
+	}
+
+	return 1
+}

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/lz4_test.go

@@ -1,63 +0,0 @@
-package lz4
-
-import (
-	"bytes"
-	"io/ioutil"
-	"testing"
-)
-
-var testfile, _ = ioutil.ReadFile("testdata/pg1661.txt")
-
-func roundtrip(t *testing.T, input []byte) {
-
-	dst, err := Encode(nil, input)
-	if err != nil {
-		t.Errorf("got error during compression: %s", err)
-	}
-
-	output, err := Decode(nil, dst)
-
-	if err != nil {
-		t.Errorf("got error during decompress: %s", err)
-	}
-
-	if !bytes.Equal(output, input) {
-		t.Errorf("roundtrip failed")
-	}
-}
-
-func TestEmpty(t *testing.T) {
-	roundtrip(t, nil)
-}
-
-func TestLengths(t *testing.T) {
-
-	for i := 0; i < 1024; i++ {
-		roundtrip(t, testfile[:i])
-	}
-
-	for i := 1024; i < 4096; i += 23 {
-		roundtrip(t, testfile[:i])
-	}
-}
-
-func TestWords(t *testing.T) {
-	roundtrip(t, testfile)
-}
-
-func BenchmarkLZ4Encode(b *testing.B) {
-	for i := 0; i < b.N; i++ {
-		Encode(nil, testfile)
-	}
-}
-
-func BenchmarkLZ4Decode(b *testing.B) {
-
-	var compressed, _ = Encode(nil, testfile)
-
-	b.ResetTimer()
-
-	for i := 0; i < b.N; i++ {
-		Decode(nil, compressed)
-	}
-}

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/reader.go

@@ -1,194 +1,199 @@
-/*
- * Copyright 2011-2012 Branimir Karadzic. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without modification,
- * are permitted provided that the following conditions are met:
- *
- *    1. Redistributions of source code must retain the above copyright notice, this
- *       list of conditions and the following disclaimer.
- *
- *    2. Redistributions in binary form must reproduce the above copyright notice,
- *       this list of conditions and the following disclaimer in the documentation
- *       and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
- * SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
- * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-package lz4
-
-import (
-	"encoding/binary"
-	"errors"
-	"io"
-)
-
-var (
-	// ErrCorrupt indicates the input was corrupt
-	ErrCorrupt = errors.New("corrupt input")
-)
-
-const (
-	mlBits  = 4
-	mlMask  = (1 << mlBits) - 1
-	runBits = 8 - mlBits
-	runMask = (1 << runBits) - 1
-)
-
-type decoder struct {
-	src  []byte
-	dst  []byte
-	spos uint32
-	dpos uint32
-	ref  uint32
-}
-
-func (d *decoder) readByte() (uint8, error) {
-	if int(d.spos) == len(d.src) {
-		return 0, io.EOF
-	}
-	b := d.src[d.spos]
-	d.spos++
-	return b, nil
-}
-
-func (d *decoder) getLen() (uint32, error) {
-
-	length := uint32(0)
-	ln, err := d.readByte()
-	if err != nil {
-		return 0, ErrCorrupt
-	}
-	for ln == 255 {
-		length += 255
-		ln, err = d.readByte()
-		if err != nil {
-			return 0, ErrCorrupt
-		}
-	}
-	length += uint32(ln)
-
-	return length, nil
-}
-
-func (d *decoder) cp(length, decr uint32) {
-
-	if int(d.ref+length) < int(d.dpos) {
-		copy(d.dst[d.dpos:], d.dst[d.ref:d.ref+length])
-	} else {
-		for ii := uint32(0); ii < length; ii++ {
-			d.dst[d.dpos+ii] = d.dst[d.ref+ii]
-		}
-	}
-	d.dpos += length
-	d.ref += length - decr
-}
-
-func (d *decoder) finish(err error) error {
-	if err == io.EOF {
-		return nil
-	}
-
-	return err
-}
-
-// Decode returns the decoded form of src.  The returned slice may be a
-// subslice of dst if it was large enough to hold the entire decoded block.
-func Decode(dst, src []byte) ([]byte, error) {
-
-	if len(src) < 4 {
-		return nil, ErrCorrupt
-	}
-
-	uncompressedLen := binary.LittleEndian.Uint32(src)
-
-	if uncompressedLen == 0 {
-		return nil, nil
-	}
-
-	if uncompressedLen > MaxInputSize {
-		return nil, ErrTooLarge
-	}
-
-	if dst == nil || len(dst) < int(uncompressedLen) {
-		dst = make([]byte, uncompressedLen)
-	}
-
-	d := decoder{src: src, dst: dst[:uncompressedLen], spos: 4}
-
-	decr := []uint32{0, 3, 2, 3}
-
-	for {
-		code, err := d.readByte()
-		if err != nil {
-			return d.dst, d.finish(err)
-		}
-
-		length := uint32(code >> mlBits)
-		if length == runMask {
-			ln, err := d.getLen()
-			if err != nil {
-				return nil, ErrCorrupt
-			}
-			length += ln
-		}
-
-		if int(d.spos+length) > len(d.src) {
-			return nil, ErrCorrupt
-		}
-
-		for ii := uint32(0); ii < length; ii++ {
-			d.dst[d.dpos+ii] = d.src[d.spos+ii]
-		}
-
-		d.spos += length
-		d.dpos += length
-
-		if int(d.spos) == len(d.src) {
-			return d.dst, nil
-		}
-
-		if int(d.spos+2) >= len(d.src) {
-			return nil, ErrCorrupt
-		}
-
-		back := uint32(d.src[d.spos]) | uint32(d.src[d.spos+1])<<8
-
-		if back > d.dpos {
-			return nil, ErrCorrupt
-		}
-
-		d.spos += 2
-		d.ref = d.dpos - back
-
-		length = uint32(code & mlMask)
-		if length == mlMask {
-			ln, err := d.getLen()
-			if err != nil {
-				return nil, ErrCorrupt
-			}
-			length += ln
-		}
-
-		literal := d.dpos - d.ref
-		if literal < 4 {
-			d.cp(4, decr[literal])
-		} else {
-			length += 4
-		}
-
-		if d.dpos+length > uncompressedLen {
-			return nil, ErrCorrupt
-		}
-
-		d.cp(length, 0)
-	}
-}
+/*
+ * Copyright 2011-2012 Branimir Karadzic. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice, this
+ *       list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above copyright notice,
+ *       this list of conditions and the following disclaimer in the documentation
+ *       and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package lz4
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+)
+
+var (
+	// ErrCorrupt indicates the input was corrupt
+	ErrCorrupt = errors.New("corrupt input")
+)
+
+const (
+	mlBits  = 4
+	mlMask  = (1 << mlBits) - 1
+	runBits = 8 - mlBits
+	runMask = (1 << runBits) - 1
+)
+
+type decoder struct {
+	src  []byte
+	dst  []byte
+	spos uint32
+	dpos uint32
+	ref  uint32
+}
+
+func (d *decoder) readByte() (uint8, error) {
+	if int(d.spos) == len(d.src) {
+		return 0, io.EOF
+	}
+	b := d.src[d.spos]
+	d.spos++
+	return b, nil
+}
+
+func (d *decoder) getLen() (uint32, error) {
+
+	length := uint32(0)
+	ln, err := d.readByte()
+	if err != nil {
+		return 0, ErrCorrupt
+	}
+	for ln == 255 {
+		length += 255
+		ln, err = d.readByte()
+		if err != nil {
+			return 0, ErrCorrupt
+		}
+	}
+	length += uint32(ln)
+
+	return length, nil
+}
+
+func (d *decoder) cp(length, decr uint32) {
+
+	if int(d.ref+length) < int(d.dpos) {
+		copy(d.dst[d.dpos:], d.dst[d.ref:d.ref+length])
+	} else {
+		for ii := uint32(0); ii < length; ii++ {
+			d.dst[d.dpos+ii] = d.dst[d.ref+ii]
+		}
+	}
+	d.dpos += length
+	d.ref += length - decr
+}
+
+func (d *decoder) finish(err error) error {
+	if err == io.EOF {
+		return nil
+	}
+
+	return err
+}
+
+// Decode returns the decoded form of src.  The returned slice may be a
+// subslice of dst if it was large enough to hold the entire decoded block.
+func Decode(dst, src []byte) ([]byte, error) {
+
+	if len(src) < 4 {
+		return nil, ErrCorrupt
+	}
+
+	uncompressedLen := binary.LittleEndian.Uint32(src)
+
+	if uncompressedLen == 0 {
+		return nil, nil
+	}
+
+	if uncompressedLen > MaxInputSize {
+		return nil, ErrTooLarge
+	}
+
+	if dst == nil || len(dst) < int(uncompressedLen) {
+		dst = make([]byte, uncompressedLen)
+	}
+
+	d := decoder{src: src, dst: dst[:uncompressedLen], spos: 4}
+
+	decr := []uint32{0, 3, 2, 3}
+
+	for {
+		code, err := d.readByte()
+		if err != nil {
+			return d.dst, d.finish(err)
+		}
+
+		length := uint32(code >> mlBits)
+		if length == runMask {
+			ln, err := d.getLen()
+			if err != nil {
+				return nil, ErrCorrupt
+			}
+			length += ln
+		}
+
+		if int(d.spos+length) > len(d.src) || int(d.dpos+length) > len(d.dst) {
+			return nil, ErrCorrupt
+		}
+
+		for ii := uint32(0); ii < length; ii++ {
+			d.dst[d.dpos+ii] = d.src[d.spos+ii]
+		}
+
+		d.spos += length
+		d.dpos += length
+
+		if int(d.spos) == len(d.src) {
+			return d.dst, nil
+		}
+
+		if int(d.spos+2) >= len(d.src) {
+			return nil, ErrCorrupt
+		}
+
+		back := uint32(d.src[d.spos]) | uint32(d.src[d.spos+1])<<8
+
+		if back > d.dpos {
+			return nil, ErrCorrupt
+		}
+
+		d.spos += 2
+		d.ref = d.dpos - back
+
+		length = uint32(code & mlMask)
+		if length == mlMask {
+			ln, err := d.getLen()
+			if err != nil {
+				return nil, ErrCorrupt
+			}
+			length += ln
+		}
+
+		literal := d.dpos - d.ref
+
+		if literal < 4 {
+			if int(d.dpos+4) > len(d.dst) {
+				return nil, ErrCorrupt
+			}
+
+			d.cp(4, decr[literal])
+		} else {
+			length += 4
+		}
+
+		if d.dpos+length > uncompressedLen {
+			return nil, ErrCorrupt
+		}
+
+		d.cp(length, 0)
+	}
+}

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/writer.go

@@ -1,188 +1,190 @@
-/*
- * Copyright 2011-2012 Branimir Karadzic. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without modification,
- * are permitted provided that the following conditions are met:
- *
- *    1. Redistributions of source code must retain the above copyright notice, this
- *       list of conditions and the following disclaimer.
- *
- *    2. Redistributions in binary form must reproduce the above copyright notice,
- *       this list of conditions and the following disclaimer in the documentation
- *       and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
- * SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
- * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-package lz4
-
-import "encoding/binary"
-import "errors"
-
-const (
-	minMatch              = 4
-	hashLog               = 17
-	hashTableSize         = 1 << hashLog
-	hashShift             = (minMatch * 8) - hashLog
-	incompressible uint32 = 128
-	uninitHash            = 0x88888888
-
-	// MaxInputSize is the largest buffer than can be compressed in a single block
-	MaxInputSize = 0x7E000000
-)
-
-var (
-	// ErrTooLarge indicates the input buffer was too large
-	ErrTooLarge = errors.New("input too large")
-)
-
-type encoder struct {
-	src       []byte
-	dst       []byte
-	hashTable []uint32
-	pos       uint32
-	anchor    uint32
-	dpos      uint32
-}
-
-// CompressBound returns the maximum length of a lz4 block, given it's uncompressed length
-func CompressBound(isize int) int {
-	if isize > MaxInputSize {
-		return 0
-	}
-	return isize + ((isize) / 255) + 16 + 4
-}
-
-func (e *encoder) writeLiterals(length, mlLen, pos uint32) {
-
-	ln := length
-
-	var code byte
-	if ln > runMask-1 {
-		code = runMask
-	} else {
-		code = byte(ln)
-	}
-
-	if mlLen > mlMask-1 {
-		e.dst[e.dpos] = (code << mlBits) + byte(mlMask)
-	} else {
-		e.dst[e.dpos] = (code << mlBits) + byte(mlLen)
-	}
-	e.dpos++
-
-	if code == runMask {
-		ln -= runMask
-		for ; ln > 254; ln -= 255 {
-			e.dst[e.dpos] = 255
-			e.dpos++
-		}
-
-		e.dst[e.dpos] = byte(ln)
-		e.dpos++
-	}
-
-	for ii := uint32(0); ii < length; ii++ {
-		e.dst[e.dpos+ii] = e.src[pos+ii]
-	}
-
-	e.dpos += length
-}
-
-// Encode returns the encoded form of src.  The returned array may be a
-// sub-slice of dst if it was large enough to hold the entire output.
-func Encode(dst, src []byte) ([]byte, error) {
-
-	if len(src) >= MaxInputSize {
-		return nil, ErrTooLarge
-	}
-
-	if n := CompressBound(len(src)); len(dst) < n {
-		dst = make([]byte, n)
-	}
-
-	e := encoder{src: src, dst: dst, hashTable: make([]uint32, hashTableSize)}
-
-	binary.LittleEndian.PutUint32(dst, uint32(len(src)))
-	e.dpos = 4
-
-	var (
-		step  uint32 = 1
-		limit        = incompressible
-	)
-
-	for {
-		if int(e.pos)+12 >= len(e.src) {
-			e.writeLiterals(uint32(len(e.src))-e.anchor, 0, e.anchor)
-			return e.dst[:e.dpos], nil
-		}
-
-		sequence := uint32(e.src[e.pos+3])<<24 | uint32(e.src[e.pos+2])<<16 | uint32(e.src[e.pos+1])<<8 | uint32(e.src[e.pos+0])
-
-		hash := (sequence * 2654435761) >> hashShift
-		ref := e.hashTable[hash] + uninitHash
-		e.hashTable[hash] = e.pos - uninitHash
-
-		if ((e.pos-ref)>>16) != 0 || uint32(e.src[ref+3])<<24|uint32(e.src[ref+2])<<16|uint32(e.src[ref+1])<<8|uint32(e.src[ref+0]) != sequence {
-			if e.pos-e.anchor > limit {
-				limit <<= 1
-				step += 1 + (step >> 2)
-			}
-			e.pos += step
-			continue
-		}
-
-		if step > 1 {
-			e.hashTable[hash] = ref - uninitHash
-			e.pos -= step - 1
-			step = 1
-			continue
-		}
-		limit = incompressible
-
-		ln := e.pos - e.anchor
-		back := e.pos - ref
-
-		anchor := e.anchor
-
-		e.pos += minMatch
-		ref += minMatch
-		e.anchor = e.pos
-
-		for int(e.pos) < len(e.src)-5 && e.src[e.pos] == e.src[ref] {
-			e.pos++
-			ref++
-		}
-
-		mlLen := e.pos - e.anchor
-
-		e.writeLiterals(ln, mlLen, anchor)
-		e.dst[e.dpos] = uint8(back)
-		e.dst[e.dpos+1] = uint8(back >> 8)
-		e.dpos += 2
-
-		if mlLen > mlMask-1 {
-			mlLen -= mlMask
-			for mlLen > 254 {
-				mlLen -= 255
-
-				e.dst[e.dpos] = 255
-				e.dpos++
-			}
-
-			e.dst[e.dpos] = byte(mlLen)
-			e.dpos++
-		}
-
-		e.anchor = e.pos
-	}
-}
+/*
+ * Copyright 2011-2012 Branimir Karadzic. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice, this
+ *       list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above copyright notice,
+ *       this list of conditions and the following disclaimer in the documentation
+ *       and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package lz4
+
+import (
+	"encoding/binary"
+	"errors"
+)
+
+const (
+	minMatch              = 4
+	hashLog               = 17
+	hashTableSize         = 1 << hashLog
+	hashShift             = (minMatch * 8) - hashLog
+	incompressible uint32 = 128
+	uninitHash            = 0x88888888
+
+	// MaxInputSize is the largest buffer than can be compressed in a single block
+	MaxInputSize = 0x7E000000
+)
+
+var (
+	// ErrTooLarge indicates the input buffer was too large
+	ErrTooLarge = errors.New("input too large")
+)
+
+type encoder struct {
+	src       []byte
+	dst       []byte
+	hashTable []uint32
+	pos       uint32
+	anchor    uint32
+	dpos      uint32
+}
+
+// CompressBound returns the maximum length of a lz4 block, given it's uncompressed length
+func CompressBound(isize int) int {
+	if isize > MaxInputSize {
+		return 0
+	}
+	return isize + ((isize) / 255) + 16 + 4
+}
+
+func (e *encoder) writeLiterals(length, mlLen, pos uint32) {
+
+	ln := length
+
+	var code byte
+	if ln > runMask-1 {
+		code = runMask
+	} else {
+		code = byte(ln)
+	}
+
+	if mlLen > mlMask-1 {
+		e.dst[e.dpos] = (code << mlBits) + byte(mlMask)
+	} else {
+		e.dst[e.dpos] = (code << mlBits) + byte(mlLen)
+	}
+	e.dpos++
+
+	if code == runMask {
+		ln -= runMask
+		for ; ln > 254; ln -= 255 {
+			e.dst[e.dpos] = 255
+			e.dpos++
+		}
+
+		e.dst[e.dpos] = byte(ln)
+		e.dpos++
+	}
+
+	for ii := uint32(0); ii < length; ii++ {
+		e.dst[e.dpos+ii] = e.src[pos+ii]
+	}
+
+	e.dpos += length
+}
+
+// Encode returns the encoded form of src.  The returned array may be a
+// sub-slice of dst if it was large enough to hold the entire output.
+func Encode(dst, src []byte) ([]byte, error) {
+
+	if len(src) >= MaxInputSize {
+		return nil, ErrTooLarge
+	}
+
+	if n := CompressBound(len(src)); len(dst) < n {
+		dst = make([]byte, n)
+	}
+
+	e := encoder{src: src, dst: dst, hashTable: make([]uint32, hashTableSize)}
+
+	binary.LittleEndian.PutUint32(dst, uint32(len(src)))
+	e.dpos = 4
+
+	var (
+		step  uint32 = 1
+		limit        = incompressible
+	)
+
+	for {
+		if int(e.pos)+12 >= len(e.src) {
+			e.writeLiterals(uint32(len(e.src))-e.anchor, 0, e.anchor)
+			return e.dst[:e.dpos], nil
+		}
+
+		sequence := uint32(e.src[e.pos+3])<<24 | uint32(e.src[e.pos+2])<<16 | uint32(e.src[e.pos+1])<<8 | uint32(e.src[e.pos+0])
+
+		hash := (sequence * 2654435761) >> hashShift
+		ref := e.hashTable[hash] + uninitHash
+		e.hashTable[hash] = e.pos - uninitHash
+
+		if ((e.pos-ref)>>16) != 0 || uint32(e.src[ref+3])<<24|uint32(e.src[ref+2])<<16|uint32(e.src[ref+1])<<8|uint32(e.src[ref+0]) != sequence {
+			if e.pos-e.anchor > limit {
+				limit <<= 1
+				step += 1 + (step >> 2)
+			}
+			e.pos += step
+			continue
+		}
+
+		if step > 1 {
+			e.hashTable[hash] = ref - uninitHash
+			e.pos -= step - 1
+			step = 1
+			continue
+		}
+		limit = incompressible
+
+		ln := e.pos - e.anchor
+		back := e.pos - ref
+
+		anchor := e.anchor
+
+		e.pos += minMatch
+		ref += minMatch
+		e.anchor = e.pos
+
+		for int(e.pos) < len(e.src)-5 && e.src[e.pos] == e.src[ref] {
+			e.pos++
+			ref++
+		}
+
+		mlLen := e.pos - e.anchor
+
+		e.writeLiterals(ln, mlLen, anchor)
+		e.dst[e.dpos] = uint8(back)
+		e.dst[e.dpos+1] = uint8(back >> 8)
+		e.dpos += 2
+
+		if mlLen > mlMask-1 {
+			mlLen -= mlMask
+			for mlLen > 254 {
+				mlLen -= 255
+
+				e.dst[e.dpos] = 255
+				e.dpos++
+			}
+
+			e.dst[e.dpos] = byte(mlLen)
+			e.dpos++
+		}
+
+		e.anchor = e.pos
+	}
+}

Godeps/_workspace/src/github.com/calmh/du/LICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org>

Godeps/_workspace/src/github.com/calmh/du/README.md

@@ -0,0 +1,14 @@
+du
+==
+
+Get total and available disk space on a given volume.
+
+Documentation
+-------------
+
+http://godoc.org/github.com/calmh/du
+
+License
+-------
+
+Public Domain

Godeps/_workspace/src/github.com/calmh/du/cmd/du/main.go

@@ -0,0 +1,21 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/calmh/du"
+)
+
+var KB = int64(1024)
+
+func main() {
+	usage, err := du.Get(os.Args[1])
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println("Free:", usage.FreeBytes/(KB*KB), "MiB")
+	fmt.Println("Available:", usage.AvailBytes/(KB*KB), "MiB")
+	fmt.Println("Size:", usage.TotalBytes/(KB*KB), "MiB")
+}

Godeps/_workspace/src/github.com/calmh/du/diskusage.go

@@ -0,0 +1,8 @@
+package du
+
+// Usage holds information about total and available storage on a volume.
+type Usage struct {
+	TotalBytes int64 // Size of volume
+	FreeBytes  int64 // Unused size
+	AvailBytes int64 // Available to a non-privileged user
+}

Godeps/_workspace/src/github.com/calmh/du/diskusage_posix.go

@@ -0,0 +1,24 @@
+// +build !windows,!netbsd,!openbsd,!solaris
+
+package du
+
+import (
+	"path/filepath"
+	"syscall"
+)
+
+// Get returns the Usage of a given path, or an error if usage data is
+// unavailable.
+func Get(path string) (Usage, error) {
+	var stat syscall.Statfs_t
+	err := syscall.Statfs(filepath.Clean(path), &stat)
+	if err != nil {
+		return Usage{}, err
+	}
+	u := Usage{
+		FreeBytes:  int64(stat.Bfree) * int64(stat.Bsize),
+		TotalBytes: int64(stat.Blocks) * int64(stat.Bsize),
+		AvailBytes: int64(stat.Bavail) * int64(stat.Bsize),
+	}
+	return u, nil
+}

Godeps/_workspace/src/github.com/calmh/du/diskusage_unsupported.go

@@ -0,0 +1,13 @@
+// +build netbsd openbsd solaris
+
+package du
+
+import "errors"
+
+var ErrUnsupported = errors.New("unsupported platform")
+
+// Get returns the Usage of a given path, or an error if usage data is
+// unavailable.
+func Get(path string) (Usage, error) {
+	return Usage{}, ErrUnsupported
+}

Godeps/_workspace/src/github.com/calmh/du/diskusage_windows.go

@@ -0,0 +1,27 @@
+package du
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+// Get returns the Usage of a given path, or an error if usage data is
+// unavailable.
+func Get(path string) (Usage, error) {
+	h := syscall.MustLoadDLL("kernel32.dll")
+	c := h.MustFindProc("GetDiskFreeSpaceExW")
+
+	var u Usage
+
+	ret, _, err := c.Call(
+		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
+		uintptr(unsafe.Pointer(&u.FreeBytes)),
+		uintptr(unsafe.Pointer(&u.TotalBytes)),
+		uintptr(unsafe.Pointer(&u.AvailBytes)))
+
+	if ret == 0 {
+		return u, err
+	}
+
+	return u, nil
+}

Godeps/_workspace/src/github.com/calmh/logger/README.md

@@ -1,15 +0,0 @@
-logger
-======
-
-A small wrapper around `log` to provide log levels.
-
-Documentation
--------------
-
-http://godoc.org/github.com/calmh/logger
-
-License
--------
-
-MIT
-

Godeps/_workspace/src/github.com/calmh/logger/logger.go

@@ -1,160 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-// Package logger implements a standardized logger with callback functionality
-package logger
-
-import (
-	"fmt"
-	"log"
-	"os"
-	"strings"
-	"sync"
-)
-
-type LogLevel int
-
-const (
-	LevelDebug LogLevel = iota
-	LevelInfo
-	LevelOK
-	LevelWarn
-	LevelFatal
-	NumLevels
-)
-
-// A MessageHandler is called with the log level and message text.
-type MessageHandler func(l LogLevel, msg string)
-
-type Logger struct {
-	logger   *log.Logger
-	handlers [NumLevels][]MessageHandler
-	mut      sync.Mutex
-}
-
-// The default logger logs to standard output with a time prefix.
-var DefaultLogger = New()
-
-func New() *Logger {
-	return &Logger{
-		logger: log.New(os.Stdout, "", log.Ltime),
-	}
-}
-
-// AddHandler registers a new MessageHandler to receive messages with the
-// specified log level or above.
-func (l *Logger) AddHandler(level LogLevel, h MessageHandler) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	l.handlers[level] = append(l.handlers[level], h)
-}
-
-// See log.SetFlags
-func (l *Logger) SetFlags(flag int) {
-	l.logger.SetFlags(flag)
-}
-
-// See log.SetPrefix
-func (l *Logger) SetPrefix(prefix string) {
-	l.logger.SetPrefix(prefix)
-}
-
-func (l *Logger) callHandlers(level LogLevel, s string) {
-	for _, h := range l.handlers[level] {
-		h(level, strings.TrimSpace(s))
-	}
-}
-
-// Debugln logs a line with a DEBUG prefix.
-func (l *Logger) Debugln(vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintln(vals...)
-	l.logger.Output(2, "DEBUG: "+s)
-	l.callHandlers(LevelDebug, s)
-}
-
-// Debugf logs a formatted line with a DEBUG prefix.
-func (l *Logger) Debugf(format string, vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintf(format, vals...)
-	l.logger.Output(2, "DEBUG: "+s)
-	l.callHandlers(LevelDebug, s)
-}
-
-// Infoln logs a line with an INFO prefix.
-func (l *Logger) Infoln(vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintln(vals...)
-	l.logger.Output(2, "INFO: "+s)
-	l.callHandlers(LevelInfo, s)
-}
-
-// Infof logs a formatted line with an INFO prefix.
-func (l *Logger) Infof(format string, vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintf(format, vals...)
-	l.logger.Output(2, "INFO: "+s)
-	l.callHandlers(LevelInfo, s)
-}
-
-// Okln logs a line with an OK prefix.
-func (l *Logger) Okln(vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintln(vals...)
-	l.logger.Output(2, "OK: "+s)
-	l.callHandlers(LevelOK, s)
-}
-
-// Okf logs a formatted line with an OK prefix.
-func (l *Logger) Okf(format string, vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintf(format, vals...)
-	l.logger.Output(2, "OK: "+s)
-	l.callHandlers(LevelOK, s)
-}
-
-// Warnln logs a formatted line with a WARNING prefix.
-func (l *Logger) Warnln(vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintln(vals...)
-	l.logger.Output(2, "WARNING: "+s)
-	l.callHandlers(LevelWarn, s)
-}
-
-// Warnf logs a formatted line with a WARNING prefix.
-func (l *Logger) Warnf(format string, vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintf(format, vals...)
-	l.logger.Output(2, "WARNING: "+s)
-	l.callHandlers(LevelWarn, s)
-}
-
-// Fatalln logs a line with a FATAL prefix and exits the process with exit
-// code 1.
-func (l *Logger) Fatalln(vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintln(vals...)
-	l.logger.Output(2, "FATAL: "+s)
-	l.callHandlers(LevelFatal, s)
-	os.Exit(1)
-}
-
-// Fatalf logs a formatted line with a FATAL prefix and exits the process with
-// exit code 1.
-func (l *Logger) Fatalf(format string, vals ...interface{}) {
-	l.mut.Lock()
-	defer l.mut.Unlock()
-	s := fmt.Sprintf(format, vals...)
-	l.logger.Output(2, "FATAL: "+s)
-	l.callHandlers(LevelFatal, s)
-	os.Exit(1)
-}

Godeps/_workspace/src/github.com/calmh/logger/logger_test.go

@@ -1,58 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-package logger
-
-import (
-	"strings"
-	"testing"
-)
-
-func TestAPI(t *testing.T) {
-	l := New()
-	l.SetFlags(0)
-	l.SetPrefix("testing")
-
-	debug := 0
-	l.AddHandler(LevelDebug, checkFunc(t, LevelDebug, "test 0", &debug))
-	info := 0
-	l.AddHandler(LevelInfo, checkFunc(t, LevelInfo, "test 1", &info))
-	warn := 0
-	l.AddHandler(LevelWarn, checkFunc(t, LevelWarn, "test 2", &warn))
-	ok := 0
-	l.AddHandler(LevelOK, checkFunc(t, LevelOK, "test 3", &ok))
-
-	l.Debugf("test %d", 0)
-	l.Debugln("test", 0)
-	l.Infof("test %d", 1)
-	l.Infoln("test", 1)
-	l.Warnf("test %d", 2)
-	l.Warnln("test", 2)
-	l.Okf("test %d", 3)
-	l.Okln("test", 3)
-
-	if debug != 2 {
-		t.Errorf("Debug handler called %d != 2 times", debug)
-	}
-	if info != 2 {
-		t.Errorf("Info handler called %d != 2 times", info)
-	}
-	if warn != 2 {
-		t.Errorf("Warn handler called %d != 2 times", warn)
-	}
-	if ok != 2 {
-		t.Errorf("Ok handler called %d != 2 times", ok)
-	}
-}
-
-func checkFunc(t *testing.T, expectl LogLevel, expectmsg string, counter *int) func(LogLevel, string) {
-	return func(l LogLevel, msg string) {
-		*counter++
-		if l != expectl {
-			t.Errorf("Incorrect message level %d != %d", l, expectl)
-		}
-		if !strings.HasSuffix(msg, expectmsg) {
-			t.Errorf("%q does not end with %q", msg, expectmsg)
-		}
-	}
-}

Godeps/_workspace/src/github.com/calmh/luhn/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2014 Jakob Borg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+- The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Godeps/_workspace/src/github.com/calmh/luhn/luhn.go

@@ -1,17 +1,4 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// Copyright (C) 2014 Jakob Borg
 
 // Package luhn generates and validates Luhn mod N check digits.
 package luhn

Godeps/_workspace/src/github.com/calmh/osext/LICENSE

@@ -1,20 +0,0 @@
-Copyright (c) 2012 Daniel Theophanes
-
-This software is provided 'as-is', without any express or implied
-warranty. In no event will the authors be held liable for any damages
-arising from the use of this software.
-
-Permission is granted to anyone to use this software for any purpose,
-including commercial applications, and to alter it and redistribute it
-freely, subject to the following restrictions:
-
-   1. The origin of this software must not be misrepresented; you must not
-   claim that you wrote the original software. If you use this software
-   in a product, an acknowledgment in the product documentation would be
-   appreciated but is not required.
-
-   2. Altered source versions must be plainly marked as such, and must not be
-   misrepresented as being the original software.
-
-   3. This notice may not be removed or altered from any source
-   distribution.

Godeps/_workspace/src/github.com/calmh/osext/osext_test.go

@@ -1,79 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build darwin linux freebsd netbsd windows
-
-package osext
-
-import (
-	"fmt"
-	"os"
-	oexec "os/exec"
-	"path/filepath"
-	"runtime"
-	"testing"
-)
-
-const execPath_EnvVar = "OSTEST_OUTPUT_EXECPATH"
-
-func TestExecPath(t *testing.T) {
-	ep, err := Executable()
-	if err != nil {
-		t.Fatalf("ExecPath failed: %v", err)
-	}
-	// we want fn to be of the form "dir/prog"
-	dir := filepath.Dir(filepath.Dir(ep))
-	fn, err := filepath.Rel(dir, ep)
-	if err != nil {
-		t.Fatalf("filepath.Rel: %v", err)
-	}
-	cmd := &oexec.Cmd{}
-	// make child start with a relative program path
-	cmd.Dir = dir
-	cmd.Path = fn
-	// forge argv[0] for child, so that we can verify we could correctly
-	// get real path of the executable without influenced by argv[0].
-	cmd.Args = []string{"-", "-test.run=XXXX"}
-	cmd.Env = []string{fmt.Sprintf("%s=1", execPath_EnvVar)}
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Fatalf("exec(self) failed: %v", err)
-	}
-	outs := string(out)
-	if !filepath.IsAbs(outs) {
-		t.Fatalf("Child returned %q, want an absolute path", out)
-	}
-	if !sameFile(outs, ep) {
-		t.Fatalf("Child returned %q, not the same file as %q", out, ep)
-	}
-}
-
-func sameFile(fn1, fn2 string) bool {
-	fi1, err := os.Stat(fn1)
-	if err != nil {
-		return false
-	}
-	fi2, err := os.Stat(fn2)
-	if err != nil {
-		return false
-	}
-	return os.SameFile(fi1, fi2)
-}
-
-func init() {
-	if e := os.Getenv(execPath_EnvVar); e != "" {
-		// first chdir to another path
-		dir := "/"
-		if runtime.GOOS == "windows" {
-			dir = filepath.VolumeName(".")
-		}
-		os.Chdir(dir)
-		if ep, err := Executable(); err != nil {
-			fmt.Fprint(os.Stderr, "ERROR: ", err)
-		} else {
-			fmt.Fprint(os.Stderr, ep)
-		}
-		os.Exit(0)
-	}
-}

Godeps/_workspace/src/github.com/calmh/xdr/.travis.yml

@@ -4,7 +4,7 @@ go:
 
 install:
 - export PATH=$PATH:$HOME/gopath/bin
-- go get code.google.com/p/go.tools/cmd/cover
+- go get golang.org/x/tools/cover
 - go get github.com/mattn/goveralls
 
 script:

Godeps/_workspace/src/github.com/calmh/xdr/README.md

@@ -1,7 +1,7 @@
 xdr
 ===
 
-[![Build Status](https://img.shields.io/travis/calmh/xdr.svg?style=flat)](https://travis-ci.org/calmh/xdr)
+[![Build Status](https://img.shields.io/circleci/project/calmh/xdr.svg?style=flat-square)](https://circleci.com/gh/calmh/xdr)
 [![Coverage Status](https://img.shields.io/coveralls/calmh/xdr.svg?style=flat)](https://coveralls.io/r/calmh/xdr?branch=master)
 [![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat)](http://godoc.org/github.com/calmh/xdr)
 [![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://opensource.org/licenses/MIT)

Godeps/_workspace/src/github.com/calmh/xdr/bench_test.go

@@ -1,117 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-package xdr_test
-
-import (
-	"io"
-	"io/ioutil"
-	"testing"
-
-	"github.com/calmh/xdr"
-)
-
-type XDRBenchStruct struct {
-	I1  uint64
-	I2  uint32
-	I3  uint16
-	I4  uint8
-	Bs0 []byte // max:128
-	Bs1 []byte
-	S0  string // max:128
-	S1  string
-}
-
-var res []byte // no to be optimized away
-var s = XDRBenchStruct{
-	I1:  42,
-	I2:  43,
-	I3:  44,
-	I4:  45,
-	Bs0: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18},
-	Bs1: []byte{11, 12, 13, 14, 15, 16, 17, 18, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
-	S0:  "Hello World! String one.",
-	S1:  "Hello World! String two.",
-}
-var e []byte
-
-func init() {
-	e, _ = s.MarshalXDR()
-}
-
-func BenchmarkThisMarshal(b *testing.B) {
-	for i := 0; i < b.N; i++ {
-		res, _ = s.MarshalXDR()
-	}
-}
-
-func BenchmarkThisUnmarshal(b *testing.B) {
-	var t XDRBenchStruct
-	for i := 0; i < b.N; i++ {
-		err := t.UnmarshalXDR(e)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func BenchmarkThisEncode(b *testing.B) {
-	for i := 0; i < b.N; i++ {
-		_, err := s.EncodeXDR(ioutil.Discard)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func BenchmarkThisEncoder(b *testing.B) {
-	w := xdr.NewWriter(ioutil.Discard)
-	for i := 0; i < b.N; i++ {
-		_, err := s.encodeXDR(w)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-type repeatReader struct {
-	data []byte
-}
-
-func (r *repeatReader) Read(bs []byte) (n int, err error) {
-	if len(bs) > len(r.data) {
-		err = io.EOF
-	}
-	n = copy(bs, r.data)
-	r.data = r.data[n:]
-	return n, err
-}
-
-func (r *repeatReader) Reset(bs []byte) {
-	r.data = bs
-}
-
-func BenchmarkThisDecode(b *testing.B) {
-	rr := &repeatReader{e}
-	var t XDRBenchStruct
-	for i := 0; i < b.N; i++ {
-		err := t.DecodeXDR(rr)
-		if err != nil {
-			b.Fatal(err)
-		}
-		rr.Reset(e)
-	}
-}
-
-func BenchmarkThisDecoder(b *testing.B) {
-	rr := &repeatReader{e}
-	r := xdr.NewReader(rr)
-	var t XDRBenchStruct
-	for i := 0; i < b.N; i++ {
-		err := t.decodeXDR(r)
-		if err != nil {
-			b.Fatal(err)
-		}
-		rr.Reset(e)
-	}
-}

Godeps/_workspace/src/github.com/calmh/xdr/bench_xdr_test.go

@@ -1,199 +0,0 @@
-// ************************************************************
-// This file is automatically generated by genxdr. Do not edit.
-// ************************************************************
-
-package xdr_test
-
-import (
-	"bytes"
-	"io"
-
-	"github.com/calmh/xdr"
-)
-
-/*
-
-XDRBenchStruct Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                                                               |
-+                         I1 (64 bits)                          +
-|                                                               |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                              I2                               |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|            0x0000             |              I3               |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             uint8                             |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                         Length of Bs0                         |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                     Bs0 (variable length)                     \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                         Length of Bs1                         |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                     Bs1 (variable length)                     \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                         Length of S0                          |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                     S0 (variable length)                      \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                         Length of S1                          |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                     S1 (variable length)                      \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct XDRBenchStruct {
-	unsigned hyper I1;
-	unsigned int I2;
-	unsigned int I3;
-	uint8 I4;
-	opaque Bs0<128>;
-	opaque Bs1<>;
-	string S0<128>;
-	string S1<>;
-}
-
-*/
-
-func (o XDRBenchStruct) EncodeXDR(w io.Writer) (int, error) {
-	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
-}
-
-func (o XDRBenchStruct) MarshalXDR() ([]byte, error) {
-	return o.AppendXDR(make([]byte, 0, 128))
-}
-
-func (o XDRBenchStruct) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o XDRBenchStruct) AppendXDR(bs []byte) ([]byte, error) {
-	var aw = xdr.AppendWriter(bs)
-	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
-	return []byte(aw), err
-}
-
-func (o XDRBenchStruct) encodeXDR(xw *xdr.Writer) (int, error) {
-	xw.WriteUint64(o.I1)
-	xw.WriteUint32(o.I2)
-	xw.WriteUint16(o.I3)
-	xw.WriteUint8(o.I4)
-	if l := len(o.Bs0); l > 128 {
-		return xw.Tot(), xdr.ElementSizeExceeded("Bs0", l, 128)
-	}
-	xw.WriteBytes(o.Bs0)
-	xw.WriteBytes(o.Bs1)
-	if l := len(o.S0); l > 128 {
-		return xw.Tot(), xdr.ElementSizeExceeded("S0", l, 128)
-	}
-	xw.WriteString(o.S0)
-	xw.WriteString(o.S1)
-	return xw.Tot(), xw.Error()
-}
-
-func (o *XDRBenchStruct) DecodeXDR(r io.Reader) error {
-	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
-}
-
-func (o *XDRBenchStruct) UnmarshalXDR(bs []byte) error {
-	var br = bytes.NewReader(bs)
-	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
-}
-
-func (o *XDRBenchStruct) decodeXDR(xr *xdr.Reader) error {
-	o.I1 = xr.ReadUint64()
-	o.I2 = xr.ReadUint32()
-	o.I3 = xr.ReadUint16()
-	o.I4 = xr.ReadUint8()
-	o.Bs0 = xr.ReadBytesMax(128)
-	o.Bs1 = xr.ReadBytes()
-	o.S0 = xr.ReadStringMax(128)
-	o.S1 = xr.ReadString()
-	return xr.Error()
-}
-
-/*
-
-repeatReader Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                        Length of data                         |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                    data (variable length)                     \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct repeatReader {
-	opaque data<>;
-}
-
-*/
-
-func (o repeatReader) EncodeXDR(w io.Writer) (int, error) {
-	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
-}
-
-func (o repeatReader) MarshalXDR() ([]byte, error) {
-	return o.AppendXDR(make([]byte, 0, 128))
-}
-
-func (o repeatReader) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o repeatReader) AppendXDR(bs []byte) ([]byte, error) {
-	var aw = xdr.AppendWriter(bs)
-	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
-	return []byte(aw), err
-}
-
-func (o repeatReader) encodeXDR(xw *xdr.Writer) (int, error) {
-	xw.WriteBytes(o.data)
-	return xw.Tot(), xw.Error()
-}
-
-func (o *repeatReader) DecodeXDR(r io.Reader) error {
-	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
-}
-
-func (o *repeatReader) UnmarshalXDR(bs []byte) error {
-	var br = bytes.NewReader(bs)
-	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
-}
-
-func (o *repeatReader) decodeXDR(xr *xdr.Reader) error {
-	o.data = xr.ReadBytes()
-	return xr.Error()
-}

Godeps/_workspace/src/github.com/calmh/xdr/circle.yml

@@ -0,0 +1,3 @@
+dependencies:
+  post:
+      - ./generate.sh

Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go

@@ -28,6 +28,7 @@ type fieldInfo struct {
 	Encoder   string // the encoder name, i.e. "Uint64" for Read/WriteUint64
 	Convert   string // what to convert to when encoding, i.e. "uint64"
 	Max       int    // max size for slices and strings
+	Submax    int    // max size for strings inside slices
 }
 
 type structInfo struct {
@@ -52,7 +53,7 @@ import (
 var encodeTpl = template.Must(template.New("encoder").Parse(`
 func (o {{.TypeName}}) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }//+n
 
 func (o {{.TypeName}}) MarshalXDR() ([]byte, error) {
@@ -70,11 +71,11 @@ func (o {{.TypeName}}) MustMarshalXDR() []byte {
 func (o {{.TypeName}}) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }//+n
 
-func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o {{.TypeName}}) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	{{range $fieldInfo := .Fields}}
 		{{if not $fieldInfo.IsSlice}}
 			{{if ne $fieldInfo.Convert ""}}
@@ -87,7 +88,7 @@ func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
 				{{end}}
 				xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}})
 			{{else}}
-				_, err := o.{{$fieldInfo.Name}}.encodeXDR(xw)
+				_, err := o.{{$fieldInfo.Name}}.EncodeXDRInto(xw)
 				if err != nil {
 					return xw.Tot(), err
 				}
@@ -105,7 +106,7 @@ func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
 			{{else if $fieldInfo.IsBasic}}
 				xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}}[i])
 			{{else}}
-				_, err := o.{{$fieldInfo.Name}}[i].encodeXDR(xw)
+				_, err := o.{{$fieldInfo.Name}}[i].EncodeXDRInto(xw)
 				if err != nil {
 					return xw.Tot(), err
 				}
@@ -118,16 +119,16 @@ func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *{{.TypeName}}) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }//+n
 
 func (o *{{.TypeName}}) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }//+n
 
-func (o *{{.TypeName}}) decodeXDR(xr *xdr.Reader) error {
+func (o *{{.TypeName}}) DecodeXDRFrom(xr *xdr.Reader) error {
 	{{range $fieldInfo := .Fields}}
 		{{if not $fieldInfo.IsSlice}}
 			{{if ne $fieldInfo.Convert ""}}
@@ -139,10 +140,13 @@ func (o *{{.TypeName}}) decodeXDR(xr *xdr.Reader) error {
 					o.{{$fieldInfo.Name}} = xr.Read{{$fieldInfo.Encoder}}()
 				{{end}}
 			{{else}}
-				(&o.{{$fieldInfo.Name}}).decodeXDR(xr)
+				(&o.{{$fieldInfo.Name}}).DecodeXDRFrom(xr)
 			{{end}}
 		{{else}}
 			_{{$fieldInfo.Name}}Size := int(xr.ReadUint32())
+			if _{{$fieldInfo.Name}}Size < 0 {
+				return xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", _{{$fieldInfo.Name}}Size, {{$fieldInfo.Max}})
+			}
 			{{if ge $fieldInfo.Max 1}}
 				if _{{$fieldInfo.Name}}Size > {{$fieldInfo.Max}} {
 					return xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", _{{$fieldInfo.Name}}Size, {{$fieldInfo.Max}})
@@ -153,9 +157,13 @@ func (o *{{.TypeName}}) decodeXDR(xr *xdr.Reader) error {
 				{{if ne $fieldInfo.Convert ""}}
 					o.{{$fieldInfo.Name}}[i] = {{$fieldInfo.FieldType}}(xr.Read{{$fieldInfo.Encoder}}())
 				{{else if $fieldInfo.IsBasic}}
-					o.{{$fieldInfo.Name}}[i] = xr.Read{{$fieldInfo.Encoder}}()
+					{{if ge $fieldInfo.Submax 1}}
+						o.{{$fieldInfo.Name}}[i] = xr.Read{{$fieldInfo.Encoder}}Max({{$fieldInfo.Submax}})
+					{{else}}
+						o.{{$fieldInfo.Name}}[i] = xr.Read{{$fieldInfo.Encoder}}()
+					{{end}}
 				{{else}}
-					(&o.{{$fieldInfo.Name}}[i]).decodeXDR(xr)
+					(&o.{{$fieldInfo.Name}}[i]).DecodeXDRFrom(xr)
 				{{end}}
 			}
 		{{end}}
@@ -163,7 +171,7 @@ func (o *{{.TypeName}}) decodeXDR(xr *xdr.Reader) error {
 	return xr.Error()
 }`))
 
-var maxRe = regexp.MustCompile(`\Wmax:(\d+)`)
+var maxRe = regexp.MustCompile(`(?:\Wmax:)(\d+)(?:\s*,\s*(\d+))?`)
 
 type typeSet struct {
 	Type    string
@@ -195,11 +203,15 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 		}
 
 		fn := sf.Names[0].Name
-		var max = 0
+		var max1, max2 int
 		if sf.Comment != nil {
 			c := sf.Comment.List[0].Text
-			if m := maxRe.FindStringSubmatch(c); m != nil {
-				max, _ = strconv.Atoi(m[1])
+			m := maxRe.FindStringSubmatch(c)
+			if len(m) >= 2 {
+				max1, _ = strconv.Atoi(m[1])
+			}
+			if len(m) >= 3 {
+				max2, _ = strconv.Atoi(m[2])
 			}
 			if strings.Contains(c, "noencode") {
 				continue
@@ -217,14 +229,16 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 					FieldType: tn,
 					Encoder:   enc.Encoder,
 					Convert:   enc.Type,
-					Max:       max,
+					Max:       max1,
+					Submax:    max2,
 				}
 			} else {
 				f = fieldInfo{
 					Name:      fn,
 					IsBasic:   false,
 					FieldType: tn,
-					Max:       max,
+					Max:       max1,
+					Submax:    max2,
 				}
 			}
 
@@ -242,7 +256,8 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 					FieldType: tn,
 					Encoder:   enc.Encoder,
 					Convert:   enc.Type,
-					Max:       max,
+					Max:       max1,
+					Submax:    max2,
 				}
 			} else if enc, ok := xdrEncoders[tn]; ok {
 				f = fieldInfo{
@@ -252,17 +267,26 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 					FieldType: tn,
 					Encoder:   enc.Encoder,
 					Convert:   enc.Type,
-					Max:       max,
+					Max:       max1,
+					Submax:    max2,
 				}
 			} else {
 				f = fieldInfo{
 					Name:      fn,
-					IsBasic:   false,
 					IsSlice:   true,
 					FieldType: tn,
-					Max:       max,
+					Max:       max1,
+					Submax:    max2,
 				}
 			}
+
+		case *ast.SelectorExpr:
+			f = fieldInfo{
+				Name:      fn,
+				FieldType: ft.Sel.Name,
+				Max:       max1,
+				Submax:    max2,
+			}
 		}
 
 		fs = append(fs, f)
@@ -310,10 +334,9 @@ func generateDiagram(output io.Writer, s structInfo) {
 
 	for _, f := range fs {
 		tn := f.FieldType
-		sl := f.IsSlice
 		name := uncamelize(f.Name)
 
-		if sl {
+		if f.IsSlice {
 			fmt.Fprintf(output, "| %s |\n", center("Number of "+name, 61))
 			fmt.Fprintln(output, line)
 		}
@@ -321,10 +344,10 @@ func generateDiagram(output io.Writer, s structInfo) {
 		case "bool":
 			fmt.Fprintf(output, "| %s |V|\n", center(name+" (V=0 or 1)", 59))
 			fmt.Fprintln(output, line)
-		case "uint16":
+		case "int16", "uint16":
 			fmt.Fprintf(output, "| %s | %s |\n", center("0x0000", 29), center(name, 29))
 			fmt.Fprintln(output, line)
-		case "uint32":
+		case "int32", "uint32":
 			fmt.Fprintf(output, "| %s |\n", center(name, 61))
 			fmt.Fprintln(output, line)
 		case "int64", "uint64":
@@ -340,13 +363,16 @@ func generateDiagram(output io.Writer, s structInfo) {
 			fmt.Fprintf(output, "/ %61s /\n", "")
 			fmt.Fprintln(output, line)
 		default:
-			if sl {
+			if f.IsSlice {
 				tn = "Zero or more " + tn + " Structures"
 				fmt.Fprintf(output, "/ %s /\n", center("", 61))
 				fmt.Fprintf(output, "\\ %s \\\n", center(tn, 61))
 				fmt.Fprintf(output, "/ %s /\n", center("", 61))
 			} else {
-				fmt.Fprintf(output, "| %s |\n", center(tn, 61))
+				tn = tn + " Structure"
+				fmt.Fprintf(output, "/ %s /\n", center("", 61))
+				fmt.Fprintf(output, "\\ %s \\\n", center(tn, 61))
+				fmt.Fprintf(output, "/ %s /\n", center("", 61))
 			}
 			fmt.Fprintln(output, line)
 		}
@@ -374,6 +400,8 @@ func generateXdr(output io.Writer, s structInfo) {
 		}
 
 		switch tn {
+		case "int16", "int32":
+			fmt.Fprintf(output, "\tint %s%s;\n", fn, suf)
 		case "uint16", "uint32":
 			fmt.Fprintf(output, "\tunsigned int %s%s;\n", fn, suf)
 		case "int64":

Godeps/_workspace/src/github.com/calmh/xdr/encdec_test.go

@@ -1,79 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-package xdr_test
-
-import (
-	"bytes"
-	"math/rand"
-	"reflect"
-	"testing"
-	"testing/quick"
-
-	"github.com/calmh/xdr"
-)
-
-// Contains all supported types
-type TestStruct struct {
-	I    int
-	I8   int8
-	UI8  uint8
-	I16  int16
-	UI16 uint16
-	I32  int32
-	UI32 uint32
-	I64  int64
-	UI64 uint64
-	BS   []byte // max:1024
-	S    string // max:1024
-	C    Opaque
-	SS   []string // max:1024
-}
-
-type Opaque [32]byte
-
-func (u *Opaque) encodeXDR(w *xdr.Writer) (int, error) {
-	return w.WriteRaw(u[:])
-}
-
-func (u *Opaque) decodeXDR(r *xdr.Reader) (int, error) {
-	return r.ReadRaw(u[:])
-}
-
-func (Opaque) Generate(rand *rand.Rand, size int) reflect.Value {
-	var u Opaque
-	for i := range u[:] {
-		u[i] = byte(rand.Int())
-	}
-	return reflect.ValueOf(u)
-}
-
-func TestEncDec(t *testing.T) {
-	fn := func(t0 TestStruct) bool {
-		bs, err := t0.MarshalXDR()
-		if err != nil {
-			t.Fatal(err)
-		}
-		var t1 TestStruct
-		err = t1.UnmarshalXDR(bs)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		// Not comparing with DeepEqual since we'll unmarshal nil slices as empty
-		if t0.I != t1.I ||
-			t0.I16 != t1.I16 || t0.UI16 != t1.UI16 ||
-			t0.I32 != t1.I32 || t0.UI32 != t1.UI32 ||
-			t0.I64 != t1.I64 || t0.UI64 != t1.UI64 ||
-			bytes.Compare(t0.BS, t1.BS) != 0 ||
-			t0.S != t1.S || t0.C != t1.C {
-			t.Logf("%#v", t0)
-			t.Logf("%#v", t1)
-			return false
-		}
-		return true
-	}
-	if err := quick.Check(fn, nil); err != nil {
-		t.Error(err)
-	}
-}

Godeps/_workspace/src/github.com/calmh/xdr/encdec_xdr_test.go

@@ -1,174 +0,0 @@
-// ************************************************************
-// This file is automatically generated by genxdr. Do not edit.
-// ************************************************************
-
-package xdr_test
-
-import (
-	"bytes"
-	"io"
-
-	"github.com/calmh/xdr"
-)
-
-/*
-
-TestStruct Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                              int                              |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             int8                              |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             uint8                             |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             int16                             |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|            0x0000             |             UI16              |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             int32                             |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             UI32                              |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                                                               |
-+                         I64 (64 bits)                         +
-|                                                               |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                                                               |
-+                        UI64 (64 bits)                         +
-|                                                               |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                         Length of BS                          |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                     BS (variable length)                      \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                          Length of S                          |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                      S (variable length)                      \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                            Opaque                             |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                         Number of SS                          |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                         Length of SS                          |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                     SS (variable length)                      \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct TestStruct {
-	int I;
-	int8 I8;
-	uint8 UI8;
-	int16 I16;
-	unsigned int UI16;
-	int32 I32;
-	unsigned int UI32;
-	hyper I64;
-	unsigned hyper UI64;
-	opaque BS<1024>;
-	string S<1024>;
-	Opaque C;
-	string SS<1024>;
-}
-
-*/
-
-func (o TestStruct) EncodeXDR(w io.Writer) (int, error) {
-	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
-}
-
-func (o TestStruct) MarshalXDR() ([]byte, error) {
-	return o.AppendXDR(make([]byte, 0, 128))
-}
-
-func (o TestStruct) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o TestStruct) AppendXDR(bs []byte) ([]byte, error) {
-	var aw = xdr.AppendWriter(bs)
-	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
-	return []byte(aw), err
-}
-
-func (o TestStruct) encodeXDR(xw *xdr.Writer) (int, error) {
-	xw.WriteUint64(uint64(o.I))
-	xw.WriteUint8(uint8(o.I8))
-	xw.WriteUint8(o.UI8)
-	xw.WriteUint16(uint16(o.I16))
-	xw.WriteUint16(o.UI16)
-	xw.WriteUint32(uint32(o.I32))
-	xw.WriteUint32(o.UI32)
-	xw.WriteUint64(uint64(o.I64))
-	xw.WriteUint64(o.UI64)
-	if l := len(o.BS); l > 1024 {
-		return xw.Tot(), xdr.ElementSizeExceeded("BS", l, 1024)
-	}
-	xw.WriteBytes(o.BS)
-	if l := len(o.S); l > 1024 {
-		return xw.Tot(), xdr.ElementSizeExceeded("S", l, 1024)
-	}
-	xw.WriteString(o.S)
-	_, err := o.C.encodeXDR(xw)
-	if err != nil {
-		return xw.Tot(), err
-	}
-	if l := len(o.SS); l > 1024 {
-		return xw.Tot(), xdr.ElementSizeExceeded("SS", l, 1024)
-	}
-	xw.WriteUint32(uint32(len(o.SS)))
-	for i := range o.SS {
-		xw.WriteString(o.SS[i])
-	}
-	return xw.Tot(), xw.Error()
-}
-
-func (o *TestStruct) DecodeXDR(r io.Reader) error {
-	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
-}
-
-func (o *TestStruct) UnmarshalXDR(bs []byte) error {
-	var br = bytes.NewReader(bs)
-	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
-}
-
-func (o *TestStruct) decodeXDR(xr *xdr.Reader) error {
-	o.I = int(xr.ReadUint64())
-	o.I8 = int8(xr.ReadUint8())
-	o.UI8 = xr.ReadUint8()
-	o.I16 = int16(xr.ReadUint16())
-	o.UI16 = xr.ReadUint16()
-	o.I32 = int32(xr.ReadUint32())
-	o.UI32 = xr.ReadUint32()
-	o.I64 = int64(xr.ReadUint64())
-	o.UI64 = xr.ReadUint64()
-	o.BS = xr.ReadBytesMax(1024)
-	o.S = xr.ReadStringMax(1024)
-	(&o.C).decodeXDR(xr)
-	_SSSize := int(xr.ReadUint32())
-	if _SSSize > 1024 {
-		return xdr.ElementSizeExceeded("SS", _SSSize, 1024)
-	}
-	o.SS = make([]string, _SSSize)
-	for i := range o.SS {
-		o.SS[i] = xr.ReadString()
-	}
-	return xr.Error()
-}

Godeps/_workspace/src/github.com/calmh/xdr/reader.go

@@ -68,7 +68,8 @@ func (r *Reader) ReadBytesMaxInto(max int, dst []byte) []byte {
 	if r.err != nil {
 		return nil
 	}
-	if max > 0 && l > max {
+	if l < 0 || max > 0 && l > max {
+		// l may be negative on 32 bit builds
 		r.err = ElementSizeExceeded("bytes field", l, max)
 		return nil
 	}
@@ -154,6 +155,10 @@ func (e XDRError) Error() string {
 	return "xdr " + e.op + ": " + e.err.Error()
 }
 
+func (e XDRError) IsEOF() bool {
+	return e.err == io.EOF
+}
+
 func (r *Reader) Error() error {
 	if r.err == nil {
 		return nil

Godeps/_workspace/src/github.com/calmh/xdr/refl_test.go

@@ -1,44 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-// +build refl
-
-package xdr_test
-
-import (
-	"bytes"
-	"testing"
-
-	refl "github.com/davecgh/go-xdr/xdr"
-)
-
-func TestCompareMarshals(t *testing.T) {
-	e0 := s.MarshalXDR()
-	e1, err := refl.Marshal(s)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if bytes.Compare(e0, e1) != 0 {
-		t.Fatalf("Encoding mismatch;\n\t%x (this)\n\t%x (refl)", e0, e1)
-	}
-}
-
-func BenchmarkReflMarshal(b *testing.B) {
-	var err error
-	for i := 0; i < b.N; i++ {
-		res, err = refl.Marshal(s)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func BenchmarkReflUnmarshal(b *testing.B) {
-	var t XDRBenchStruct
-	for i := 0; i < b.N; i++ {
-		_, err := refl.Unmarshal(e, &t)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}

Godeps/_workspace/src/github.com/calmh/xdr/xdr_test.go

@@ -1,93 +0,0 @@
-// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
-// is governed by an MIT-style license that can be found in the LICENSE file.
-
-package xdr
-
-import (
-	"bytes"
-	"strings"
-	"testing"
-	"testing/quick"
-)
-
-func TestBytesNil(t *testing.T) {
-	fn := func(bs []byte) bool {
-		var b = new(bytes.Buffer)
-		var w = NewWriter(b)
-		var r = NewReader(b)
-		w.WriteBytes(bs)
-		w.WriteBytes(bs)
-		r.ReadBytes()
-		res := r.ReadBytes()
-		return bytes.Compare(bs, res) == 0
-	}
-	if err := quick.Check(fn, nil); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestBytesGiven(t *testing.T) {
-	fn := func(bs []byte) bool {
-		var b = new(bytes.Buffer)
-		var w = NewWriter(b)
-		var r = NewReader(b)
-		w.WriteBytes(bs)
-		w.WriteBytes(bs)
-		res := make([]byte, 12)
-		res = r.ReadBytesInto(res)
-		res = r.ReadBytesInto(res)
-		return bytes.Compare(bs, res) == 0
-	}
-	if err := quick.Check(fn, nil); err != nil {
-		t.Error(err)
-	}
-}
-
-func TestReadBytesMaxInto(t *testing.T) {
-	var max = 64
-	for tot := 32; tot < 128; tot++ {
-		for diff := -32; diff <= 32; diff++ {
-			var b = new(bytes.Buffer)
-			var r = NewReader(b)
-			var w = NewWriter(b)
-
-			var toWrite = make([]byte, tot)
-			w.WriteBytes(toWrite)
-
-			var buf = make([]byte, tot+diff)
-			var bs = r.ReadBytesMaxInto(max, buf)
-
-			if tot <= max {
-				if read := len(bs); read != tot {
-					t.Errorf("Incorrect read bytes, wrote=%d, buf=%d, max=%d, read=%d", tot, tot+diff, max, read)
-				}
-			} else if !strings.Contains(r.err.Error(), "exceeds size") {
-				t.Errorf("Unexpected non-ErrElementSizeExceeded error for wrote=%d, max=%d: %v", tot, max, r.err)
-			}
-		}
-	}
-}
-
-func TestReadStringMax(t *testing.T) {
-	for tot := 42; tot < 72; tot++ {
-		for max := 0; max < 128; max++ {
-			var b = new(bytes.Buffer)
-			var r = NewReader(b)
-			var w = NewWriter(b)
-
-			var toWrite = make([]byte, tot)
-			w.WriteBytes(toWrite)
-
-			var str = r.ReadStringMax(max)
-			var read = len(str)
-
-			if max == 0 || tot <= max {
-				if read != tot {
-					t.Errorf("Incorrect read bytes, wrote=%d, max=%d, read=%d", tot, max, read)
-				}
-			} else if !strings.Contains(r.err.Error(), "exceeds size") {
-				t.Errorf("Unexpected non-ErrElementSizeExceeded error for wrote=%d, max=%d, read=%d: %v", tot, max, read, r.err)
-			}
-		}
-	}
-}

Godeps/_workspace/src/github.com/golang/snappy/AUTHORS

@@ -0,0 +1,14 @@
+# This is the official list of Snappy-Go authors for copyright purposes.
+# This file is distinct from the CONTRIBUTORS files.
+# See the latter for an explanation.
+
+# Names should be added to this file as
+#	Name or Organization <email address>
+# The email address is not required for organizations.
+
+# Please keep the list sorted.
+
+Damian Gryski <dgryski@gmail.com>
+Google Inc.
+Jan Mercl <0xjnml@gmail.com>
+Sebastien Binet <seb.binet@gmail.com>

Godeps/_workspace/src/github.com/golang/snappy/CONTRIBUTORS

@@ -0,0 +1,36 @@
+# This is the official list of people who can contribute
+# (and typically have contributed) code to the Snappy-Go repository.
+# The AUTHORS file lists the copyright holders; this file
+# lists people.  For example, Google employees are listed here
+# but not in AUTHORS, because Google holds the copyright.
+#
+# The submission process automatically checks to make sure
+# that people submitting code are listed in this file (by email address).
+#
+# Names should be added to this file only after verifying that
+# the individual or the individual's organization has agreed to
+# the appropriate Contributor License Agreement, found here:
+#
+#     http://code.google.com/legal/individual-cla-v1.0.html
+#     http://code.google.com/legal/corporate-cla-v1.0.html
+#
+# The agreement for individuals can be filled out on the web.
+#
+# When adding J Random Contributor's name to this file,
+# either J's name or J's organization's name should be
+# added to the AUTHORS file, depending on whether the
+# individual or corporate CLA was used.
+
+# Names should be added to this file like so:
+#     Name <email address>
+
+# Please keep the list sorted.
+
+Damian Gryski <dgryski@gmail.com>
+Jan Mercl <0xjnml@gmail.com>
+Kai Backman <kaib@golang.org>
+Marc-Antoine Ruel <maruel@chromium.org>
+Nigel Tao <nigeltao@golang.org>
+Rob Pike <r@golang.org>
+Russ Cox <rsc@golang.org>
+Sebastien Binet <seb.binet@gmail.com>

Godeps/_workspace/src/github.com/golang/snappy/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2011 The Snappy-Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Godeps/_workspace/src/github.com/golang/snappy/README

@@ -0,0 +1,7 @@
+The Snappy compression format in the Go programming language.
+
+To download and install from source:
+$ go get github.com/golang/snappy
+
+Unless otherwise noted, the Snappy-Go source files are distributed
+under the BSD-style license found in the LICENSE file.

Godeps/_workspace/src/github.com/golang/snappy/decode.go

@@ -0,0 +1,294 @@
+// Copyright 2011 The Snappy-Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package snappy
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+)
+
+var (
+	// ErrCorrupt reports that the input is invalid.
+	ErrCorrupt = errors.New("snappy: corrupt input")
+	// ErrTooLarge reports that the uncompressed length is too large.
+	ErrTooLarge = errors.New("snappy: decoded block is too large")
+	// ErrUnsupported reports that the input isn't supported.
+	ErrUnsupported = errors.New("snappy: unsupported input")
+)
+
+// DecodedLen returns the length of the decoded block.
+func DecodedLen(src []byte) (int, error) {
+	v, _, err := decodedLen(src)
+	return v, err
+}
+
+// decodedLen returns the length of the decoded block and the number of bytes
+// that the length header occupied.
+func decodedLen(src []byte) (blockLen, headerLen int, err error) {
+	v, n := binary.Uvarint(src)
+	if n <= 0 || v > 0xffffffff {
+		return 0, 0, ErrCorrupt
+	}
+
+	const wordSize = 32 << (^uint(0) >> 32 & 1)
+	if wordSize == 32 && v > 0x7fffffff {
+		return 0, 0, ErrTooLarge
+	}
+	return int(v), n, nil
+}
+
+// Decode returns the decoded form of src. The returned slice may be a sub-
+// slice of dst if dst was large enough to hold the entire decoded block.
+// Otherwise, a newly allocated slice will be returned.
+// It is valid to pass a nil dst.
+func Decode(dst, src []byte) ([]byte, error) {
+	dLen, s, err := decodedLen(src)
+	if err != nil {
+		return nil, err
+	}
+	if len(dst) < dLen {
+		dst = make([]byte, dLen)
+	}
+
+	var d, offset, length int
+	for s < len(src) {
+		switch src[s] & 0x03 {
+		case tagLiteral:
+			x := uint(src[s] >> 2)
+			switch {
+			case x < 60:
+				s++
+			case x == 60:
+				s += 2
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-1])
+			case x == 61:
+				s += 3
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-2]) | uint(src[s-1])<<8
+			case x == 62:
+				s += 4
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-3]) | uint(src[s-2])<<8 | uint(src[s-1])<<16
+			case x == 63:
+				s += 5
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-4]) | uint(src[s-3])<<8 | uint(src[s-2])<<16 | uint(src[s-1])<<24
+			}
+			length = int(x + 1)
+			if length <= 0 {
+				return nil, errors.New("snappy: unsupported literal length")
+			}
+			if length > len(dst)-d || length > len(src)-s {
+				return nil, ErrCorrupt
+			}
+			copy(dst[d:], src[s:s+length])
+			d += length
+			s += length
+			continue
+
+		case tagCopy1:
+			s += 2
+			if s > len(src) {
+				return nil, ErrCorrupt
+			}
+			length = 4 + int(src[s-2])>>2&0x7
+			offset = int(src[s-2])&0xe0<<3 | int(src[s-1])
+
+		case tagCopy2:
+			s += 3
+			if s > len(src) {
+				return nil, ErrCorrupt
+			}
+			length = 1 + int(src[s-3])>>2
+			offset = int(src[s-2]) | int(src[s-1])<<8
+
+		case tagCopy4:
+			return nil, errors.New("snappy: unsupported COPY_4 tag")
+		}
+
+		end := d + length
+		if offset > d || end > len(dst) {
+			return nil, ErrCorrupt
+		}
+		for ; d < end; d++ {
+			dst[d] = dst[d-offset]
+		}
+	}
+	if d != dLen {
+		return nil, ErrCorrupt
+	}
+	return dst[:d], nil
+}
+
+// NewReader returns a new Reader that decompresses from r, using the framing
+// format described at
+// https://github.com/google/snappy/blob/master/framing_format.txt
+func NewReader(r io.Reader) *Reader {
+	return &Reader{
+		r:       r,
+		decoded: make([]byte, maxUncompressedChunkLen),
+		buf:     make([]byte, MaxEncodedLen(maxUncompressedChunkLen)+checksumSize),
+	}
+}
+
+// Reader is an io.Reader than can read Snappy-compressed bytes.
+type Reader struct {
+	r       io.Reader
+	err     error
+	decoded []byte
+	buf     []byte
+	// decoded[i:j] contains decoded bytes that have not yet been passed on.
+	i, j       int
+	readHeader bool
+}
+
+// Reset discards any buffered data, resets all state, and switches the Snappy
+// reader to read from r. This permits reusing a Reader rather than allocating
+// a new one.
+func (r *Reader) Reset(reader io.Reader) {
+	r.r = reader
+	r.err = nil
+	r.i = 0
+	r.j = 0
+	r.readHeader = false
+}
+
+func (r *Reader) readFull(p []byte) (ok bool) {
+	if _, r.err = io.ReadFull(r.r, p); r.err != nil {
+		if r.err == io.ErrUnexpectedEOF {
+			r.err = ErrCorrupt
+		}
+		return false
+	}
+	return true
+}
+
+// Read satisfies the io.Reader interface.
+func (r *Reader) Read(p []byte) (int, error) {
+	if r.err != nil {
+		return 0, r.err
+	}
+	for {
+		if r.i < r.j {
+			n := copy(p, r.decoded[r.i:r.j])
+			r.i += n
+			return n, nil
+		}
+		if !r.readFull(r.buf[:4]) {
+			return 0, r.err
+		}
+		chunkType := r.buf[0]
+		if !r.readHeader {
+			if chunkType != chunkTypeStreamIdentifier {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			r.readHeader = true
+		}
+		chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16
+		if chunkLen > len(r.buf) {
+			r.err = ErrUnsupported
+			return 0, r.err
+		}
+
+		// The chunk types are specified at
+		// https://github.com/google/snappy/blob/master/framing_format.txt
+		switch chunkType {
+		case chunkTypeCompressedData:
+			// Section 4.2. Compressed data (chunk type 0x00).
+			if chunkLen < checksumSize {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			buf := r.buf[:chunkLen]
+			if !r.readFull(buf) {
+				return 0, r.err
+			}
+			checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
+			buf = buf[checksumSize:]
+
+			n, err := DecodedLen(buf)
+			if err != nil {
+				r.err = err
+				return 0, r.err
+			}
+			if n > len(r.decoded) {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			if _, err := Decode(r.decoded, buf); err != nil {
+				r.err = err
+				return 0, r.err
+			}
+			if crc(r.decoded[:n]) != checksum {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			r.i, r.j = 0, n
+			continue
+
+		case chunkTypeUncompressedData:
+			// Section 4.3. Uncompressed data (chunk type 0x01).
+			if chunkLen < checksumSize {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			buf := r.buf[:checksumSize]
+			if !r.readFull(buf) {
+				return 0, r.err
+			}
+			checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
+			// Read directly into r.decoded instead of via r.buf.
+			n := chunkLen - checksumSize
+			if !r.readFull(r.decoded[:n]) {
+				return 0, r.err
+			}
+			if crc(r.decoded[:n]) != checksum {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			r.i, r.j = 0, n
+			continue
+
+		case chunkTypeStreamIdentifier:
+			// Section 4.1. Stream identifier (chunk type 0xff).
+			if chunkLen != len(magicBody) {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			if !r.readFull(r.buf[:len(magicBody)]) {
+				return 0, r.err
+			}
+			for i := 0; i < len(magicBody); i++ {
+				if r.buf[i] != magicBody[i] {
+					r.err = ErrCorrupt
+					return 0, r.err
+				}
+			}
+			continue
+		}
+
+		if chunkType <= 0x7f {
+			// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
+			r.err = ErrUnsupported
+			return 0, r.err
+		}
+		// Section 4.4 Padding (chunk type 0xfe).
+		// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
+		if !r.readFull(r.buf[:chunkLen]) {
+			return 0, r.err
+		}
+	}
+}

Godeps/_workspace/src/github.com/golang/snappy/encode.go

@@ -6,6 +6,7 @@ package snappy
 
 import (
 	"encoding/binary"
+	"io"
 )
 
 // We limit how far copy back-references can go, the same as the C++ code.
@@ -78,7 +79,7 @@ func emitCopy(dst []byte, offset, length int) int {
 // slice of dst if dst was large enough to hold the entire encoded block.
 // Otherwise, a newly allocated slice will be returned.
 // It is valid to pass a nil dst.
-func Encode(dst, src []byte) ([]byte, error) {
+func Encode(dst, src []byte) []byte {
 	if n := MaxEncodedLen(len(src)); len(dst) < n {
 		dst = make([]byte, n)
 	}
@@ -91,7 +92,7 @@ func Encode(dst, src []byte) ([]byte, error) {
 		if len(src) != 0 {
 			d += emitLiteral(dst[d:], src)
 		}
-		return dst[:d], nil
+		return dst[:d]
 	}
 
 	// Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive.
@@ -144,7 +145,7 @@ func Encode(dst, src []byte) ([]byte, error) {
 	if lit != len(src) {
 		d += emitLiteral(dst[d:], src[lit:])
 	}
-	return dst[:d], nil
+	return dst[:d]
 }
 
 // MaxEncodedLen returns the maximum length of a snappy block, given its
@@ -172,3 +173,82 @@ func MaxEncodedLen(srcLen int) int {
 	// This last factor dominates the blowup, so the final estimate is:
 	return 32 + srcLen + srcLen/6
 }
+
+// NewWriter returns a new Writer that compresses to w, using the framing
+// format described at
+// https://github.com/google/snappy/blob/master/framing_format.txt
+func NewWriter(w io.Writer) *Writer {
+	return &Writer{
+		w:   w,
+		enc: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)),
+	}
+}
+
+// Writer is an io.Writer than can write Snappy-compressed bytes.
+type Writer struct {
+	w           io.Writer
+	err         error
+	enc         []byte
+	buf         [checksumSize + chunkHeaderSize]byte
+	wroteHeader bool
+}
+
+// Reset discards the writer's state and switches the Snappy writer to write to
+// w. This permits reusing a Writer rather than allocating a new one.
+func (w *Writer) Reset(writer io.Writer) {
+	w.w = writer
+	w.err = nil
+	w.wroteHeader = false
+}
+
+// Write satisfies the io.Writer interface.
+func (w *Writer) Write(p []byte) (n int, errRet error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+	if !w.wroteHeader {
+		copy(w.enc, magicChunk)
+		if _, err := w.w.Write(w.enc[:len(magicChunk)]); err != nil {
+			w.err = err
+			return n, err
+		}
+		w.wroteHeader = true
+	}
+	for len(p) > 0 {
+		var uncompressed []byte
+		if len(p) > maxUncompressedChunkLen {
+			uncompressed, p = p[:maxUncompressedChunkLen], p[maxUncompressedChunkLen:]
+		} else {
+			uncompressed, p = p, nil
+		}
+		checksum := crc(uncompressed)
+
+		// Compress the buffer, discarding the result if the improvement
+		// isn't at least 12.5%.
+		chunkType := uint8(chunkTypeCompressedData)
+		chunkBody := Encode(w.enc, uncompressed)
+		if len(chunkBody) >= len(uncompressed)-len(uncompressed)/8 {
+			chunkType, chunkBody = chunkTypeUncompressedData, uncompressed
+		}
+
+		chunkLen := 4 + len(chunkBody)
+		w.buf[0] = chunkType
+		w.buf[1] = uint8(chunkLen >> 0)
+		w.buf[2] = uint8(chunkLen >> 8)
+		w.buf[3] = uint8(chunkLen >> 16)
+		w.buf[4] = uint8(checksum >> 0)
+		w.buf[5] = uint8(checksum >> 8)
+		w.buf[6] = uint8(checksum >> 16)
+		w.buf[7] = uint8(checksum >> 24)
+		if _, err := w.w.Write(w.buf[:]); err != nil {
+			w.err = err
+			return n, err
+		}
+		if _, err := w.w.Write(chunkBody); err != nil {
+			w.err = err
+			return n, err
+		}
+		n += len(uncompressed)
+	}
+	return n, nil
+}

Godeps/_workspace/src/github.com/golang/snappy/snappy.go

@@ -5,9 +5,13 @@
 // Package snappy implements the snappy block-based compression format.
 // It aims for very high speeds and reasonable compression.
 //
-// The C++ snappy implementation is at http://code.google.com/p/snappy/
+// The C++ snappy implementation is at https://github.com/google/snappy
 package snappy
 
+import (
+	"hash/crc32"
+)
+
 /*
 Each encoded block begins with the varint-encoded length of the decoded data,
 followed by a sequence of chunks. Chunks begin and end on byte boundaries. The
@@ -36,3 +40,29 @@ const (
 	tagCopy2   = 0x02
 	tagCopy4   = 0x03
 )
+
+const (
+	checksumSize    = 4
+	chunkHeaderSize = 4
+	magicChunk      = "\xff\x06\x00\x00" + magicBody
+	magicBody       = "sNaPpY"
+	// https://github.com/google/snappy/blob/master/framing_format.txt says
+	// that "the uncompressed data in a chunk must be no longer than 65536 bytes".
+	maxUncompressedChunkLen = 65536
+)
+
+const (
+	chunkTypeCompressedData   = 0x00
+	chunkTypeUncompressedData = 0x01
+	chunkTypePadding          = 0xfe
+	chunkTypeStreamIdentifier = 0xff
+)
+
+var crcTable = crc32.MakeTable(crc32.Castagnoli)
+
+// crc implements the checksum specified in section 3 of
+// https://github.com/google/snappy/blob/master/framing_format.txt
+func crc(b []byte) uint32 {
+	c := crc32.Update(0, crcTable, b)
+	return uint32(c>>15|c<<17) + 0xa282ead8
+}

Godeps/_workspace/src/github.com/juju/ratelimit/LICENSE

@@ -1,3 +1,9 @@
+All files in this repository are licensed as follows. If you contribute
+to this repository, it is assumed that you license your contribution
+under the same license unless you state otherwise.
+
+All files Copyright (C) 2015 Canonical Ltd. unless otherwise specified in the file.
+
 This software is licensed under the LGPLv3, included below.
 
 As a special exception to the GNU Lesser General Public License version 3

Godeps/_workspace/src/github.com/juju/ratelimit/README.md

@@ -20,7 +20,7 @@ token in the bucket represents one byte.
 ```go
 func Writer(w io.Writer, bucket *Bucket) io.Writer
 ```
-Writer returns a reader that is rate limited by the given token bucket. Each
+Writer returns a writer that is rate limited by the given token bucket. Each
 token in the bucket represents one byte.
 
 #### type Bucket

Godeps/_workspace/src/github.com/juju/ratelimit/ratelimit.go

@@ -2,7 +2,8 @@
 // Licensed under the LGPLv3 with static-linking exception.
 // See LICENCE file for details.
 
-// The ratelimit package provides an efficient token bucket implementation.
+// The ratelimit package provides an efficient token bucket implementation
+// that can be used to limit the rate of arbitrary things.
 // See http://en.wikipedia.org/wiki/Token_bucket.
 package ratelimit
 
@@ -10,6 +11,7 @@ import (
 	"strconv"
 	"sync"
 	"time"
+	"math"
 )
 
 // Bucket represents a token bucket that fills at a predetermined rate.
@@ -55,7 +57,7 @@ func NewBucketWithRate(rate float64, capacity int64) *Bucket {
 			continue
 		}
 		tb := NewBucketWithQuantum(fillInterval, capacity, quantum)
-		if diff := abs(tb.Rate() - rate); diff/rate <= rateMargin {
+		if diff := math.Abs(tb.Rate() - rate); diff/rate <= rateMargin {
 			return tb
 		}
 	}
@@ -217,10 +219,3 @@ func (tb *Bucket) adjust(now time.Time) (currentTick int64) {
 	tb.availTick = currentTick
 	return
 }
-
-func abs(f float64) float64 {
-	if f < 0 {
-		return -f
-	}
-	return f
-}

Godeps/_workspace/src/github.com/juju/ratelimit/ratelimit_test.go

@@ -1,328 +0,0 @@
-// Copyright 2014 Canonical Ltd.
-// Licensed under the LGPLv3 with static-linking exception.
-// See LICENCE file for details.
-
-package ratelimit
-
-import (
-	gc "launchpad.net/gocheck"
-
-	"testing"
-	"time"
-)
-
-func TestPackage(t *testing.T) {
-	gc.TestingT(t)
-}
-
-type rateLimitSuite struct{}
-
-var _ = gc.Suite(rateLimitSuite{})
-
-type takeReq struct {
-	time       time.Duration
-	count      int64
-	expectWait time.Duration
-}
-
-var takeTests = []struct {
-	about        string
-	fillInterval time.Duration
-	capacity     int64
-	reqs         []takeReq
-}{{
-	about:        "serial requests",
-	fillInterval: 250 * time.Millisecond,
-	capacity:     10,
-	reqs: []takeReq{{
-		time:       0,
-		count:      0,
-		expectWait: 0,
-	}, {
-		time:       0,
-		count:      10,
-		expectWait: 0,
-	}, {
-		time:       0,
-		count:      1,
-		expectWait: 250 * time.Millisecond,
-	}, {
-		time:       250 * time.Millisecond,
-		count:      1,
-		expectWait: 250 * time.Millisecond,
-	}},
-}, {
-	about:        "concurrent requests",
-	fillInterval: 250 * time.Millisecond,
-	capacity:     10,
-	reqs: []takeReq{{
-		time:       0,
-		count:      10,
-		expectWait: 0,
-	}, {
-		time:       0,
-		count:      2,
-		expectWait: 500 * time.Millisecond,
-	}, {
-		time:       0,
-		count:      2,
-		expectWait: 1000 * time.Millisecond,
-	}, {
-		time:       0,
-		count:      1,
-		expectWait: 1250 * time.Millisecond,
-	}},
-}, {
-	about:        "more than capacity",
-	fillInterval: 1 * time.Millisecond,
-	capacity:     10,
-	reqs: []takeReq{{
-		time:       0,
-		count:      10,
-		expectWait: 0,
-	}, {
-		time:       20 * time.Millisecond,
-		count:      15,
-		expectWait: 5 * time.Millisecond,
-	}},
-}, {
-	about:        "sub-quantum time",
-	fillInterval: 10 * time.Millisecond,
-	capacity:     10,
-	reqs: []takeReq{{
-		time:       0,
-		count:      10,
-		expectWait: 0,
-	}, {
-		time:       7 * time.Millisecond,
-		count:      1,
-		expectWait: 3 * time.Millisecond,
-	}, {
-		time:       8 * time.Millisecond,
-		count:      1,
-		expectWait: 12 * time.Millisecond,
-	}},
-}, {
-	about:        "within capacity",
-	fillInterval: 10 * time.Millisecond,
-	capacity:     5,
-	reqs: []takeReq{{
-		time:       0,
-		count:      5,
-		expectWait: 0,
-	}, {
-		time:       60 * time.Millisecond,
-		count:      5,
-		expectWait: 0,
-	}, {
-		time:       60 * time.Millisecond,
-		count:      1,
-		expectWait: 10 * time.Millisecond,
-	}, {
-		time:       80 * time.Millisecond,
-		count:      2,
-		expectWait: 10 * time.Millisecond,
-	}},
-}}
-
-func (rateLimitSuite) TestTake(c *gc.C) {
-	for i, test := range takeTests {
-		tb := NewBucket(test.fillInterval, test.capacity)
-		for j, req := range test.reqs {
-			d, ok := tb.take(tb.startTime.Add(req.time), req.count, infinityDuration)
-			c.Assert(ok, gc.Equals, true)
-			if d != req.expectWait {
-				c.Fatalf("test %d.%d, %s, got %v want %v", i, j, test.about, d, req.expectWait)
-			}
-		}
-	}
-}
-
-func (rateLimitSuite) TestTakeMaxDuration(c *gc.C) {
-	for i, test := range takeTests {
-		tb := NewBucket(test.fillInterval, test.capacity)
-		for j, req := range test.reqs {
-			if req.expectWait > 0 {
-				d, ok := tb.take(tb.startTime.Add(req.time), req.count, req.expectWait-1)
-				c.Assert(ok, gc.Equals, false)
-				c.Assert(d, gc.Equals, time.Duration(0))
-			}
-			d, ok := tb.take(tb.startTime.Add(req.time), req.count, req.expectWait)
-			c.Assert(ok, gc.Equals, true)
-			if d != req.expectWait {
-				c.Fatalf("test %d.%d, %s, got %v want %v", i, j, test.about, d, req.expectWait)
-			}
-		}
-	}
-}
-
-type takeAvailableReq struct {
-	time   time.Duration
-	count  int64
-	expect int64
-}
-
-var takeAvailableTests = []struct {
-	about        string
-	fillInterval time.Duration
-	capacity     int64
-	reqs         []takeAvailableReq
-}{{
-	about:        "serial requests",
-	fillInterval: 250 * time.Millisecond,
-	capacity:     10,
-	reqs: []takeAvailableReq{{
-		time:   0,
-		count:  0,
-		expect: 0,
-	}, {
-		time:   0,
-		count:  10,
-		expect: 10,
-	}, {
-		time:   0,
-		count:  1,
-		expect: 0,
-	}, {
-		time:   250 * time.Millisecond,
-		count:  1,
-		expect: 1,
-	}},
-}, {
-	about:        "concurrent requests",
-	fillInterval: 250 * time.Millisecond,
-	capacity:     10,
-	reqs: []takeAvailableReq{{
-		time:   0,
-		count:  5,
-		expect: 5,
-	}, {
-		time:   0,
-		count:  2,
-		expect: 2,
-	}, {
-		time:   0,
-		count:  5,
-		expect: 3,
-	}, {
-		time:   0,
-		count:  1,
-		expect: 0,
-	}},
-}, {
-	about:        "more than capacity",
-	fillInterval: 1 * time.Millisecond,
-	capacity:     10,
-	reqs: []takeAvailableReq{{
-		time:   0,
-		count:  10,
-		expect: 10,
-	}, {
-		time:   20 * time.Millisecond,
-		count:  15,
-		expect: 10,
-	}},
-}, {
-	about:        "within capacity",
-	fillInterval: 10 * time.Millisecond,
-	capacity:     5,
-	reqs: []takeAvailableReq{{
-		time:   0,
-		count:  5,
-		expect: 5,
-	}, {
-		time:   60 * time.Millisecond,
-		count:  5,
-		expect: 5,
-	}, {
-		time:   70 * time.Millisecond,
-		count:  1,
-		expect: 1,
-	}},
-}}
-
-func (rateLimitSuite) TestTakeAvailable(c *gc.C) {
-	for i, test := range takeAvailableTests {
-		tb := NewBucket(test.fillInterval, test.capacity)
-		for j, req := range test.reqs {
-			d := tb.takeAvailable(tb.startTime.Add(req.time), req.count)
-			if d != req.expect {
-				c.Fatalf("test %d.%d, %s, got %v want %v", i, j, test.about, d, req.expect)
-			}
-		}
-	}
-}
-
-func (rateLimitSuite) TestPanics(c *gc.C) {
-	c.Assert(func() { NewBucket(0, 1) }, gc.PanicMatches, "token bucket fill interval is not > 0")
-	c.Assert(func() { NewBucket(-2, 1) }, gc.PanicMatches, "token bucket fill interval is not > 0")
-	c.Assert(func() { NewBucket(1, 0) }, gc.PanicMatches, "token bucket capacity is not > 0")
-	c.Assert(func() { NewBucket(1, -2) }, gc.PanicMatches, "token bucket capacity is not > 0")
-}
-
-func isCloseTo(x, y, tolerance float64) bool {
-	return abs(x-y)/y < tolerance
-}
-
-func (rateLimitSuite) TestRate(c *gc.C) {
-	tb := NewBucket(1, 1)
-	if !isCloseTo(tb.Rate(), 1e9, 0.00001) {
-		c.Fatalf("got %v want 1e9", tb.Rate())
-	}
-	tb = NewBucket(2*time.Second, 1)
-	if !isCloseTo(tb.Rate(), 0.5, 0.00001) {
-		c.Fatalf("got %v want 0.5", tb.Rate())
-	}
-	tb = NewBucketWithQuantum(100*time.Millisecond, 1, 5)
-	if !isCloseTo(tb.Rate(), 50, 0.00001) {
-		c.Fatalf("got %v want 50", tb.Rate())
-	}
-}
-
-func checkRate(c *gc.C, rate float64) {
-	tb := NewBucketWithRate(rate, 1<<62)
-	if !isCloseTo(tb.Rate(), rate, rateMargin) {
-		c.Fatalf("got %g want %v", tb.Rate(), rate)
-	}
-	d, ok := tb.take(tb.startTime, 1<<62, infinityDuration)
-	c.Assert(ok, gc.Equals, true)
-	c.Assert(d, gc.Equals, time.Duration(0))
-
-	// Check that the actual rate is as expected by
-	// asking for a not-quite multiple of the bucket's
-	// quantum and checking that the wait time
-	// correct.
-	d, ok = tb.take(tb.startTime, tb.quantum*2-tb.quantum/2, infinityDuration)
-	c.Assert(ok, gc.Equals, true)
-	expectTime := 1e9 * float64(tb.quantum) * 2 / rate
-	if !isCloseTo(float64(d), expectTime, rateMargin) {
-		c.Fatalf("rate %g: got %g want %v", rate, float64(d), expectTime)
-	}
-}
-
-func (rateLimitSuite) TestNewWithRate(c *gc.C) {
-	for rate := float64(1); rate < 1e6; rate += 7 {
-		checkRate(c, rate)
-	}
-	for _, rate := range []float64{
-		1024 * 1024 * 1024,
-		1e-5,
-		0.9e-5,
-		0.5,
-		0.9,
-		0.9e8,
-		3e12,
-		4e18,
-	} {
-		checkRate(c, rate)
-		checkRate(c, rate/3)
-		checkRate(c, rate*1.3)
-	}
-}
-
-func BenchmarkWait(b *testing.B) {
-	tb := NewBucket(1, 16*1024)
-	for i := b.N - 1; i >= 0; i-- {
-		tb.Wait(1)
-	}
-}

Godeps/_workspace/src/github.com/kardianos/osext/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2012 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Godeps/_workspace/src/github.com/kardianos/osext/README.md

@@ -0,0 +1,16 @@
+### Extensions to the "os" package.
+
+## Find the current Executable and ExecutableFolder.
+
+There is sometimes utility in finding the current executable file
+that is running. This can be used for upgrading the current executable
+or finding resources located relative to the executable file. Both
+working directory and the os.Args[0] value are arbitrary and cannot
+be relied on; os.Args[0] can be "faked".
+
+Multi-platform and supports:
+ * Linux
+ * OS X
+ * Windows
+ * Plan 9
+ * BSDs.

Godeps/_workspace/src/github.com/kardianos/osext/osext.go

@@ -16,17 +16,12 @@ func Executable() (string, error) {
 }
 
 // Returns same path as Executable, returns just the folder
-// path. Excludes the executable name.
+// path. Excludes the executable name and any trailing slash.
 func ExecutableFolder() (string, error) {
 	p, err := Executable()
 	if err != nil {
 		return "", err
 	}
-	folder, _ := filepath.Split(p)
-	return folder, nil
-}
 
-// Depricated. Same as Executable().
-func GetExePath() (exePath string, err error) {
-	return Executable()
+	return filepath.Dir(p), nil
 }

Godeps/_workspace/src/github.com/kardianos/osext/osext_plan9.go

@@ -5,16 +5,16 @@
 package osext
 
 import (
-    "syscall"
-    "os"
-    "strconv"
+	"os"
+	"strconv"
+	"syscall"
 )
 
 func executable() (string, error) {
-    f, err := os.Open("/proc/" + strconv.Itoa(os.Getpid()) + "/text")
-    if err != nil {
-        return "", err
-    }
-    defer f.Close()
-    return syscall.Fd2path(int(f.Fd()))
+	f, err := os.Open("/proc/" + strconv.Itoa(os.Getpid()) + "/text")
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	return syscall.Fd2path(int(f.Fd()))
 }

Godeps/_workspace/src/github.com/kardianos/osext/osext_procfs.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build linux netbsd openbsd solaris
+// +build linux netbsd openbsd solaris dragonfly
 
 package osext
 
@@ -11,15 +11,23 @@ import (
 	"fmt"
 	"os"
 	"runtime"
+	"strings"
 )
 
 func executable() (string, error) {
 	switch runtime.GOOS {
 	case "linux":
-		return os.Readlink("/proc/self/exe")
+		const deletedTag = " (deleted)"
+		execpath, err := os.Readlink("/proc/self/exe")
+		if err != nil {
+			return execpath, err
+		}
+		execpath = strings.TrimSuffix(execpath, deletedTag)
+		execpath = strings.TrimPrefix(execpath, deletedTag)
+		return execpath, nil
 	case "netbsd":
 		return os.Readlink("/proc/curproc/exe")
-	case "openbsd":
+	case "openbsd", "dragonfly":
 		return os.Readlink("/proc/curproc/file")
 	case "solaris":
 		return os.Readlink(fmt.Sprintf("/proc/%d/path/a.out", os.Getpid()))

Godeps/_workspace/src/github.com/rcrowley/go-metrics/.gitignore

@@ -0,0 +1,9 @@
+*.[68]
+*.a
+*.out
+*.swp
+_obj
+_testmain.go
+cmd/metrics-bench/metrics-bench
+cmd/metrics-example/metrics-example
+cmd/never-read/never-read

Godeps/_workspace/src/github.com/rcrowley/go-metrics/.travis.yml

@@ -0,0 +1,13 @@
+language: go
+
+go:
+    - 1.2
+    - 1.3
+    - 1.4
+
+script:
+    - ./validate.sh
+
+# this should give us faster builds according to 
+# http://docs.travis-ci.com/user/migrating-from-legacy/
+sudo: false

Godeps/_workspace/src/github.com/rcrowley/go-metrics/LICENSE

@@ -0,0 +1,29 @@
+Copyright 2012 Richard Crowley. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    1.  Redistributions of source code must retain the above copyright
+        notice, this list of conditions and the following disclaimer.
+
+    2.  Redistributions in binary form must reproduce the above
+        copyright notice, this list of conditions and the following
+        disclaimer in the documentation and/or other materials provided
+        with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY RICHARD CROWLEY ``AS IS'' AND ANY EXPRESS
+OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL RICHARD CROWLEY OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
+
+The views and conclusions contained in the software and documentation
+are those of the authors and should not be interpreted as representing
+official policies, either expressed or implied, of Richard Crowley.

Godeps/_workspace/src/github.com/rcrowley/go-metrics/README.md

@@ -0,0 +1,126 @@
+go-metrics
+==========
+
+![travis build status](https://travis-ci.org/rcrowley/go-metrics.svg?branch=master)
+
+Go port of Coda Hale's Metrics library: <https://github.com/dropwizard/metrics>.
+
+Documentation: <http://godoc.org/github.com/rcrowley/go-metrics>.
+
+Usage
+-----
+
+Create and update metrics:
+
+```go
+c := metrics.NewCounter()
+metrics.Register("foo", c)
+c.Inc(47)
+
+g := metrics.NewGauge()
+metrics.Register("bar", g)
+g.Update(47)
+
+s := metrics.NewExpDecaySample(1028, 0.015) // or metrics.NewUniformSample(1028)
+h := metrics.NewHistogram(s)
+metrics.Register("baz", h)
+h.Update(47)
+
+m := metrics.NewMeter()
+metrics.Register("quux", m)
+m.Mark(47)
+
+t := metrics.NewTimer()
+metrics.Register("bang", t)
+t.Time(func() {})
+t.Update(47)
+```
+
+Periodically log every metric in human-readable form to standard error:
+
+```go
+go metrics.Log(metrics.DefaultRegistry, 60e9, log.New(os.Stderr, "metrics: ", log.Lmicroseconds))
+```
+
+Periodically log every metric in slightly-more-parseable form to syslog:
+
+```go
+w, _ := syslog.Dial("unixgram", "/dev/log", syslog.LOG_INFO, "metrics")
+go metrics.Syslog(metrics.DefaultRegistry, 60e9, w)
+```
+
+Periodically emit every metric to Graphite using the [Graphite client](https://github.com/cyberdelia/go-metrics-graphite):
+
+```go
+
+import "github.com/cyberdelia/go-metrics-graphite"
+
+addr, _ := net.ResolveTCPAddr("tcp", "127.0.0.1:2003")
+go graphite.Graphite(metrics.DefaultRegistry, 10e9, "metrics", addr)
+```
+
+Periodically emit every metric into InfluxDB:
+
+**NOTE:** this has been pulled out of the library due to constant fluctuations
+in the InfluxDB API. In fact, all client libraries are on their way out. see
+issues [#121](https://github.com/rcrowley/go-metrics/issues/121) and
+[#124](https://github.com/rcrowley/go-metrics/issues/124) for progress and details.
+
+```go
+import "github.com/rcrowley/go-metrics/influxdb"
+
+go influxdb.Influxdb(metrics.DefaultRegistry, 10e9, &influxdb.Config{
+    Host:     "127.0.0.1:8086",
+    Database: "metrics",
+    Username: "test",
+    Password: "test",
+})
+```
+
+Periodically upload every metric to Librato using the [Librato client](https://github.com/mihasya/go-metrics-librato):
+
+**Note**: the client included with this repository under the `librato` package
+has been deprecated and moved to the repository linked above.
+
+```go
+import "github.com/mihasya/go-metrics-librato"
+
+go librato.Librato(metrics.DefaultRegistry,
+    10e9,                  // interval
+    "example@example.com", // account owner email address
+    "token",               // Librato API token
+    "hostname",            // source
+    []float64{0.95},       // percentiles to send
+    time.Millisecond,      // time unit
+)
+```
+
+Periodically emit every metric to StatHat:
+
+```go
+import "github.com/rcrowley/go-metrics/stathat"
+
+go stathat.Stathat(metrics.DefaultRegistry, 10e9, "example@example.com")
+```
+
+Installation
+------------
+
+```sh
+go get github.com/rcrowley/go-metrics
+```
+
+StatHat support additionally requires their Go client:
+
+```sh
+go get github.com/stathat/go
+```
+
+Publishing Metrics
+------------------
+
+Clients are available for the following destinations:
+
+* Librato - [https://github.com/mihasya/go-metrics-librato](https://github.com/mihasya/go-metrics-librato)
+* Graphite - [https://github.com/cyberdelia/go-metrics-graphite](https://github.com/cyberdelia/go-metrics-graphite)
+* InfluxDB - [https://github.com/vrischmann/go-metrics-influxdb](https://github.com/vrischmann/go-metrics-influxdb)

Godeps/_workspace/src/github.com/rcrowley/go-metrics/cmd/metrics-bench/metrics-bench.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"fmt"
+	"github.com/rcrowley/go-metrics"
+	"time"
+)
+
+func main() {
+	r := metrics.NewRegistry()
+	for i := 0; i < 10000; i++ {
+		r.Register(fmt.Sprintf("counter-%d", i), metrics.NewCounter())
+		r.Register(fmt.Sprintf("gauge-%d", i), metrics.NewGauge())
+		r.Register(fmt.Sprintf("gaugefloat64-%d", i), metrics.NewGaugeFloat64())
+		r.Register(fmt.Sprintf("histogram-uniform-%d", i), metrics.NewHistogram(metrics.NewUniformSample(1028)))
+		r.Register(fmt.Sprintf("histogram-exp-%d", i), metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015)))
+		r.Register(fmt.Sprintf("meter-%d", i), metrics.NewMeter())
+	}
+	time.Sleep(600e9)
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/cmd/metrics-example/metrics-example.go

@@ -0,0 +1,154 @@
+package main
+
+import (
+	"errors"
+	"github.com/rcrowley/go-metrics"
+	// "github.com/rcrowley/go-metrics/stathat"
+	"log"
+	"math/rand"
+	"os"
+	// "syslog"
+	"time"
+)
+
+const fanout = 10
+
+func main() {
+
+	r := metrics.NewRegistry()
+
+	c := metrics.NewCounter()
+	r.Register("foo", c)
+	for i := 0; i < fanout; i++ {
+		go func() {
+			for {
+				c.Dec(19)
+				time.Sleep(300e6)
+			}
+		}()
+		go func() {
+			for {
+				c.Inc(47)
+				time.Sleep(400e6)
+			}
+		}()
+	}
+
+	g := metrics.NewGauge()
+	r.Register("bar", g)
+	for i := 0; i < fanout; i++ {
+		go func() {
+			for {
+				g.Update(19)
+				time.Sleep(300e6)
+			}
+		}()
+		go func() {
+			for {
+				g.Update(47)
+				time.Sleep(400e6)
+			}
+		}()
+	}
+
+	gf := metrics.NewGaugeFloat64()
+	r.Register("barfloat64", gf)
+	for i := 0; i < fanout; i++ {
+		go func() {
+			for {
+				g.Update(19.0)
+				time.Sleep(300e6)
+			}
+		}()
+		go func() {
+			for {
+				g.Update(47.0)
+				time.Sleep(400e6)
+			}
+		}()
+	}
+
+	hc := metrics.NewHealthcheck(func(h metrics.Healthcheck) {
+		if 0 < rand.Intn(2) {
+			h.Healthy()
+		} else {
+			h.Unhealthy(errors.New("baz"))
+		}
+	})
+	r.Register("baz", hc)
+
+	s := metrics.NewExpDecaySample(1028, 0.015)
+	//s := metrics.NewUniformSample(1028)
+	h := metrics.NewHistogram(s)
+	r.Register("bang", h)
+	for i := 0; i < fanout; i++ {
+		go func() {
+			for {
+				h.Update(19)
+				time.Sleep(300e6)
+			}
+		}()
+		go func() {
+			for {
+				h.Update(47)
+				time.Sleep(400e6)
+			}
+		}()
+	}
+
+	m := metrics.NewMeter()
+	r.Register("quux", m)
+	for i := 0; i < fanout; i++ {
+		go func() {
+			for {
+				m.Mark(19)
+				time.Sleep(300e6)
+			}
+		}()
+		go func() {
+			for {
+				m.Mark(47)
+				time.Sleep(400e6)
+			}
+		}()
+	}
+
+	t := metrics.NewTimer()
+	r.Register("hooah", t)
+	for i := 0; i < fanout; i++ {
+		go func() {
+			for {
+				t.Time(func() { time.Sleep(300e6) })
+			}
+		}()
+		go func() {
+			for {
+				t.Time(func() { time.Sleep(400e6) })
+			}
+		}()
+	}
+
+	metrics.RegisterDebugGCStats(r)
+	go metrics.CaptureDebugGCStats(r, 5e9)
+
+	metrics.RegisterRuntimeMemStats(r)
+	go metrics.CaptureRuntimeMemStats(r, 5e9)
+
+	metrics.Log(r, 60e9, log.New(os.Stderr, "metrics: ", log.Lmicroseconds))
+
+	/*
+		w, err := syslog.Dial("unixgram", "/dev/log", syslog.LOG_INFO, "metrics")
+		if nil != err { log.Fatalln(err) }
+		metrics.Syslog(r, 60e9, w)
+	*/
+
+	/*
+		addr, _ := net.ResolveTCPAddr("tcp", "127.0.0.1:2003")
+		metrics.Graphite(r, 10e9, "metrics", addr)
+	*/
+
+	/*
+		stathat.Stathat(r, 10e9, "example@example.com")
+	*/
+
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/cmd/never-read/never-read.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"log"
+	"net"
+)
+
+func main() {
+	addr, _ := net.ResolveTCPAddr("tcp", "127.0.0.1:2003")
+	l, err := net.ListenTCP("tcp", addr)
+	if nil != err {
+		log.Fatalln(err)
+	}
+	log.Println("listening", l.Addr())
+	for {
+		c, err := l.AcceptTCP()
+		if nil != err {
+			log.Fatalln(err)
+		}
+		log.Println("accepted", c.RemoteAddr())
+	}
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/counter.go

@@ -0,0 +1,112 @@
+package metrics
+
+import "sync/atomic"
+
+// Counters hold an int64 value that can be incremented and decremented.
+type Counter interface {
+	Clear()
+	Count() int64
+	Dec(int64)
+	Inc(int64)
+	Snapshot() Counter
+}
+
+// GetOrRegisterCounter returns an existing Counter or constructs and registers
+// a new StandardCounter.
+func GetOrRegisterCounter(name string, r Registry) Counter {
+	if nil == r {
+		r = DefaultRegistry
+	}
+	return r.GetOrRegister(name, NewCounter).(Counter)
+}
+
+// NewCounter constructs a new StandardCounter.
+func NewCounter() Counter {
+	if UseNilMetrics {
+		return NilCounter{}
+	}
+	return &StandardCounter{0}
+}
+
+// NewRegisteredCounter constructs and registers a new StandardCounter.
+func NewRegisteredCounter(name string, r Registry) Counter {
+	c := NewCounter()
+	if nil == r {
+		r = DefaultRegistry
+	}
+	r.Register(name, c)
+	return c
+}
+
+// CounterSnapshot is a read-only copy of another Counter.
+type CounterSnapshot int64
+
+// Clear panics.
+func (CounterSnapshot) Clear() {
+	panic("Clear called on a CounterSnapshot")
+}
+
+// Count returns the count at the time the snapshot was taken.
+func (c CounterSnapshot) Count() int64 { return int64(c) }
+
+// Dec panics.
+func (CounterSnapshot) Dec(int64) {
+	panic("Dec called on a CounterSnapshot")
+}
+
+// Inc panics.
+func (CounterSnapshot) Inc(int64) {
+	panic("Inc called on a CounterSnapshot")
+}
+
+// Snapshot returns the snapshot.
+func (c CounterSnapshot) Snapshot() Counter { return c }
+
+// NilCounter is a no-op Counter.
+type NilCounter struct{}
+
+// Clear is a no-op.
+func (NilCounter) Clear() {}
+
+// Count is a no-op.
+func (NilCounter) Count() int64 { return 0 }
+
+// Dec is a no-op.
+func (NilCounter) Dec(i int64) {}
+
+// Inc is a no-op.
+func (NilCounter) Inc(i int64) {}
+
+// Snapshot is a no-op.
+func (NilCounter) Snapshot() Counter { return NilCounter{} }
+
+// StandardCounter is the standard implementation of a Counter and uses the
+// sync/atomic package to manage a single int64 value.
+type StandardCounter struct {
+	count int64
+}
+
+// Clear sets the counter to zero.
+func (c *StandardCounter) Clear() {
+	atomic.StoreInt64(&c.count, 0)
+}
+
+// Count returns the current count.
+func (c *StandardCounter) Count() int64 {
+	return atomic.LoadInt64(&c.count)
+}
+
+// Dec decrements the counter by the given amount.
+func (c *StandardCounter) Dec(i int64) {
+	atomic.AddInt64(&c.count, -i)
+}
+
+// Inc increments the counter by the given amount.
+func (c *StandardCounter) Inc(i int64) {
+	atomic.AddInt64(&c.count, i)
+}
+
+// Snapshot returns a read-only copy of the counter.
+func (c *StandardCounter) Snapshot() Counter {
+	return CounterSnapshot(c.Count())
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/debug.go

@@ -0,0 +1,76 @@
+package metrics
+
+import (
+	"runtime/debug"
+	"time"
+)
+
+var (
+	debugMetrics struct {
+		GCStats struct {
+			LastGC Gauge
+			NumGC  Gauge
+			Pause  Histogram
+			//PauseQuantiles Histogram
+			PauseTotal Gauge
+		}
+		ReadGCStats Timer
+	}
+	gcStats debug.GCStats
+)
+
+// Capture new values for the Go garbage collector statistics exported in
+// debug.GCStats.  This is designed to be called as a goroutine.
+func CaptureDebugGCStats(r Registry, d time.Duration) {
+	for _ = range time.Tick(d) {
+		CaptureDebugGCStatsOnce(r)
+	}
+}
+
+// Capture new values for the Go garbage collector statistics exported in
+// debug.GCStats.  This is designed to be called in a background goroutine.
+// Giving a registry which has not been given to RegisterDebugGCStats will
+// panic.
+//
+// Be careful (but much less so) with this because debug.ReadGCStats calls
+// the C function runtime·lock(runtime·mheap) which, while not a stop-the-world
+// operation, isn't something you want to be doing all the time.
+func CaptureDebugGCStatsOnce(r Registry) {
+	lastGC := gcStats.LastGC
+	t := time.Now()
+	debug.ReadGCStats(&gcStats)
+	debugMetrics.ReadGCStats.UpdateSince(t)
+
+	debugMetrics.GCStats.LastGC.Update(int64(gcStats.LastGC.UnixNano()))
+	debugMetrics.GCStats.NumGC.Update(int64(gcStats.NumGC))
+	if lastGC != gcStats.LastGC && 0 < len(gcStats.Pause) {
+		debugMetrics.GCStats.Pause.Update(int64(gcStats.Pause[0]))
+	}
+	//debugMetrics.GCStats.PauseQuantiles.Update(gcStats.PauseQuantiles)
+	debugMetrics.GCStats.PauseTotal.Update(int64(gcStats.PauseTotal))
+}
+
+// Register metrics for the Go garbage collector statistics exported in
+// debug.GCStats.  The metrics are named by their fully-qualified Go symbols,
+// i.e. debug.GCStats.PauseTotal.
+func RegisterDebugGCStats(r Registry) {
+	debugMetrics.GCStats.LastGC = NewGauge()
+	debugMetrics.GCStats.NumGC = NewGauge()
+	debugMetrics.GCStats.Pause = NewHistogram(NewExpDecaySample(1028, 0.015))
+	//debugMetrics.GCStats.PauseQuantiles = NewHistogram(NewExpDecaySample(1028, 0.015))
+	debugMetrics.GCStats.PauseTotal = NewGauge()
+	debugMetrics.ReadGCStats = NewTimer()
+
+	r.Register("debug.GCStats.LastGC", debugMetrics.GCStats.LastGC)
+	r.Register("debug.GCStats.NumGC", debugMetrics.GCStats.NumGC)
+	r.Register("debug.GCStats.Pause", debugMetrics.GCStats.Pause)
+	//r.Register("debug.GCStats.PauseQuantiles", debugMetrics.GCStats.PauseQuantiles)
+	r.Register("debug.GCStats.PauseTotal", debugMetrics.GCStats.PauseTotal)
+	r.Register("debug.ReadGCStats", debugMetrics.ReadGCStats)
+}
+
+// Allocate an initial slice for gcStats.Pause to avoid allocations during
+// normal operation.
+func init() {
+	gcStats.Pause = make([]time.Duration, 11)
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/ewma.go

@@ -0,0 +1,118 @@
+package metrics
+
+import (
+	"math"
+	"sync"
+	"sync/atomic"
+)
+
+// EWMAs continuously calculate an exponentially-weighted moving average
+// based on an outside source of clock ticks.
+type EWMA interface {
+	Rate() float64
+	Snapshot() EWMA
+	Tick()
+	Update(int64)
+}
+
+// NewEWMA constructs a new EWMA with the given alpha.
+func NewEWMA(alpha float64) EWMA {
+	if UseNilMetrics {
+		return NilEWMA{}
+	}
+	return &StandardEWMA{alpha: alpha}
+}
+
+// NewEWMA1 constructs a new EWMA for a one-minute moving average.
+func NewEWMA1() EWMA {
+	return NewEWMA(1 - math.Exp(-5.0/60.0/1))
+}
+
+// NewEWMA5 constructs a new EWMA for a five-minute moving average.
+func NewEWMA5() EWMA {
+	return NewEWMA(1 - math.Exp(-5.0/60.0/5))
+}
+
+// NewEWMA15 constructs a new EWMA for a fifteen-minute moving average.
+func NewEWMA15() EWMA {
+	return NewEWMA(1 - math.Exp(-5.0/60.0/15))
+}
+
+// EWMASnapshot is a read-only copy of another EWMA.
+type EWMASnapshot float64
+
+// Rate returns the rate of events per second at the time the snapshot was
+// taken.
+func (a EWMASnapshot) Rate() float64 { return float64(a) }
+
+// Snapshot returns the snapshot.
+func (a EWMASnapshot) Snapshot() EWMA { return a }
+
+// Tick panics.
+func (EWMASnapshot) Tick() {
+	panic("Tick called on an EWMASnapshot")
+}
+
+// Update panics.
+func (EWMASnapshot) Update(int64) {
+	panic("Update called on an EWMASnapshot")
+}
+
+// NilEWMA is a no-op EWMA.
+type NilEWMA struct{}
+
+// Rate is a no-op.
+func (NilEWMA) Rate() float64 { return 0.0 }
+
+// Snapshot is a no-op.
+func (NilEWMA) Snapshot() EWMA { return NilEWMA{} }
+
+// Tick is a no-op.
+func (NilEWMA) Tick() {}
+
+// Update is a no-op.
+func (NilEWMA) Update(n int64) {}
+
+// StandardEWMA is the standard implementation of an EWMA and tracks the number
+// of uncounted events and processes them on each tick.  It uses the
+// sync/atomic package to manage uncounted events.
+type StandardEWMA struct {
+	uncounted int64 // /!\ this should be the first member to ensure 64-bit alignment
+	alpha     float64
+	rate      float64
+	init      bool
+	mutex     sync.Mutex
+}
+
+// Rate returns the moving average rate of events per second.
+func (a *StandardEWMA) Rate() float64 {
+	a.mutex.Lock()
+	defer a.mutex.Unlock()
+	return a.rate * float64(1e9)
+}
+
+// Snapshot returns a read-only copy of the EWMA.
+func (a *StandardEWMA) Snapshot() EWMA {
+	return EWMASnapshot(a.Rate())
+}
+
+// Tick ticks the clock to update the moving average.  It assumes it is called
+// every five seconds.
+func (a *StandardEWMA) Tick() {
+	count := atomic.LoadInt64(&a.uncounted)
+	atomic.AddInt64(&a.uncounted, -count)
+	instantRate := float64(count) / float64(5e9)
+	a.mutex.Lock()
+	defer a.mutex.Unlock()
+	if a.init {
+		a.rate += a.alpha * (instantRate - a.rate)
+	} else {
+		a.init = true
+		a.rate = instantRate
+	}
+}
+
+// Update adds n uncounted events.
+func (a *StandardEWMA) Update(n int64) {
+	atomic.AddInt64(&a.uncounted, n)
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/gauge.go

@@ -0,0 +1,84 @@
+package metrics
+
+import "sync/atomic"
+
+// Gauges hold an int64 value that can be set arbitrarily.
+type Gauge interface {
+	Snapshot() Gauge
+	Update(int64)
+	Value() int64
+}
+
+// GetOrRegisterGauge returns an existing Gauge or constructs and registers a
+// new StandardGauge.
+func GetOrRegisterGauge(name string, r Registry) Gauge {
+	if nil == r {
+		r = DefaultRegistry
+	}
+	return r.GetOrRegister(name, NewGauge).(Gauge)
+}
+
+// NewGauge constructs a new StandardGauge.
+func NewGauge() Gauge {
+	if UseNilMetrics {
+		return NilGauge{}
+	}
+	return &StandardGauge{0}
+}
+
+// NewRegisteredGauge constructs and registers a new StandardGauge.
+func NewRegisteredGauge(name string, r Registry) Gauge {
+	c := NewGauge()
+	if nil == r {
+		r = DefaultRegistry
+	}
+	r.Register(name, c)
+	return c
+}
+
+// GaugeSnapshot is a read-only copy of another Gauge.
+type GaugeSnapshot int64
+
+// Snapshot returns the snapshot.
+func (g GaugeSnapshot) Snapshot() Gauge { return g }
+
+// Update panics.
+func (GaugeSnapshot) Update(int64) {
+	panic("Update called on a GaugeSnapshot")
+}
+
+// Value returns the value at the time the snapshot was taken.
+func (g GaugeSnapshot) Value() int64 { return int64(g) }
+
+// NilGauge is a no-op Gauge.
+type NilGauge struct{}
+
+// Snapshot is a no-op.
+func (NilGauge) Snapshot() Gauge { return NilGauge{} }
+
+// Update is a no-op.
+func (NilGauge) Update(v int64) {}
+
+// Value is a no-op.
+func (NilGauge) Value() int64 { return 0 }
+
+// StandardGauge is the standard implementation of a Gauge and uses the
+// sync/atomic package to manage a single int64 value.
+type StandardGauge struct {
+	value int64
+}
+
+// Snapshot returns a read-only copy of the gauge.
+func (g *StandardGauge) Snapshot() Gauge {
+	return GaugeSnapshot(g.Value())
+}
+
+// Update updates the gauge's value.
+func (g *StandardGauge) Update(v int64) {
+	atomic.StoreInt64(&g.value, v)
+}
+
+// Value returns the gauge's current value.
+func (g *StandardGauge) Value() int64 {
+	return atomic.LoadInt64(&g.value)
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/gauge_float64.go

@@ -0,0 +1,91 @@
+package metrics
+
+import "sync"
+
+// GaugeFloat64s hold a float64 value that can be set arbitrarily.
+type GaugeFloat64 interface {
+	Snapshot() GaugeFloat64
+	Update(float64)
+	Value() float64
+}
+
+// GetOrRegisterGaugeFloat64 returns an existing GaugeFloat64 or constructs and registers a
+// new StandardGaugeFloat64.
+func GetOrRegisterGaugeFloat64(name string, r Registry) GaugeFloat64 {
+	if nil == r {
+		r = DefaultRegistry
+	}
+	return r.GetOrRegister(name, NewGaugeFloat64()).(GaugeFloat64)
+}
+
+// NewGaugeFloat64 constructs a new StandardGaugeFloat64.
+func NewGaugeFloat64() GaugeFloat64 {
+	if UseNilMetrics {
+		return NilGaugeFloat64{}
+	}
+	return &StandardGaugeFloat64{
+		value: 0.0,
+	}
+}
+
+// NewRegisteredGaugeFloat64 constructs and registers a new StandardGaugeFloat64.
+func NewRegisteredGaugeFloat64(name string, r Registry) GaugeFloat64 {
+	c := NewGaugeFloat64()
+	if nil == r {
+		r = DefaultRegistry
+	}
+	r.Register(name, c)
+	return c
+}
+
+// GaugeFloat64Snapshot is a read-only copy of another GaugeFloat64.
+type GaugeFloat64Snapshot float64
+
+// Snapshot returns the snapshot.
+func (g GaugeFloat64Snapshot) Snapshot() GaugeFloat64 { return g }
+
+// Update panics.
+func (GaugeFloat64Snapshot) Update(float64) {
+	panic("Update called on a GaugeFloat64Snapshot")
+}
+
+// Value returns the value at the time the snapshot was taken.
+func (g GaugeFloat64Snapshot) Value() float64 { return float64(g) }
+
+// NilGauge is a no-op Gauge.
+type NilGaugeFloat64 struct{}
+
+// Snapshot is a no-op.
+func (NilGaugeFloat64) Snapshot() GaugeFloat64 { return NilGaugeFloat64{} }
+
+// Update is a no-op.
+func (NilGaugeFloat64) Update(v float64) {}
+
+// Value is a no-op.
+func (NilGaugeFloat64) Value() float64 { return 0.0 }
+
+// StandardGaugeFloat64 is the standard implementation of a GaugeFloat64 and uses
+// sync.Mutex to manage a single float64 value.
+type StandardGaugeFloat64 struct {
+	mutex sync.Mutex
+	value float64
+}
+
+// Snapshot returns a read-only copy of the gauge.
+func (g *StandardGaugeFloat64) Snapshot() GaugeFloat64 {
+	return GaugeFloat64Snapshot(g.Value())
+}
+
+// Update updates the gauge's value.
+func (g *StandardGaugeFloat64) Update(v float64) {
+	g.mutex.Lock()
+	defer g.mutex.Unlock()
+	g.value = v
+}
+
+// Value returns the gauge's current value.
+func (g *StandardGaugeFloat64) Value() float64 {
+	g.mutex.Lock()
+	defer g.mutex.Unlock()
+	return g.value
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/graphite.go

@@ -0,0 +1,113 @@
+package metrics
+
+import (
+	"bufio"
+	"fmt"
+	"log"
+	"net"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// GraphiteConfig provides a container with configuration parameters for
+// the Graphite exporter
+type GraphiteConfig struct {
+	Addr          *net.TCPAddr  // Network address to connect to
+	Registry      Registry      // Registry to be exported
+	FlushInterval time.Duration // Flush interval
+	DurationUnit  time.Duration // Time conversion unit for durations
+	Prefix        string        // Prefix to be prepended to metric names
+	Percentiles   []float64     // Percentiles to export from timers and histograms
+}
+
+// Graphite is a blocking exporter function which reports metrics in r
+// to a graphite server located at addr, flushing them every d duration
+// and prepending metric names with prefix.
+func Graphite(r Registry, d time.Duration, prefix string, addr *net.TCPAddr) {
+	GraphiteWithConfig(GraphiteConfig{
+		Addr:          addr,
+		Registry:      r,
+		FlushInterval: d,
+		DurationUnit:  time.Nanosecond,
+		Prefix:        prefix,
+		Percentiles:   []float64{0.5, 0.75, 0.95, 0.99, 0.999},
+	})
+}
+
+// GraphiteWithConfig is a blocking exporter function just like Graphite,
+// but it takes a GraphiteConfig instead.
+func GraphiteWithConfig(c GraphiteConfig) {
+	log.Printf("WARNING: This go-metrics client has been DEPRECATED! It has been moved to https://github.com/cyberdelia/go-metrics-graphite and will be removed from rcrowley/go-metrics on August 12th 2015")
+	for _ = range time.Tick(c.FlushInterval) {
+		if err := graphite(&c); nil != err {
+			log.Println(err)
+		}
+	}
+}
+
+// GraphiteOnce performs a single submission to Graphite, returning a
+// non-nil error on failed connections. This can be used in a loop
+// similar to GraphiteWithConfig for custom error handling.
+func GraphiteOnce(c GraphiteConfig) error {
+	log.Printf("WARNING: This go-metrics client has been DEPRECATED! It has been moved to https://github.com/cyberdelia/go-metrics-graphite and will be removed from rcrowley/go-metrics on August 12th 2015")
+	return graphite(&c)
+}
+
+func graphite(c *GraphiteConfig) error {
+	now := time.Now().Unix()
+	du := float64(c.DurationUnit)
+	conn, err := net.DialTCP("tcp", nil, c.Addr)
+	if nil != err {
+		return err
+	}
+	defer conn.Close()
+	w := bufio.NewWriter(conn)
+	c.Registry.Each(func(name string, i interface{}) {
+		switch metric := i.(type) {
+		case Counter:
+			fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, metric.Count(), now)
+		case Gauge:
+			fmt.Fprintf(w, "%s.%s.value %d %d\n", c.Prefix, name, metric.Value(), now)
+		case GaugeFloat64:
+			fmt.Fprintf(w, "%s.%s.value %f %d\n", c.Prefix, name, metric.Value(), now)
+		case Histogram:
+			h := metric.Snapshot()
+			ps := h.Percentiles(c.Percentiles)
+			fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, h.Count(), now)
+			fmt.Fprintf(w, "%s.%s.min %d %d\n", c.Prefix, name, h.Min(), now)
+			fmt.Fprintf(w, "%s.%s.max %d %d\n", c.Prefix, name, h.Max(), now)
+			fmt.Fprintf(w, "%s.%s.mean %.2f %d\n", c.Prefix, name, h.Mean(), now)
+			fmt.Fprintf(w, "%s.%s.std-dev %.2f %d\n", c.Prefix, name, h.StdDev(), now)
+			for psIdx, psKey := range c.Percentiles {
+				key := strings.Replace(strconv.FormatFloat(psKey*100.0, 'f', -1, 64), ".", "", 1)
+				fmt.Fprintf(w, "%s.%s.%s-percentile %.2f %d\n", c.Prefix, name, key, ps[psIdx], now)
+			}
+		case Meter:
+			m := metric.Snapshot()
+			fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, m.Count(), now)
+			fmt.Fprintf(w, "%s.%s.one-minute %.2f %d\n", c.Prefix, name, m.Rate1(), now)
+			fmt.Fprintf(w, "%s.%s.five-minute %.2f %d\n", c.Prefix, name, m.Rate5(), now)
+			fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, m.Rate15(), now)
+			fmt.Fprintf(w, "%s.%s.mean %.2f %d\n", c.Prefix, name, m.RateMean(), now)
+		case Timer:
+			t := metric.Snapshot()
+			ps := t.Percentiles(c.Percentiles)
+			fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, t.Count(), now)
+			fmt.Fprintf(w, "%s.%s.min %d %d\n", c.Prefix, name, t.Min()/int64(du), now)
+			fmt.Fprintf(w, "%s.%s.max %d %d\n", c.Prefix, name, t.Max()/int64(du), now)
+			fmt.Fprintf(w, "%s.%s.mean %.2f %d\n", c.Prefix, name, t.Mean()/du, now)
+			fmt.Fprintf(w, "%s.%s.std-dev %.2f %d\n", c.Prefix, name, t.StdDev()/du, now)
+			for psIdx, psKey := range c.Percentiles {
+				key := strings.Replace(strconv.FormatFloat(psKey*100.0, 'f', -1, 64), ".", "", 1)
+				fmt.Fprintf(w, "%s.%s.%s-percentile %.2f %d\n", c.Prefix, name, key, ps[psIdx], now)
+			}
+			fmt.Fprintf(w, "%s.%s.one-minute %.2f %d\n", c.Prefix, name, t.Rate1(), now)
+			fmt.Fprintf(w, "%s.%s.five-minute %.2f %d\n", c.Prefix, name, t.Rate5(), now)
+			fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, t.Rate15(), now)
+			fmt.Fprintf(w, "%s.%s.mean-rate %.2f %d\n", c.Prefix, name, t.RateMean(), now)
+		}
+		w.Flush()
+	})
+	return nil
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/healthcheck.go

@@ -0,0 +1,61 @@
+package metrics
+
+// Healthchecks hold an error value describing an arbitrary up/down status.
+type Healthcheck interface {
+	Check()
+	Error() error
+	Healthy()
+	Unhealthy(error)
+}
+
+// NewHealthcheck constructs a new Healthcheck which will use the given
+// function to update its status.
+func NewHealthcheck(f func(Healthcheck)) Healthcheck {
+	if UseNilMetrics {
+		return NilHealthcheck{}
+	}
+	return &StandardHealthcheck{nil, f}
+}
+
+// NilHealthcheck is a no-op.
+type NilHealthcheck struct{}
+
+// Check is a no-op.
+func (NilHealthcheck) Check() {}
+
+// Error is a no-op.
+func (NilHealthcheck) Error() error { return nil }
+
+// Healthy is a no-op.
+func (NilHealthcheck) Healthy() {}
+
+// Unhealthy is a no-op.
+func (NilHealthcheck) Unhealthy(error) {}
+
+// StandardHealthcheck is the standard implementation of a Healthcheck and
+// stores the status and a function to call to update the status.
+type StandardHealthcheck struct {
+	err error
+	f   func(Healthcheck)
+}
+
+// Check runs the healthcheck function to update the healthcheck's status.
+func (h *StandardHealthcheck) Check() {
+	h.f(h)
+}
+
+// Error returns the healthcheck's status, which will be nil if it is healthy.
+func (h *StandardHealthcheck) Error() error {
+	return h.err
+}
+
+// Healthy marks the healthcheck as healthy.
+func (h *StandardHealthcheck) Healthy() {
+	h.err = nil
+}
+
+// Unhealthy marks the healthcheck as unhealthy.  The error is stored and
+// may be retrieved by the Error method.
+func (h *StandardHealthcheck) Unhealthy(err error) {
+	h.err = err
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/histogram.go

@@ -0,0 +1,202 @@
+package metrics
+
+// Histograms calculate distribution statistics from a series of int64 values.
+type Histogram interface {
+	Clear()
+	Count() int64
+	Max() int64
+	Mean() float64
+	Min() int64
+	Percentile(float64) float64
+	Percentiles([]float64) []float64
+	Sample() Sample
+	Snapshot() Histogram
+	StdDev() float64
+	Sum() int64
+	Update(int64)
+	Variance() float64
+}
+
+// GetOrRegisterHistogram returns an existing Histogram or constructs and
+// registers a new StandardHistogram.
+func GetOrRegisterHistogram(name string, r Registry, s Sample) Histogram {
+	if nil == r {
+		r = DefaultRegistry
+	}
+	return r.GetOrRegister(name, func() Histogram { return NewHistogram(s) }).(Histogram)
+}
+
+// NewHistogram constructs a new StandardHistogram from a Sample.
+func NewHistogram(s Sample) Histogram {
+	if UseNilMetrics {
+		return NilHistogram{}
+	}
+	return &StandardHistogram{sample: s}
+}
+
+// NewRegisteredHistogram constructs and registers a new StandardHistogram from
+// a Sample.
+func NewRegisteredHistogram(name string, r Registry, s Sample) Histogram {
+	c := NewHistogram(s)
+	if nil == r {
+		r = DefaultRegistry
+	}
+	r.Register(name, c)
+	return c
+}
+
+// HistogramSnapshot is a read-only copy of another Histogram.
+type HistogramSnapshot struct {
+	sample *SampleSnapshot
+}
+
+// Clear panics.
+func (*HistogramSnapshot) Clear() {
+	panic("Clear called on a HistogramSnapshot")
+}
+
+// Count returns the number of samples recorded at the time the snapshot was
+// taken.
+func (h *HistogramSnapshot) Count() int64 { return h.sample.Count() }
+
+// Max returns the maximum value in the sample at the time the snapshot was
+// taken.
+func (h *HistogramSnapshot) Max() int64 { return h.sample.Max() }
+
+// Mean returns the mean of the values in the sample at the time the snapshot
+// was taken.
+func (h *HistogramSnapshot) Mean() float64 { return h.sample.Mean() }
+
+// Min returns the minimum value in the sample at the time the snapshot was
+// taken.
+func (h *HistogramSnapshot) Min() int64 { return h.sample.Min() }
+
+// Percentile returns an arbitrary percentile of values in the sample at the
+// time the snapshot was taken.
+func (h *HistogramSnapshot) Percentile(p float64) float64 {
+	return h.sample.Percentile(p)
+}
+
+// Percentiles returns a slice of arbitrary percentiles of values in the sample
+// at the time the snapshot was taken.
+func (h *HistogramSnapshot) Percentiles(ps []float64) []float64 {
+	return h.sample.Percentiles(ps)
+}
+
+// Sample returns the Sample underlying the histogram.
+func (h *HistogramSnapshot) Sample() Sample { return h.sample }
+
+// Snapshot returns the snapshot.
+func (h *HistogramSnapshot) Snapshot() Histogram { return h }
+
+// StdDev returns the standard deviation of the values in the sample at the
+// time the snapshot was taken.
+func (h *HistogramSnapshot) StdDev() float64 { return h.sample.StdDev() }
+
+// Sum returns the sum in the sample at the time the snapshot was taken.
+func (h *HistogramSnapshot) Sum() int64 { return h.sample.Sum() }
+
+// Update panics.
+func (*HistogramSnapshot) Update(int64) {
+	panic("Update called on a HistogramSnapshot")
+}
+
+// Variance returns the variance of inputs at the time the snapshot was taken.
+func (h *HistogramSnapshot) Variance() float64 { return h.sample.Variance() }
+
+// NilHistogram is a no-op Histogram.
+type NilHistogram struct{}
+
+// Clear is a no-op.
+func (NilHistogram) Clear() {}
+
+// Count is a no-op.
+func (NilHistogram) Count() int64 { return 0 }
+
+// Max is a no-op.
+func (NilHistogram) Max() int64 { return 0 }
+
+// Mean is a no-op.
+func (NilHistogram) Mean() float64 { return 0.0 }
+
+// Min is a no-op.
+func (NilHistogram) Min() int64 { return 0 }
+
+// Percentile is a no-op.
+func (NilHistogram) Percentile(p float64) float64 { return 0.0 }
+
+// Percentiles is a no-op.
+func (NilHistogram) Percentiles(ps []float64) []float64 {
+	return make([]float64, len(ps))
+}
+
+// Sample is a no-op.
+func (NilHistogram) Sample() Sample { return NilSample{} }
+
+// Snapshot is a no-op.
+func (NilHistogram) Snapshot() Histogram { return NilHistogram{} }
+
+// StdDev is a no-op.
+func (NilHistogram) StdDev() float64 { return 0.0 }
+
+// Sum is a no-op.
+func (NilHistogram) Sum() int64 { return 0 }
+
+// Update is a no-op.
+func (NilHistogram) Update(v int64) {}
+
+// Variance is a no-op.
+func (NilHistogram) Variance() float64 { return 0.0 }
+
+// StandardHistogram is the standard implementation of a Histogram and uses a
+// Sample to bound its memory use.
+type StandardHistogram struct {
+	sample Sample
+}
+
+// Clear clears the histogram and its sample.
+func (h *StandardHistogram) Clear() { h.sample.Clear() }
+
+// Count returns the number of samples recorded since the histogram was last
+// cleared.
+func (h *StandardHistogram) Count() int64 { return h.sample.Count() }
+
+// Max returns the maximum value in the sample.
+func (h *StandardHistogram) Max() int64 { return h.sample.Max() }
+
+// Mean returns the mean of the values in the sample.
+func (h *StandardHistogram) Mean() float64 { return h.sample.Mean() }
+
+// Min returns the minimum value in the sample.
+func (h *StandardHistogram) Min() int64 { return h.sample.Min() }
+
+// Percentile returns an arbitrary percentile of the values in the sample.
+func (h *StandardHistogram) Percentile(p float64) float64 {
+	return h.sample.Percentile(p)
+}
+
+// Percentiles returns a slice of arbitrary percentiles of the values in the
+// sample.
+func (h *StandardHistogram) Percentiles(ps []float64) []float64 {
+	return h.sample.Percentiles(ps)
+}
+
+// Sample returns the Sample underlying the histogram.
+func (h *StandardHistogram) Sample() Sample { return h.sample }
+
+// Snapshot returns a read-only copy of the histogram.
+func (h *StandardHistogram) Snapshot() Histogram {
+	return &HistogramSnapshot{sample: h.sample.Snapshot().(*SampleSnapshot)}
+}
+
+// StdDev returns the standard deviation of the values in the sample.
+func (h *StandardHistogram) StdDev() float64 { return h.sample.StdDev() }
+
+// Sum returns the sum in the sample.
+func (h *StandardHistogram) Sum() int64 { return h.sample.Sum() }
+
+// Update samples a new value.
+func (h *StandardHistogram) Update(v int64) { h.sample.Update(v) }
+
+// Variance returns the variance of the values in the sample.
+func (h *StandardHistogram) Variance() float64 { return h.sample.Variance() }

Godeps/_workspace/src/github.com/rcrowley/go-metrics/json.go

@@ -0,0 +1,83 @@
+package metrics
+
+import (
+	"encoding/json"
+	"io"
+	"time"
+)
+
+// MarshalJSON returns a byte slice containing a JSON representation of all
+// the metrics in the Registry.
+func (r *StandardRegistry) MarshalJSON() ([]byte, error) {
+	data := make(map[string]map[string]interface{})
+	r.Each(func(name string, i interface{}) {
+		values := make(map[string]interface{})
+		switch metric := i.(type) {
+		case Counter:
+			values["count"] = metric.Count()
+		case Gauge:
+			values["value"] = metric.Value()
+		case GaugeFloat64:
+			values["value"] = metric.Value()
+		case Healthcheck:
+			values["error"] = nil
+			metric.Check()
+			if err := metric.Error(); nil != err {
+				values["error"] = metric.Error().Error()
+			}
+		case Histogram:
+			h := metric.Snapshot()
+			ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
+			values["count"] = h.Count()
+			values["min"] = h.Min()
+			values["max"] = h.Max()
+			values["mean"] = h.Mean()
+			values["stddev"] = h.StdDev()
+			values["median"] = ps[0]
+			values["75%"] = ps[1]
+			values["95%"] = ps[2]
+			values["99%"] = ps[3]
+			values["99.9%"] = ps[4]
+		case Meter:
+			m := metric.Snapshot()
+			values["count"] = m.Count()
+			values["1m.rate"] = m.Rate1()
+			values["5m.rate"] = m.Rate5()
+			values["15m.rate"] = m.Rate15()
+			values["mean.rate"] = m.RateMean()
+		case Timer:
+			t := metric.Snapshot()
+			ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
+			values["count"] = t.Count()
+			values["min"] = t.Min()
+			values["max"] = t.Max()
+			values["mean"] = t.Mean()
+			values["stddev"] = t.StdDev()
+			values["median"] = ps[0]
+			values["75%"] = ps[1]
+			values["95%"] = ps[2]
+			values["99%"] = ps[3]
+			values["99.9%"] = ps[4]
+			values["1m.rate"] = t.Rate1()
+			values["5m.rate"] = t.Rate5()
+			values["15m.rate"] = t.Rate15()
+			values["mean.rate"] = t.RateMean()
+		}
+		data[name] = values
+	})
+	return json.Marshal(data)
+}
+
+// WriteJSON writes metrics from the given registry  periodically to the
+// specified io.Writer as JSON.
+func WriteJSON(r Registry, d time.Duration, w io.Writer) {
+	for _ = range time.Tick(d) {
+		WriteJSONOnce(r, w)
+	}
+}
+
+// WriteJSONOnce writes metrics from the given registry to the specified
+// io.Writer as JSON.
+func WriteJSONOnce(r Registry, w io.Writer) {
+	json.NewEncoder(w).Encode(r)
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/librato/client.go

@@ -0,0 +1,102 @@
+package librato
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+)
+
+const Operations = "operations"
+const OperationsShort = "ops"
+
+type LibratoClient struct {
+	Email, Token string
+}
+
+// property strings
+const (
+	// display attributes
+	Color             = "color"
+	DisplayMax        = "display_max"
+	DisplayMin        = "display_min"
+	DisplayUnitsLong  = "display_units_long"
+	DisplayUnitsShort = "display_units_short"
+	DisplayStacked    = "display_stacked"
+	DisplayTransform  = "display_transform"
+	// special gauge display attributes
+	SummarizeFunction = "summarize_function"
+	Aggregate         = "aggregate"
+
+	// metric keys
+	Name        = "name"
+	Period      = "period"
+	Description = "description"
+	DisplayName = "display_name"
+	Attributes  = "attributes"
+
+	// measurement keys
+	MeasureTime = "measure_time"
+	Source      = "source"
+	Value       = "value"
+
+	// special gauge keys
+	Count      = "count"
+	Sum        = "sum"
+	Max        = "max"
+	Min        = "min"
+	SumSquares = "sum_squares"
+
+	// batch keys
+	Counters = "counters"
+	Gauges   = "gauges"
+
+	MetricsPostUrl = "https://metrics-api.librato.com/v1/metrics"
+)
+
+type Measurement map[string]interface{}
+type Metric map[string]interface{}
+
+type Batch struct {
+	Gauges      []Measurement `json:"gauges,omitempty"`
+	Counters    []Measurement `json:"counters,omitempty"`
+	MeasureTime int64         `json:"measure_time"`
+	Source      string        `json:"source"`
+}
+
+func (self *LibratoClient) PostMetrics(batch Batch) (err error) {
+	var (
+		js   []byte
+		req  *http.Request
+		resp *http.Response
+	)
+
+	if len(batch.Counters) == 0 && len(batch.Gauges) == 0 {
+		return nil
+	}
+
+	if js, err = json.Marshal(batch); err != nil {
+		return
+	}
+
+	if req, err = http.NewRequest("POST", MetricsPostUrl, bytes.NewBuffer(js)); err != nil {
+		return
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+	req.SetBasicAuth(self.Email, self.Token)
+
+	if resp, err = http.DefaultClient.Do(req); err != nil {
+		return
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		var body []byte
+		if body, err = ioutil.ReadAll(resp.Body); err != nil {
+			body = []byte(fmt.Sprintf("(could not fetch response body for error: %s)", err))
+		}
+		err = fmt.Errorf("Unable to post to Librato: %d %s %s", resp.StatusCode, resp.Status, string(body))
+	}
+	return
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/librato/librato.go

@@ -0,0 +1,231 @@
+package librato
+
+import (
+	"fmt"
+	"log"
+	"math"
+	"regexp"
+	"time"
+
+	"github.com/rcrowley/go-metrics"
+)
+
+// a regexp for extracting the unit from time.Duration.String
+var unitRegexp = regexp.MustCompile("[^\\d]+$")
+
+// a helper that turns a time.Duration into librato display attributes for timer metrics
+func translateTimerAttributes(d time.Duration) (attrs map[string]interface{}) {
+	attrs = make(map[string]interface{})
+	attrs[DisplayTransform] = fmt.Sprintf("x/%d", int64(d))
+	attrs[DisplayUnitsShort] = string(unitRegexp.Find([]byte(d.String())))
+	return
+}
+
+type Reporter struct {
+	Email, Token    string
+	Source          string
+	Interval        time.Duration
+	Registry        metrics.Registry
+	Percentiles     []float64              // percentiles to report on histogram metrics
+	TimerAttributes map[string]interface{} // units in which timers will be displayed
+	intervalSec     int64
+}
+
+func NewReporter(r metrics.Registry, d time.Duration, e string, t string, s string, p []float64, u time.Duration) *Reporter {
+	return &Reporter{e, t, s, d, r, p, translateTimerAttributes(u), int64(d / time.Second)}
+}
+
+func Librato(r metrics.Registry, d time.Duration, e string, t string, s string, p []float64, u time.Duration) {
+	NewReporter(r, d, e, t, s, p, u).Run()
+}
+
+func (self *Reporter) Run() {
+	log.Printf("WARNING: This client has been DEPRECATED! It has been moved to https://github.com/mihasya/go-metrics-librato and will be removed from rcrowley/go-metrics on August 5th 2015")
+	ticker := time.Tick(self.Interval)
+	metricsApi := &LibratoClient{self.Email, self.Token}
+	for now := range ticker {
+		var metrics Batch
+		var err error
+		if metrics, err = self.BuildRequest(now, self.Registry); err != nil {
+			log.Printf("ERROR constructing librato request body %s", err)
+			continue
+		}
+		if err := metricsApi.PostMetrics(metrics); err != nil {
+			log.Printf("ERROR sending metrics to librato %s", err)
+			continue
+		}
+	}
+}
+
+// calculate sum of squares from data provided by metrics.Histogram
+// see http://en.wikipedia.org/wiki/Standard_deviation#Rapid_calculation_methods
+func sumSquares(s metrics.Sample) float64 {
+	count := float64(s.Count())
+	sumSquared := math.Pow(count*s.Mean(), 2)
+	sumSquares := math.Pow(count*s.StdDev(), 2) + sumSquared/count
+	if math.IsNaN(sumSquares) {
+		return 0.0
+	}
+	return sumSquares
+}
+func sumSquaresTimer(t metrics.Timer) float64 {
+	count := float64(t.Count())
+	sumSquared := math.Pow(count*t.Mean(), 2)
+	sumSquares := math.Pow(count*t.StdDev(), 2) + sumSquared/count
+	if math.IsNaN(sumSquares) {
+		return 0.0
+	}
+	return sumSquares
+}
+
+func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot Batch, err error) {
+	snapshot = Batch{
+		// coerce timestamps to a stepping fn so that they line up in Librato graphs
+		MeasureTime: (now.Unix() / self.intervalSec) * self.intervalSec,
+		Source:      self.Source,
+	}
+	snapshot.Gauges = make([]Measurement, 0)
+	snapshot.Counters = make([]Measurement, 0)
+	histogramGaugeCount := 1 + len(self.Percentiles)
+	r.Each(func(name string, metric interface{}) {
+		measurement := Measurement{}
+		measurement[Period] = self.Interval.Seconds()
+		switch m := metric.(type) {
+		case metrics.Counter:
+			if m.Count() > 0 {
+				measurement[Name] = fmt.Sprintf("%s.%s", name, "count")
+				measurement[Value] = float64(m.Count())
+				measurement[Attributes] = map[string]interface{}{
+					DisplayUnitsLong:  Operations,
+					DisplayUnitsShort: OperationsShort,
+					DisplayMin:        "0",
+				}
+				snapshot.Counters = append(snapshot.Counters, measurement)
+			}
+		case metrics.Gauge:
+			measurement[Name] = name
+			measurement[Value] = float64(m.Value())
+			snapshot.Gauges = append(snapshot.Gauges, measurement)
+		case metrics.GaugeFloat64:
+			measurement[Name] = name
+			measurement[Value] = float64(m.Value())
+			snapshot.Gauges = append(snapshot.Gauges, measurement)
+		case metrics.Histogram:
+			if m.Count() > 0 {
+				gauges := make([]Measurement, histogramGaugeCount, histogramGaugeCount)
+				s := m.Sample()
+				measurement[Name] = fmt.Sprintf("%s.%s", name, "hist")
+				measurement[Count] = uint64(s.Count())
+				measurement[Max] = float64(s.Max())
+				measurement[Min] = float64(s.Min())
+				measurement[Sum] = float64(s.Sum())
+				measurement[SumSquares] = sumSquares(s)
+				gauges[0] = measurement
+				for i, p := range self.Percentiles {
+					gauges[i+1] = Measurement{
+						Name:   fmt.Sprintf("%s.%.2f", measurement[Name], p),
+						Value:  s.Percentile(p),
+						Period: measurement[Period],
+					}
+				}
+				snapshot.Gauges = append(snapshot.Gauges, gauges...)
+			}
+		case metrics.Meter:
+			measurement[Name] = name
+			measurement[Value] = float64(m.Count())
+			snapshot.Counters = append(snapshot.Counters, measurement)
+			snapshot.Gauges = append(snapshot.Gauges,
+				Measurement{
+					Name:   fmt.Sprintf("%s.%s", name, "1min"),
+					Value:  m.Rate1(),
+					Period: int64(self.Interval.Seconds()),
+					Attributes: map[string]interface{}{
+						DisplayUnitsLong:  Operations,
+						DisplayUnitsShort: OperationsShort,
+						DisplayMin:        "0",
+					},
+				},
+				Measurement{
+					Name:   fmt.Sprintf("%s.%s", name, "5min"),
+					Value:  m.Rate5(),
+					Period: int64(self.Interval.Seconds()),
+					Attributes: map[string]interface{}{
+						DisplayUnitsLong:  Operations,
+						DisplayUnitsShort: OperationsShort,
+						DisplayMin:        "0",
+					},
+				},
+				Measurement{
+					Name:   fmt.Sprintf("%s.%s", name, "15min"),
+					Value:  m.Rate15(),
+					Period: int64(self.Interval.Seconds()),
+					Attributes: map[string]interface{}{
+						DisplayUnitsLong:  Operations,
+						DisplayUnitsShort: OperationsShort,
+						DisplayMin:        "0",
+					},
+				},
+			)
+		case metrics.Timer:
+			measurement[Name] = name
+			measurement[Value] = float64(m.Count())
+			snapshot.Counters = append(snapshot.Counters, measurement)
+			if m.Count() > 0 {
+				libratoName := fmt.Sprintf("%s.%s", name, "timer.mean")
+				gauges := make([]Measurement, histogramGaugeCount, histogramGaugeCount)
+				gauges[0] = Measurement{
+					Name:       libratoName,
+					Count:      uint64(m.Count()),
+					Sum:        m.Mean() * float64(m.Count()),
+					Max:        float64(m.Max()),
+					Min:        float64(m.Min()),
+					SumSquares: sumSquaresTimer(m),
+					Period:     int64(self.Interval.Seconds()),
+					Attributes: self.TimerAttributes,
+				}
+				for i, p := range self.Percentiles {
+					gauges[i+1] = Measurement{
+						Name:       fmt.Sprintf("%s.timer.%2.0f", name, p*100),
+						Value:      m.Percentile(p),
+						Period:     int64(self.Interval.Seconds()),
+						Attributes: self.TimerAttributes,
+					}
+				}
+				snapshot.Gauges = append(snapshot.Gauges, gauges...)
+				snapshot.Gauges = append(snapshot.Gauges,
+					Measurement{
+						Name:   fmt.Sprintf("%s.%s", name, "rate.1min"),
+						Value:  m.Rate1(),
+						Period: int64(self.Interval.Seconds()),
+						Attributes: map[string]interface{}{
+							DisplayUnitsLong:  Operations,
+							DisplayUnitsShort: OperationsShort,
+							DisplayMin:        "0",
+						},
+					},
+					Measurement{
+						Name:   fmt.Sprintf("%s.%s", name, "rate.5min"),
+						Value:  m.Rate5(),
+						Period: int64(self.Interval.Seconds()),
+						Attributes: map[string]interface{}{
+							DisplayUnitsLong:  Operations,
+							DisplayUnitsShort: OperationsShort,
+							DisplayMin:        "0",
+						},
+					},
+					Measurement{
+						Name:   fmt.Sprintf("%s.%s", name, "rate.15min"),
+						Value:  m.Rate15(),
+						Period: int64(self.Interval.Seconds()),
+						Attributes: map[string]interface{}{
+							DisplayUnitsLong:  Operations,
+							DisplayUnitsShort: OperationsShort,
+							DisplayMin:        "0",
+						},
+					},
+				)
+			}
+		}
+	})
+	return
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/log.go

@@ -0,0 +1,70 @@
+package metrics
+
+import (
+	"log"
+	"time"
+)
+
+// Output each metric in the given registry periodically using the given
+// logger.
+func Log(r Registry, d time.Duration, l *log.Logger) {
+	for _ = range time.Tick(d) {
+		r.Each(func(name string, i interface{}) {
+			switch metric := i.(type) {
+			case Counter:
+				l.Printf("counter %s\n", name)
+				l.Printf("  count:       %9d\n", metric.Count())
+			case Gauge:
+				l.Printf("gauge %s\n", name)
+				l.Printf("  value:       %9d\n", metric.Value())
+			case GaugeFloat64:
+				l.Printf("gauge %s\n", name)
+				l.Printf("  value:       %f\n", metric.Value())
+			case Healthcheck:
+				metric.Check()
+				l.Printf("healthcheck %s\n", name)
+				l.Printf("  error:       %v\n", metric.Error())
+			case Histogram:
+				h := metric.Snapshot()
+				ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
+				l.Printf("histogram %s\n", name)
+				l.Printf("  count:       %9d\n", h.Count())
+				l.Printf("  min:         %9d\n", h.Min())
+				l.Printf("  max:         %9d\n", h.Max())
+				l.Printf("  mean:        %12.2f\n", h.Mean())
+				l.Printf("  stddev:      %12.2f\n", h.StdDev())
+				l.Printf("  median:      %12.2f\n", ps[0])
+				l.Printf("  75%%:         %12.2f\n", ps[1])
+				l.Printf("  95%%:         %12.2f\n", ps[2])
+				l.Printf("  99%%:         %12.2f\n", ps[3])
+				l.Printf("  99.9%%:       %12.2f\n", ps[4])
+			case Meter:
+				m := metric.Snapshot()
+				l.Printf("meter %s\n", name)
+				l.Printf("  count:       %9d\n", m.Count())
+				l.Printf("  1-min rate:  %12.2f\n", m.Rate1())
+				l.Printf("  5-min rate:  %12.2f\n", m.Rate5())
+				l.Printf("  15-min rate: %12.2f\n", m.Rate15())
+				l.Printf("  mean rate:   %12.2f\n", m.RateMean())
+			case Timer:
+				t := metric.Snapshot()
+				ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
+				l.Printf("timer %s\n", name)
+				l.Printf("  count:       %9d\n", t.Count())
+				l.Printf("  min:         %9d\n", t.Min())
+				l.Printf("  max:         %9d\n", t.Max())
+				l.Printf("  mean:        %12.2f\n", t.Mean())
+				l.Printf("  stddev:      %12.2f\n", t.StdDev())
+				l.Printf("  median:      %12.2f\n", ps[0])
+				l.Printf("  75%%:         %12.2f\n", ps[1])
+				l.Printf("  95%%:         %12.2f\n", ps[2])
+				l.Printf("  99%%:         %12.2f\n", ps[3])
+				l.Printf("  99.9%%:       %12.2f\n", ps[4])
+				l.Printf("  1-min rate:  %12.2f\n", t.Rate1())
+				l.Printf("  5-min rate:  %12.2f\n", t.Rate5())
+				l.Printf("  15-min rate: %12.2f\n", t.Rate15())
+				l.Printf("  mean rate:   %12.2f\n", t.RateMean())
+			}
+		})
+	}
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/memory.md

@@ -0,0 +1,285 @@
+Memory usage
+============
+
+(Highly unscientific.)
+
+Command used to gather static memory usage:
+
+```sh
+grep ^Vm "/proc/$(ps fax | grep [m]etrics-bench | awk '{print $1}')/status"
+```
+
+Program used to gather baseline memory usage:
+
+```go
+package main
+
+import "time"
+
+func main() {
+	time.Sleep(600e9)
+}
+```
+
+Baseline
+--------
+
+```
+VmPeak:    42604 kB
+VmSize:    42604 kB
+VmLck:         0 kB
+VmHWM:      1120 kB
+VmRSS:      1120 kB
+VmData:    35460 kB
+VmStk:       136 kB
+VmExe:      1020 kB
+VmLib:      1848 kB
+VmPTE:        36 kB
+VmSwap:        0 kB
+```
+
+Program used to gather metric memory usage (with other metrics being similar):
+
+```go
+package main
+
+import (
+	"fmt"
+	"metrics"
+	"time"
+)
+
+func main() {
+	fmt.Sprintf("foo")
+	metrics.NewRegistry()
+	time.Sleep(600e9)
+}
+```
+
+1000 counters registered
+------------------------
+
+```
+VmPeak:    44016 kB
+VmSize:    44016 kB
+VmLck:         0 kB
+VmHWM:      1928 kB
+VmRSS:      1928 kB
+VmData:    36868 kB
+VmStk:       136 kB
+VmExe:      1024 kB
+VmLib:      1848 kB
+VmPTE:        40 kB
+VmSwap:        0 kB
+```
+
+**1.412 kB virtual, TODO 0.808 kB resident per counter.**
+
+100000 counters registered
+--------------------------
+
+```
+VmPeak:    55024 kB
+VmSize:    55024 kB
+VmLck:         0 kB
+VmHWM:     12440 kB
+VmRSS:     12440 kB
+VmData:    47876 kB
+VmStk:       136 kB
+VmExe:      1024 kB
+VmLib:      1848 kB
+VmPTE:        64 kB
+VmSwap:        0 kB
+```
+
+**0.1242 kB virtual, 0.1132 kB resident per counter.**
+
+1000 gauges registered
+----------------------
+
+```
+VmPeak:    44012 kB
+VmSize:    44012 kB
+VmLck:         0 kB
+VmHWM:      1928 kB
+VmRSS:      1928 kB
+VmData:    36868 kB
+VmStk:       136 kB
+VmExe:      1020 kB
+VmLib:      1848 kB
+VmPTE:        40 kB
+VmSwap:        0 kB
+```
+
+**1.408 kB virtual, 0.808 kB resident per counter.**
+
+100000 gauges registered
+------------------------
+
+```
+VmPeak:    55020 kB
+VmSize:    55020 kB
+VmLck:         0 kB
+VmHWM:     12432 kB
+VmRSS:     12432 kB
+VmData:    47876 kB
+VmStk:       136 kB
+VmExe:      1020 kB
+VmLib:      1848 kB
+VmPTE:        60 kB
+VmSwap:        0 kB
+```
+
+**0.12416 kB virtual, 0.11312 resident per gauge.**
+
+1000 histograms with a uniform sample size of 1028
+--------------------------------------------------
+
+```
+VmPeak:    72272 kB
+VmSize:    72272 kB
+VmLck:         0 kB
+VmHWM:     16204 kB
+VmRSS:     16204 kB
+VmData:    65100 kB
+VmStk:       136 kB
+VmExe:      1048 kB
+VmLib:      1848 kB
+VmPTE:        80 kB
+VmSwap:        0 kB
+```
+
+**29.668 kB virtual, TODO 15.084 resident per histogram.**
+
+10000 histograms with a uniform sample size of 1028
+---------------------------------------------------
+
+```
+VmPeak:   256912 kB
+VmSize:   256912 kB
+VmLck:         0 kB
+VmHWM:    146204 kB
+VmRSS:    146204 kB
+VmData:   249740 kB
+VmStk:       136 kB
+VmExe:      1048 kB
+VmLib:      1848 kB
+VmPTE:       448 kB
+VmSwap:        0 kB
+```
+
+**21.4308 kB virtual, 14.5084 kB resident per histogram.**
+
+50000 histograms with a uniform sample size of 1028
+---------------------------------------------------
+
+```
+VmPeak:   908112 kB
+VmSize:   908112 kB
+VmLck:         0 kB
+VmHWM:    645832 kB
+VmRSS:    645588 kB
+VmData:   900940 kB
+VmStk:       136 kB
+VmExe:      1048 kB
+VmLib:      1848 kB
+VmPTE:      1716 kB
+VmSwap:     1544 kB
+```
+
+**17.31016 kB virtual, 12.88936 kB resident per histogram.**
+
+1000 histograms with an exponentially-decaying sample size of 1028 and alpha of 0.015
+-------------------------------------------------------------------------------------
+
+```
+VmPeak:    62480 kB
+VmSize:    62480 kB
+VmLck:         0 kB
+VmHWM:     11572 kB
+VmRSS:     11572 kB
+VmData:    55308 kB
+VmStk:       136 kB
+VmExe:      1048 kB
+VmLib:      1848 kB
+VmPTE:        64 kB
+VmSwap:        0 kB
+```
+
+**19.876 kB virtual, 10.452 kB resident per histogram.**
+
+10000 histograms with an exponentially-decaying sample size of 1028 and alpha of 0.015
+--------------------------------------------------------------------------------------
+
+```
+VmPeak:   153296 kB
+VmSize:   153296 kB
+VmLck:         0 kB
+VmHWM:    101176 kB
+VmRSS:    101176 kB
+VmData:   146124 kB
+VmStk:       136 kB
+VmExe:      1048 kB
+VmLib:      1848 kB
+VmPTE:       240 kB
+VmSwap:        0 kB
+```
+
+**11.0692 kB virtual, 10.0056 kB resident per histogram.**
+
+50000 histograms with an exponentially-decaying sample size of 1028 and alpha of 0.015
+--------------------------------------------------------------------------------------
+
+```
+VmPeak:   557264 kB
+VmSize:   557264 kB
+VmLck:         0 kB
+VmHWM:    501056 kB
+VmRSS:    501056 kB
+VmData:   550092 kB
+VmStk:       136 kB
+VmExe:      1048 kB
+VmLib:      1848 kB
+VmPTE:      1032 kB
+VmSwap:        0 kB
+```
+
+**10.2932 kB virtual, 9.99872 kB resident per histogram.**
+
+1000 meters
+-----------
+
+```
+VmPeak:    74504 kB
+VmSize:    74504 kB
+VmLck:         0 kB
+VmHWM:     24124 kB
+VmRSS:     24124 kB
+VmData:    67340 kB
+VmStk:       136 kB
+VmExe:      1040 kB
+VmLib:      1848 kB
+VmPTE:        92 kB
+VmSwap:        0 kB
+```
+
+**31.9 kB virtual, 23.004 kB resident per meter.**
+
+10000 meters
+------------
+
+```
+VmPeak:   278920 kB
+VmSize:   278920 kB
+VmLck:         0 kB
+VmHWM:    227300 kB
+VmRSS:    227300 kB
+VmData:   271756 kB
+VmStk:       136 kB
+VmExe:      1040 kB
+VmLib:      1848 kB
+VmPTE:       488 kB
+VmSwap:        0 kB
+```
+
+**23.6316 kB virtual, 22.618 kB resident per meter.**

Godeps/_workspace/src/github.com/rcrowley/go-metrics/meter.go

@@ -0,0 +1,233 @@
+package metrics
+
+import (
+	"sync"
+	"time"
+)
+
+// Meters count events to produce exponentially-weighted moving average rates
+// at one-, five-, and fifteen-minutes and a mean rate.
+type Meter interface {
+	Count() int64
+	Mark(int64)
+	Rate1() float64
+	Rate5() float64
+	Rate15() float64
+	RateMean() float64
+	Snapshot() Meter
+}
+
+// GetOrRegisterMeter returns an existing Meter or constructs and registers a
+// new StandardMeter.
+func GetOrRegisterMeter(name string, r Registry) Meter {
+	if nil == r {
+		r = DefaultRegistry
+	}
+	return r.GetOrRegister(name, NewMeter).(Meter)
+}
+
+// NewMeter constructs a new StandardMeter and launches a goroutine.
+func NewMeter() Meter {
+	if UseNilMetrics {
+		return NilMeter{}
+	}
+	m := newStandardMeter()
+	arbiter.Lock()
+	defer arbiter.Unlock()
+	arbiter.meters = append(arbiter.meters, m)
+	if !arbiter.started {
+		arbiter.started = true
+		go arbiter.tick()
+	}
+	return m
+}
+
+// NewMeter constructs and registers a new StandardMeter and launches a
+// goroutine.
+func NewRegisteredMeter(name string, r Registry) Meter {
+	c := NewMeter()
+	if nil == r {
+		r = DefaultRegistry
+	}
+	r.Register(name, c)
+	return c
+}
+
+// MeterSnapshot is a read-only copy of another Meter.
+type MeterSnapshot struct {
+	count                          int64
+	rate1, rate5, rate15, rateMean float64
+}
+
+// Count returns the count of events at the time the snapshot was taken.
+func (m *MeterSnapshot) Count() int64 { return m.count }
+
+// Mark panics.
+func (*MeterSnapshot) Mark(n int64) {
+	panic("Mark called on a MeterSnapshot")
+}
+
+// Rate1 returns the one-minute moving average rate of events per second at the
+// time the snapshot was taken.
+func (m *MeterSnapshot) Rate1() float64 { return m.rate1 }
+
+// Rate5 returns the five-minute moving average rate of events per second at
+// the time the snapshot was taken.
+func (m *MeterSnapshot) Rate5() float64 { return m.rate5 }
+
+// Rate15 returns the fifteen-minute moving average rate of events per second
+// at the time the snapshot was taken.
+func (m *MeterSnapshot) Rate15() float64 { return m.rate15 }
+
+// RateMean returns the meter's mean rate of events per second at the time the
+// snapshot was taken.
+func (m *MeterSnapshot) RateMean() float64 { return m.rateMean }
+
+// Snapshot returns the snapshot.
+func (m *MeterSnapshot) Snapshot() Meter { return m }
+
+// NilMeter is a no-op Meter.
+type NilMeter struct{}
+
+// Count is a no-op.
+func (NilMeter) Count() int64 { return 0 }
+
+// Mark is a no-op.
+func (NilMeter) Mark(n int64) {}
+
+// Rate1 is a no-op.
+func (NilMeter) Rate1() float64 { return 0.0 }
+
+// Rate5 is a no-op.
+func (NilMeter) Rate5() float64 { return 0.0 }
+
+// Rate15is a no-op.
+func (NilMeter) Rate15() float64 { return 0.0 }
+
+// RateMean is a no-op.
+func (NilMeter) RateMean() float64 { return 0.0 }
+
+// Snapshot is a no-op.
+func (NilMeter) Snapshot() Meter { return NilMeter{} }
+
+// StandardMeter is the standard implementation of a Meter.
+type StandardMeter struct {
+	lock        sync.RWMutex
+	snapshot    *MeterSnapshot
+	a1, a5, a15 EWMA
+	startTime   time.Time
+}
+
+func newStandardMeter() *StandardMeter {
+	return &StandardMeter{
+		snapshot:  &MeterSnapshot{},
+		a1:        NewEWMA1(),
+		a5:        NewEWMA5(),
+		a15:       NewEWMA15(),
+		startTime: time.Now(),
+	}
+}
+
+// Count returns the number of events recorded.
+func (m *StandardMeter) Count() int64 {
+	m.lock.RLock()
+	count := m.snapshot.count
+	m.lock.RUnlock()
+	return count
+}
+
+// Mark records the occurance of n events.
+func (m *StandardMeter) Mark(n int64) {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.snapshot.count += n
+	m.a1.Update(n)
+	m.a5.Update(n)
+	m.a15.Update(n)
+	m.updateSnapshot()
+}
+
+// Rate1 returns the one-minute moving average rate of events per second.
+func (m *StandardMeter) Rate1() float64 {
+	m.lock.RLock()
+	rate1 := m.snapshot.rate1
+	m.lock.RUnlock()
+	return rate1
+}
+
+// Rate5 returns the five-minute moving average rate of events per second.
+func (m *StandardMeter) Rate5() float64 {
+	m.lock.RLock()
+	rate5 := m.snapshot.rate5
+	m.lock.RUnlock()
+	return rate5
+}
+
+// Rate15 returns the fifteen-minute moving average rate of events per second.
+func (m *StandardMeter) Rate15() float64 {
+	m.lock.RLock()
+	rate15 := m.snapshot.rate15
+	m.lock.RUnlock()
+	return rate15
+}
+
+// RateMean returns the meter's mean rate of events per second.
+func (m *StandardMeter) RateMean() float64 {
+	m.lock.RLock()
+	rateMean := m.snapshot.rateMean
+	m.lock.RUnlock()
+	return rateMean
+}
+
+// Snapshot returns a read-only copy of the meter.
+func (m *StandardMeter) Snapshot() Meter {
+	m.lock.RLock()
+	snapshot := *m.snapshot
+	m.lock.RUnlock()
+	return &snapshot
+}
+
+func (m *StandardMeter) updateSnapshot() {
+	// should run with write lock held on m.lock
+	snapshot := m.snapshot
+	snapshot.rate1 = m.a1.Rate()
+	snapshot.rate5 = m.a5.Rate()
+	snapshot.rate15 = m.a15.Rate()
+	snapshot.rateMean = float64(snapshot.count) / time.Since(m.startTime).Seconds()
+}
+
+func (m *StandardMeter) tick() {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	m.a1.Tick()
+	m.a5.Tick()
+	m.a15.Tick()
+	m.updateSnapshot()
+}
+
+type meterArbiter struct {
+	sync.RWMutex
+	started bool
+	meters  []*StandardMeter
+	ticker  *time.Ticker
+}
+
+var arbiter = meterArbiter{ticker: time.NewTicker(5e9)}
+
+// Ticks meters on the scheduled interval
+func (ma *meterArbiter) tick() {
+	for {
+		select {
+		case <-ma.ticker.C:
+			ma.tickMeters()
+		}
+	}
+}
+
+func (ma *meterArbiter) tickMeters() {
+	ma.RLock()
+	defer ma.RUnlock()
+	for _, meter := range ma.meters {
+		meter.tick()
+	}
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/metrics.go

@@ -0,0 +1,13 @@
+// Go port of Coda Hale's Metrics library
+//
+// <https://github.com/rcrowley/go-metrics>
+//
+// Coda Hale's original work: <https://github.com/codahale/metrics>
+package metrics
+
+// UseNilMetrics is checked by the constructor functions for all of the
+// standard metrics.  If it is true, the metric returned is a stub.
+//
+// This global kill-switch helps quantify the observer effect and makes
+// for less cluttered pprof profiles.
+var UseNilMetrics bool = false

Godeps/_workspace/src/github.com/rcrowley/go-metrics/opentsdb.go

@@ -0,0 +1,119 @@
+package metrics
+
+import (
+	"bufio"
+	"fmt"
+	"log"
+	"net"
+	"os"
+	"strings"
+	"time"
+)
+
+var shortHostName string = ""
+
+// OpenTSDBConfig provides a container with configuration parameters for
+// the OpenTSDB exporter
+type OpenTSDBConfig struct {
+	Addr          *net.TCPAddr  // Network address to connect to
+	Registry      Registry      // Registry to be exported
+	FlushInterval time.Duration // Flush interval
+	DurationUnit  time.Duration // Time conversion unit for durations
+	Prefix        string        // Prefix to be prepended to metric names
+}
+
+// OpenTSDB is a blocking exporter function which reports metrics in r
+// to a TSDB server located at addr, flushing them every d duration
+// and prepending metric names with prefix.
+func OpenTSDB(r Registry, d time.Duration, prefix string, addr *net.TCPAddr) {
+	OpenTSDBWithConfig(OpenTSDBConfig{
+		Addr:          addr,
+		Registry:      r,
+		FlushInterval: d,
+		DurationUnit:  time.Nanosecond,
+		Prefix:        prefix,
+	})
+}
+
+// OpenTSDBWithConfig is a blocking exporter function just like OpenTSDB,
+// but it takes a OpenTSDBConfig instead.
+func OpenTSDBWithConfig(c OpenTSDBConfig) {
+	for _ = range time.Tick(c.FlushInterval) {
+		if err := openTSDB(&c); nil != err {
+			log.Println(err)
+		}
+	}
+}
+
+func getShortHostname() string {
+	if shortHostName == "" {
+		host, _ := os.Hostname()
+		if index := strings.Index(host, "."); index > 0 {
+			shortHostName = host[:index]
+		} else {
+			shortHostName = host
+		}
+	}
+	return shortHostName
+}
+
+func openTSDB(c *OpenTSDBConfig) error {
+	shortHostname := getShortHostname()
+	now := time.Now().Unix()
+	du := float64(c.DurationUnit)
+	conn, err := net.DialTCP("tcp", nil, c.Addr)
+	if nil != err {
+		return err
+	}
+	defer conn.Close()
+	w := bufio.NewWriter(conn)
+	c.Registry.Each(func(name string, i interface{}) {
+		switch metric := i.(type) {
+		case Counter:
+			fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, metric.Count(), shortHostname)
+		case Gauge:
+			fmt.Fprintf(w, "put %s.%s.value %d %d host=%s\n", c.Prefix, name, now, metric.Value(), shortHostname)
+		case GaugeFloat64:
+			fmt.Fprintf(w, "put %s.%s.value %d %f host=%s\n", c.Prefix, name, now, metric.Value(), shortHostname)
+		case Histogram:
+			h := metric.Snapshot()
+			ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
+			fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, h.Count(), shortHostname)
+			fmt.Fprintf(w, "put %s.%s.min %d %d host=%s\n", c.Prefix, name, now, h.Min(), shortHostname)
+			fmt.Fprintf(w, "put %s.%s.max %d %d host=%s\n", c.Prefix, name, now, h.Max(), shortHostname)
+			fmt.Fprintf(w, "put %s.%s.mean %d %.2f host=%s\n", c.Prefix, name, now, h.Mean(), shortHostname)
+			fmt.Fprintf(w, "put %s.%s.std-dev %d %.2f host=%s\n", c.Prefix, name, now, h.StdDev(), shortHostname)
+			fmt.Fprintf(w, "put %s.%s.50-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[0], shortHostname)
+			fmt.Fprintf(w, "put %s.%s.75-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[1], shortHostname)
+			fmt.Fprintf(w, "put %s.%s.95-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[2], shortHostname)
+			fmt.Fprintf(w, "put %s.%s.99-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[3], shortHostname)
+			fmt.Fprintf(w, "put %s.%s.999-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[4], shortHostname)
+		case Meter:
+			m := metric.Snapshot()
+			fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, m.Count(), shortHostname)
+			fmt.Fprintf(w, "put %s.%s.one-minute %d %.2f host=%s\n", c.Prefix, name, now, m.Rate1(), shortHostname)
+			fmt.Fprintf(w, "put %s.%s.five-minute %d %.2f host=%s\n", c.Prefix, name, now, m.Rate5(), shortHostname)
+			fmt.Fprintf(w, "put %s.%s.fifteen-minute %d %.2f host=%s\n", c.Prefix, name, now, m.Rate15(), shortHostname)
+			fmt.Fprintf(w, "put %s.%s.mean %d %.2f host=%s\n", c.Prefix, name, now, m.RateMean(), shortHostname)
+		case Timer:
+			t := metric.Snapshot()
+			ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
+			fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, t.Count(), shortHostname)
+			fmt.Fprintf(w, "put %s.%s.min %d %d host=%s\n", c.Prefix, name, now, t.Min()/int64(du), shortHostname)
+			fmt.Fprintf(w, "put %s.%s.max %d %d host=%s\n", c.Prefix, name, now, t.Max()/int64(du), shortHostname)
+			fmt.Fprintf(w, "put %s.%s.mean %d %.2f host=%s\n", c.Prefix, name, now, t.Mean()/du, shortHostname)
+			fmt.Fprintf(w, "put %s.%s.std-dev %d %.2f host=%s\n", c.Prefix, name, now, t.StdDev()/du, shortHostname)
+			fmt.Fprintf(w, "put %s.%s.50-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[0]/du, shortHostname)
+			fmt.Fprintf(w, "put %s.%s.75-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[1]/du, shortHostname)
+			fmt.Fprintf(w, "put %s.%s.95-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[2]/du, shortHostname)
+			fmt.Fprintf(w, "put %s.%s.99-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[3]/du, shortHostname)
+			fmt.Fprintf(w, "put %s.%s.999-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[4]/du, shortHostname)
+			fmt.Fprintf(w, "put %s.%s.one-minute %d %.2f host=%s\n", c.Prefix, name, now, t.Rate1(), shortHostname)
+			fmt.Fprintf(w, "put %s.%s.five-minute %d %.2f host=%s\n", c.Prefix, name, now, t.Rate5(), shortHostname)
+			fmt.Fprintf(w, "put %s.%s.fifteen-minute %d %.2f host=%s\n", c.Prefix, name, now, t.Rate15(), shortHostname)
+			fmt.Fprintf(w, "put %s.%s.mean-rate %d %.2f host=%s\n", c.Prefix, name, now, t.RateMean(), shortHostname)
+		}
+		w.Flush()
+	})
+	return nil
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/registry.go

@@ -0,0 +1,240 @@
+package metrics
+
+import (
+	"fmt"
+	"reflect"
+	"sync"
+)
+
+// DuplicateMetric is the error returned by Registry.Register when a metric
+// already exists.  If you mean to Register that metric you must first
+// Unregister the existing metric.
+type DuplicateMetric string
+
+func (err DuplicateMetric) Error() string {
+	return fmt.Sprintf("duplicate metric: %s", string(err))
+}
+
+// A Registry holds references to a set of metrics by name and can iterate
+// over them, calling callback functions provided by the user.
+//
+// This is an interface so as to encourage other structs to implement
+// the Registry API as appropriate.
+type Registry interface {
+
+	// Call the given function for each registered metric.
+	Each(func(string, interface{}))
+
+	// Get the metric by the given name or nil if none is registered.
+	Get(string) interface{}
+
+	// Gets an existing metric or registers the given one.
+	// The interface can be the metric to register if not found in registry,
+	// or a function returning the metric for lazy instantiation.
+	GetOrRegister(string, interface{}) interface{}
+
+	// Register the given metric under the given name.
+	Register(string, interface{}) error
+
+	// Run all registered healthchecks.
+	RunHealthchecks()
+
+	// Unregister the metric with the given name.
+	Unregister(string)
+
+	// Unregister all metrics.  (Mostly for testing.)
+	UnregisterAll()
+}
+
+// The standard implementation of a Registry is a mutex-protected map
+// of names to metrics.
+type StandardRegistry struct {
+	metrics map[string]interface{}
+	mutex   sync.Mutex
+}
+
+// Create a new registry.
+func NewRegistry() Registry {
+	return &StandardRegistry{metrics: make(map[string]interface{})}
+}
+
+// Call the given function for each registered metric.
+func (r *StandardRegistry) Each(f func(string, interface{})) {
+	for name, i := range r.registered() {
+		f(name, i)
+	}
+}
+
+// Get the metric by the given name or nil if none is registered.
+func (r *StandardRegistry) Get(name string) interface{} {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+	return r.metrics[name]
+}
+
+// Gets an existing metric or creates and registers a new one. Threadsafe
+// alternative to calling Get and Register on failure.
+// The interface can be the metric to register if not found in registry,
+// or a function returning the metric for lazy instantiation.
+func (r *StandardRegistry) GetOrRegister(name string, i interface{}) interface{} {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+	if metric, ok := r.metrics[name]; ok {
+		return metric
+	}
+	if v := reflect.ValueOf(i); v.Kind() == reflect.Func {
+		i = v.Call(nil)[0].Interface()
+	}
+	r.register(name, i)
+	return i
+}
+
+// Register the given metric under the given name.  Returns a DuplicateMetric
+// if a metric by the given name is already registered.
+func (r *StandardRegistry) Register(name string, i interface{}) error {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+	return r.register(name, i)
+}
+
+// Run all registered healthchecks.
+func (r *StandardRegistry) RunHealthchecks() {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+	for _, i := range r.metrics {
+		if h, ok := i.(Healthcheck); ok {
+			h.Check()
+		}
+	}
+}
+
+// Unregister the metric with the given name.
+func (r *StandardRegistry) Unregister(name string) {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+	delete(r.metrics, name)
+}
+
+// Unregister all metrics.  (Mostly for testing.)
+func (r *StandardRegistry) UnregisterAll() {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+	for name, _ := range r.metrics {
+		delete(r.metrics, name)
+	}
+}
+
+func (r *StandardRegistry) register(name string, i interface{}) error {
+	if _, ok := r.metrics[name]; ok {
+		return DuplicateMetric(name)
+	}
+	switch i.(type) {
+	case Counter, Gauge, GaugeFloat64, Healthcheck, Histogram, Meter, Timer:
+		r.metrics[name] = i
+	}
+	return nil
+}
+
+func (r *StandardRegistry) registered() map[string]interface{} {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+	metrics := make(map[string]interface{}, len(r.metrics))
+	for name, i := range r.metrics {
+		metrics[name] = i
+	}
+	return metrics
+}
+
+type PrefixedRegistry struct {
+	underlying Registry
+	prefix     string
+}
+
+func NewPrefixedRegistry(prefix string) Registry {
+	return &PrefixedRegistry{
+		underlying: NewRegistry(),
+		prefix:     prefix,
+	}
+}
+
+// Call the given function for each registered metric.
+func (r *PrefixedRegistry) Each(fn func(string, interface{})) {
+	r.underlying.Each(fn)
+}
+
+// Get the metric by the given name or nil if none is registered.
+func (r *PrefixedRegistry) Get(name string) interface{} {
+	return r.underlying.Get(name)
+}
+
+// Gets an existing metric or registers the given one.
+// The interface can be the metric to register if not found in registry,
+// or a function returning the metric for lazy instantiation.
+func (r *PrefixedRegistry) GetOrRegister(name string, metric interface{}) interface{} {
+	realName := r.prefix + name
+	return r.underlying.GetOrRegister(realName, metric)
+}
+
+// Register the given metric under the given name. The name will be prefixed.
+func (r *PrefixedRegistry) Register(name string, metric interface{}) error {
+	realName := r.prefix + name
+	return r.underlying.Register(realName, metric)
+}
+
+// Run all registered healthchecks.
+func (r *PrefixedRegistry) RunHealthchecks() {
+	r.underlying.RunHealthchecks()
+}
+
+// Unregister the metric with the given name. The name will be prefixed.
+func (r *PrefixedRegistry) Unregister(name string) {
+	realName := r.prefix + name
+	r.underlying.Unregister(realName)
+}
+
+// Unregister all metrics.  (Mostly for testing.)
+func (r *PrefixedRegistry) UnregisterAll() {
+	r.underlying.UnregisterAll()
+}
+
+var DefaultRegistry Registry = NewRegistry()
+
+// Call the given function for each registered metric.
+func Each(f func(string, interface{})) {
+	DefaultRegistry.Each(f)
+}
+
+// Get the metric by the given name or nil if none is registered.
+func Get(name string) interface{} {
+	return DefaultRegistry.Get(name)
+}
+
+// Gets an existing metric or creates and registers a new one. Threadsafe
+// alternative to calling Get and Register on failure.
+func GetOrRegister(name string, i interface{}) interface{} {
+	return DefaultRegistry.GetOrRegister(name, i)
+}
+
+// Register the given metric under the given name.  Returns a DuplicateMetric
+// if a metric by the given name is already registered.
+func Register(name string, i interface{}) error {
+	return DefaultRegistry.Register(name, i)
+}
+
+// Register the given metric under the given name.  Panics if a metric by the
+// given name is already registered.
+func MustRegister(name string, i interface{}) {
+	if err := Register(name, i); err != nil {
+		panic(err)
+	}
+}
+
+// Run all registered healthchecks.
+func RunHealthchecks() {
+	DefaultRegistry.RunHealthchecks()
+}
+
+// Unregister the metric with the given name.
+func Unregister(name string) {
+	DefaultRegistry.Unregister(name)
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/runtime.go

@@ -0,0 +1,200 @@
+package metrics
+
+import (
+	"runtime"
+	"time"
+)
+
+var (
+	memStats       runtime.MemStats
+	runtimeMetrics struct {
+		MemStats struct {
+			Alloc        Gauge
+			BuckHashSys  Gauge
+			DebugGC      Gauge
+			EnableGC     Gauge
+			Frees        Gauge
+			HeapAlloc    Gauge
+			HeapIdle     Gauge
+			HeapInuse    Gauge
+			HeapObjects  Gauge
+			HeapReleased Gauge
+			HeapSys      Gauge
+			LastGC       Gauge
+			Lookups      Gauge
+			Mallocs      Gauge
+			MCacheInuse  Gauge
+			MCacheSys    Gauge
+			MSpanInuse   Gauge
+			MSpanSys     Gauge
+			NextGC       Gauge
+			NumGC        Gauge
+			PauseNs      Histogram
+			PauseTotalNs Gauge
+			StackInuse   Gauge
+			StackSys     Gauge
+			Sys          Gauge
+			TotalAlloc   Gauge
+		}
+		NumCgoCall   Gauge
+		NumGoroutine Gauge
+		ReadMemStats Timer
+	}
+	frees       uint64
+	lookups     uint64
+	mallocs     uint64
+	numGC       uint32
+	numCgoCalls int64
+)
+
+// Capture new values for the Go runtime statistics exported in
+// runtime.MemStats.  This is designed to be called as a goroutine.
+func CaptureRuntimeMemStats(r Registry, d time.Duration) {
+	for _ = range time.Tick(d) {
+		CaptureRuntimeMemStatsOnce(r)
+	}
+}
+
+// Capture new values for the Go runtime statistics exported in
+// runtime.MemStats.  This is designed to be called in a background
+// goroutine.  Giving a registry which has not been given to
+// RegisterRuntimeMemStats will panic.
+//
+// Be very careful with this because runtime.ReadMemStats calls the C
+// functions runtime·semacquire(&runtime·worldsema) and runtime·stoptheworld()
+// and that last one does what it says on the tin.
+func CaptureRuntimeMemStatsOnce(r Registry) {
+	t := time.Now()
+	runtime.ReadMemStats(&memStats) // This takes 50-200us.
+	runtimeMetrics.ReadMemStats.UpdateSince(t)
+
+	runtimeMetrics.MemStats.Alloc.Update(int64(memStats.Alloc))
+	runtimeMetrics.MemStats.BuckHashSys.Update(int64(memStats.BuckHashSys))
+	if memStats.DebugGC {
+		runtimeMetrics.MemStats.DebugGC.Update(1)
+	} else {
+		runtimeMetrics.MemStats.DebugGC.Update(0)
+	}
+	if memStats.EnableGC {
+		runtimeMetrics.MemStats.EnableGC.Update(1)
+	} else {
+		runtimeMetrics.MemStats.EnableGC.Update(0)
+	}
+
+	runtimeMetrics.MemStats.Frees.Update(int64(memStats.Frees - frees))
+	runtimeMetrics.MemStats.HeapAlloc.Update(int64(memStats.HeapAlloc))
+	runtimeMetrics.MemStats.HeapIdle.Update(int64(memStats.HeapIdle))
+	runtimeMetrics.MemStats.HeapInuse.Update(int64(memStats.HeapInuse))
+	runtimeMetrics.MemStats.HeapObjects.Update(int64(memStats.HeapObjects))
+	runtimeMetrics.MemStats.HeapReleased.Update(int64(memStats.HeapReleased))
+	runtimeMetrics.MemStats.HeapSys.Update(int64(memStats.HeapSys))
+	runtimeMetrics.MemStats.LastGC.Update(int64(memStats.LastGC))
+	runtimeMetrics.MemStats.Lookups.Update(int64(memStats.Lookups - lookups))
+	runtimeMetrics.MemStats.Mallocs.Update(int64(memStats.Mallocs - mallocs))
+	runtimeMetrics.MemStats.MCacheInuse.Update(int64(memStats.MCacheInuse))
+	runtimeMetrics.MemStats.MCacheSys.Update(int64(memStats.MCacheSys))
+	runtimeMetrics.MemStats.MSpanInuse.Update(int64(memStats.MSpanInuse))
+	runtimeMetrics.MemStats.MSpanSys.Update(int64(memStats.MSpanSys))
+	runtimeMetrics.MemStats.NextGC.Update(int64(memStats.NextGC))
+	runtimeMetrics.MemStats.NumGC.Update(int64(memStats.NumGC - numGC))
+
+	// <https://code.google.com/p/go/source/browse/src/pkg/runtime/mgc0.c>
+	i := numGC % uint32(len(memStats.PauseNs))
+	ii := memStats.NumGC % uint32(len(memStats.PauseNs))
+	if memStats.NumGC-numGC >= uint32(len(memStats.PauseNs)) {
+		for i = 0; i < uint32(len(memStats.PauseNs)); i++ {
+			runtimeMetrics.MemStats.PauseNs.Update(int64(memStats.PauseNs[i]))
+		}
+	} else {
+		if i > ii {
+			for ; i < uint32(len(memStats.PauseNs)); i++ {
+				runtimeMetrics.MemStats.PauseNs.Update(int64(memStats.PauseNs[i]))
+			}
+			i = 0
+		}
+		for ; i < ii; i++ {
+			runtimeMetrics.MemStats.PauseNs.Update(int64(memStats.PauseNs[i]))
+		}
+	}
+	frees = memStats.Frees
+	lookups = memStats.Lookups
+	mallocs = memStats.Mallocs
+	numGC = memStats.NumGC
+
+	runtimeMetrics.MemStats.PauseTotalNs.Update(int64(memStats.PauseTotalNs))
+	runtimeMetrics.MemStats.StackInuse.Update(int64(memStats.StackInuse))
+	runtimeMetrics.MemStats.StackSys.Update(int64(memStats.StackSys))
+	runtimeMetrics.MemStats.Sys.Update(int64(memStats.Sys))
+	runtimeMetrics.MemStats.TotalAlloc.Update(int64(memStats.TotalAlloc))
+
+	currentNumCgoCalls := numCgoCall()
+	runtimeMetrics.NumCgoCall.Update(currentNumCgoCalls - numCgoCalls)
+	numCgoCalls = currentNumCgoCalls
+
+	runtimeMetrics.NumGoroutine.Update(int64(runtime.NumGoroutine()))
+}
+
+// Register runtimeMetrics for the Go runtime statistics exported in runtime and
+// specifically runtime.MemStats.  The runtimeMetrics are named by their
+// fully-qualified Go symbols, i.e. runtime.MemStats.Alloc.
+func RegisterRuntimeMemStats(r Registry) {
+	runtimeMetrics.MemStats.Alloc = NewGauge()
+	runtimeMetrics.MemStats.BuckHashSys = NewGauge()
+	runtimeMetrics.MemStats.DebugGC = NewGauge()
+	runtimeMetrics.MemStats.EnableGC = NewGauge()
+	runtimeMetrics.MemStats.Frees = NewGauge()
+	runtimeMetrics.MemStats.HeapAlloc = NewGauge()
+	runtimeMetrics.MemStats.HeapIdle = NewGauge()
+	runtimeMetrics.MemStats.HeapInuse = NewGauge()
+	runtimeMetrics.MemStats.HeapObjects = NewGauge()
+	runtimeMetrics.MemStats.HeapReleased = NewGauge()
+	runtimeMetrics.MemStats.HeapSys = NewGauge()
+	runtimeMetrics.MemStats.LastGC = NewGauge()
+	runtimeMetrics.MemStats.Lookups = NewGauge()
+	runtimeMetrics.MemStats.Mallocs = NewGauge()
+	runtimeMetrics.MemStats.MCacheInuse = NewGauge()
+	runtimeMetrics.MemStats.MCacheSys = NewGauge()
+	runtimeMetrics.MemStats.MSpanInuse = NewGauge()
+	runtimeMetrics.MemStats.MSpanSys = NewGauge()
+	runtimeMetrics.MemStats.NextGC = NewGauge()
+	runtimeMetrics.MemStats.NumGC = NewGauge()
+	runtimeMetrics.MemStats.PauseNs = NewHistogram(NewExpDecaySample(1028, 0.015))
+	runtimeMetrics.MemStats.PauseTotalNs = NewGauge()
+	runtimeMetrics.MemStats.StackInuse = NewGauge()
+	runtimeMetrics.MemStats.StackSys = NewGauge()
+	runtimeMetrics.MemStats.Sys = NewGauge()
+	runtimeMetrics.MemStats.TotalAlloc = NewGauge()
+	runtimeMetrics.NumCgoCall = NewGauge()
+	runtimeMetrics.NumGoroutine = NewGauge()
+	runtimeMetrics.ReadMemStats = NewTimer()
+
+	r.Register("runtime.MemStats.Alloc", runtimeMetrics.MemStats.Alloc)
+	r.Register("runtime.MemStats.BuckHashSys", runtimeMetrics.MemStats.BuckHashSys)
+	r.Register("runtime.MemStats.DebugGC", runtimeMetrics.MemStats.DebugGC)
+	r.Register("runtime.MemStats.EnableGC", runtimeMetrics.MemStats.EnableGC)
+	r.Register("runtime.MemStats.Frees", runtimeMetrics.MemStats.Frees)
+	r.Register("runtime.MemStats.HeapAlloc", runtimeMetrics.MemStats.HeapAlloc)
+	r.Register("runtime.MemStats.HeapIdle", runtimeMetrics.MemStats.HeapIdle)
+	r.Register("runtime.MemStats.HeapInuse", runtimeMetrics.MemStats.HeapInuse)
+	r.Register("runtime.MemStats.HeapObjects", runtimeMetrics.MemStats.HeapObjects)
+	r.Register("runtime.MemStats.HeapReleased", runtimeMetrics.MemStats.HeapReleased)
+	r.Register("runtime.MemStats.HeapSys", runtimeMetrics.MemStats.HeapSys)
+	r.Register("runtime.MemStats.LastGC", runtimeMetrics.MemStats.LastGC)
+	r.Register("runtime.MemStats.Lookups", runtimeMetrics.MemStats.Lookups)
+	r.Register("runtime.MemStats.Mallocs", runtimeMetrics.MemStats.Mallocs)
+	r.Register("runtime.MemStats.MCacheInuse", runtimeMetrics.MemStats.MCacheInuse)
+	r.Register("runtime.MemStats.MCacheSys", runtimeMetrics.MemStats.MCacheSys)
+	r.Register("runtime.MemStats.MSpanInuse", runtimeMetrics.MemStats.MSpanInuse)
+	r.Register("runtime.MemStats.MSpanSys", runtimeMetrics.MemStats.MSpanSys)
+	r.Register("runtime.MemStats.NextGC", runtimeMetrics.MemStats.NextGC)
+	r.Register("runtime.MemStats.NumGC", runtimeMetrics.MemStats.NumGC)
+	r.Register("runtime.MemStats.PauseNs", runtimeMetrics.MemStats.PauseNs)
+	r.Register("runtime.MemStats.PauseTotalNs", runtimeMetrics.MemStats.PauseTotalNs)
+	r.Register("runtime.MemStats.StackInuse", runtimeMetrics.MemStats.StackInuse)
+	r.Register("runtime.MemStats.StackSys", runtimeMetrics.MemStats.StackSys)
+	r.Register("runtime.MemStats.Sys", runtimeMetrics.MemStats.Sys)
+	r.Register("runtime.MemStats.TotalAlloc", runtimeMetrics.MemStats.TotalAlloc)
+	r.Register("runtime.NumCgoCall", runtimeMetrics.NumCgoCall)
+	r.Register("runtime.NumGoroutine", runtimeMetrics.NumGoroutine)
+	r.Register("runtime.ReadMemStats", runtimeMetrics.ReadMemStats)
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/runtime_cgo.go

@@ -0,0 +1,10 @@
+// +build cgo
+// +build !appengine
+
+package metrics
+
+import "runtime"
+
+func numCgoCall() int64 {
+	return runtime.NumCgoCall()
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/runtime_no_cgo.go

@@ -0,0 +1,7 @@
+// +build !cgo appengine
+
+package metrics
+
+func numCgoCall() int64 {
+	return 0
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/sample.go

@@ -0,0 +1,609 @@
+package metrics
+
+import (
+	"math"
+	"math/rand"
+	"sort"
+	"sync"
+	"time"
+)
+
+const rescaleThreshold = time.Hour
+
+// Samples maintain a statistically-significant selection of values from
+// a stream.
+type Sample interface {
+	Clear()
+	Count() int64
+	Max() int64
+	Mean() float64
+	Min() int64
+	Percentile(float64) float64
+	Percentiles([]float64) []float64
+	Size() int
+	Snapshot() Sample
+	StdDev() float64
+	Sum() int64
+	Update(int64)
+	Values() []int64
+	Variance() float64
+}
+
+// ExpDecaySample is an exponentially-decaying sample using a forward-decaying
+// priority reservoir.  See Cormode et al's "Forward Decay: A Practical Time
+// Decay Model for Streaming Systems".
+//
+// <http://www.research.att.com/people/Cormode_Graham/library/publications/CormodeShkapenyukSrivastavaXu09.pdf>
+type ExpDecaySample struct {
+	alpha         float64
+	count         int64
+	mutex         sync.Mutex
+	reservoirSize int
+	t0, t1        time.Time
+	values        *expDecaySampleHeap
+}
+
+// NewExpDecaySample constructs a new exponentially-decaying sample with the
+// given reservoir size and alpha.
+func NewExpDecaySample(reservoirSize int, alpha float64) Sample {
+	if UseNilMetrics {
+		return NilSample{}
+	}
+	s := &ExpDecaySample{
+		alpha:         alpha,
+		reservoirSize: reservoirSize,
+		t0:            time.Now(),
+		values:        newExpDecaySampleHeap(reservoirSize),
+	}
+	s.t1 = s.t0.Add(rescaleThreshold)
+	return s
+}
+
+// Clear clears all samples.
+func (s *ExpDecaySample) Clear() {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	s.count = 0
+	s.t0 = time.Now()
+	s.t1 = s.t0.Add(rescaleThreshold)
+	s.values.Clear()
+}
+
+// Count returns the number of samples recorded, which may exceed the
+// reservoir size.
+func (s *ExpDecaySample) Count() int64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return s.count
+}
+
+// Max returns the maximum value in the sample, which may not be the maximum
+// value ever to be part of the sample.
+func (s *ExpDecaySample) Max() int64 {
+	return SampleMax(s.Values())
+}
+
+// Mean returns the mean of the values in the sample.
+func (s *ExpDecaySample) Mean() float64 {
+	return SampleMean(s.Values())
+}
+
+// Min returns the minimum value in the sample, which may not be the minimum
+// value ever to be part of the sample.
+func (s *ExpDecaySample) Min() int64 {
+	return SampleMin(s.Values())
+}
+
+// Percentile returns an arbitrary percentile of values in the sample.
+func (s *ExpDecaySample) Percentile(p float64) float64 {
+	return SamplePercentile(s.Values(), p)
+}
+
+// Percentiles returns a slice of arbitrary percentiles of values in the
+// sample.
+func (s *ExpDecaySample) Percentiles(ps []float64) []float64 {
+	return SamplePercentiles(s.Values(), ps)
+}
+
+// Size returns the size of the sample, which is at most the reservoir size.
+func (s *ExpDecaySample) Size() int {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return s.values.Size()
+}
+
+// Snapshot returns a read-only copy of the sample.
+func (s *ExpDecaySample) Snapshot() Sample {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	vals := s.values.Values()
+	values := make([]int64, len(vals))
+	for i, v := range vals {
+		values[i] = v.v
+	}
+	return &SampleSnapshot{
+		count:  s.count,
+		values: values,
+	}
+}
+
+// StdDev returns the standard deviation of the values in the sample.
+func (s *ExpDecaySample) StdDev() float64 {
+	return SampleStdDev(s.Values())
+}
+
+// Sum returns the sum of the values in the sample.
+func (s *ExpDecaySample) Sum() int64 {
+	return SampleSum(s.Values())
+}
+
+// Update samples a new value.
+func (s *ExpDecaySample) Update(v int64) {
+	s.update(time.Now(), v)
+}
+
+// Values returns a copy of the values in the sample.
+func (s *ExpDecaySample) Values() []int64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	vals := s.values.Values()
+	values := make([]int64, len(vals))
+	for i, v := range vals {
+		values[i] = v.v
+	}
+	return values
+}
+
+// Variance returns the variance of the values in the sample.
+func (s *ExpDecaySample) Variance() float64 {
+	return SampleVariance(s.Values())
+}
+
+// update samples a new value at a particular timestamp.  This is a method all
+// its own to facilitate testing.
+func (s *ExpDecaySample) update(t time.Time, v int64) {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	s.count++
+	if s.values.Size() == s.reservoirSize {
+		s.values.Pop()
+	}
+	s.values.Push(expDecaySample{
+		k: math.Exp(t.Sub(s.t0).Seconds()*s.alpha) / rand.Float64(),
+		v: v,
+	})
+	if t.After(s.t1) {
+		values := s.values.Values()
+		t0 := s.t0
+		s.values.Clear()
+		s.t0 = t
+		s.t1 = s.t0.Add(rescaleThreshold)
+		for _, v := range values {
+			v.k = v.k * math.Exp(-s.alpha*s.t0.Sub(t0).Seconds())
+			s.values.Push(v)
+		}
+	}
+}
+
+// NilSample is a no-op Sample.
+type NilSample struct{}
+
+// Clear is a no-op.
+func (NilSample) Clear() {}
+
+// Count is a no-op.
+func (NilSample) Count() int64 { return 0 }
+
+// Max is a no-op.
+func (NilSample) Max() int64 { return 0 }
+
+// Mean is a no-op.
+func (NilSample) Mean() float64 { return 0.0 }
+
+// Min is a no-op.
+func (NilSample) Min() int64 { return 0 }
+
+// Percentile is a no-op.
+func (NilSample) Percentile(p float64) float64 { return 0.0 }
+
+// Percentiles is a no-op.
+func (NilSample) Percentiles(ps []float64) []float64 {
+	return make([]float64, len(ps))
+}
+
+// Size is a no-op.
+func (NilSample) Size() int { return 0 }
+
+// Sample is a no-op.
+func (NilSample) Snapshot() Sample { return NilSample{} }
+
+// StdDev is a no-op.
+func (NilSample) StdDev() float64 { return 0.0 }
+
+// Sum is a no-op.
+func (NilSample) Sum() int64 { return 0 }
+
+// Update is a no-op.
+func (NilSample) Update(v int64) {}
+
+// Values is a no-op.
+func (NilSample) Values() []int64 { return []int64{} }
+
+// Variance is a no-op.
+func (NilSample) Variance() float64 { return 0.0 }
+
+// SampleMax returns the maximum value of the slice of int64.
+func SampleMax(values []int64) int64 {
+	if 0 == len(values) {
+		return 0
+	}
+	var max int64 = math.MinInt64
+	for _, v := range values {
+		if max < v {
+			max = v
+		}
+	}
+	return max
+}
+
+// SampleMean returns the mean value of the slice of int64.
+func SampleMean(values []int64) float64 {
+	if 0 == len(values) {
+		return 0.0
+	}
+	return float64(SampleSum(values)) / float64(len(values))
+}
+
+// SampleMin returns the minimum value of the slice of int64.
+func SampleMin(values []int64) int64 {
+	if 0 == len(values) {
+		return 0
+	}
+	var min int64 = math.MaxInt64
+	for _, v := range values {
+		if min > v {
+			min = v
+		}
+	}
+	return min
+}
+
+// SamplePercentiles returns an arbitrary percentile of the slice of int64.
+func SamplePercentile(values int64Slice, p float64) float64 {
+	return SamplePercentiles(values, []float64{p})[0]
+}
+
+// SamplePercentiles returns a slice of arbitrary percentiles of the slice of
+// int64.
+func SamplePercentiles(values int64Slice, ps []float64) []float64 {
+	scores := make([]float64, len(ps))
+	size := len(values)
+	if size > 0 {
+		sort.Sort(values)
+		for i, p := range ps {
+			pos := p * float64(size+1)
+			if pos < 1.0 {
+				scores[i] = float64(values[0])
+			} else if pos >= float64(size) {
+				scores[i] = float64(values[size-1])
+			} else {
+				lower := float64(values[int(pos)-1])
+				upper := float64(values[int(pos)])
+				scores[i] = lower + (pos-math.Floor(pos))*(upper-lower)
+			}
+		}
+	}
+	return scores
+}
+
+// SampleSnapshot is a read-only copy of another Sample.
+type SampleSnapshot struct {
+	count  int64
+	values []int64
+}
+
+// Clear panics.
+func (*SampleSnapshot) Clear() {
+	panic("Clear called on a SampleSnapshot")
+}
+
+// Count returns the count of inputs at the time the snapshot was taken.
+func (s *SampleSnapshot) Count() int64 { return s.count }
+
+// Max returns the maximal value at the time the snapshot was taken.
+func (s *SampleSnapshot) Max() int64 { return SampleMax(s.values) }
+
+// Mean returns the mean value at the time the snapshot was taken.
+func (s *SampleSnapshot) Mean() float64 { return SampleMean(s.values) }
+
+// Min returns the minimal value at the time the snapshot was taken.
+func (s *SampleSnapshot) Min() int64 { return SampleMin(s.values) }
+
+// Percentile returns an arbitrary percentile of values at the time the
+// snapshot was taken.
+func (s *SampleSnapshot) Percentile(p float64) float64 {
+	return SamplePercentile(s.values, p)
+}
+
+// Percentiles returns a slice of arbitrary percentiles of values at the time
+// the snapshot was taken.
+func (s *SampleSnapshot) Percentiles(ps []float64) []float64 {
+	return SamplePercentiles(s.values, ps)
+}
+
+// Size returns the size of the sample at the time the snapshot was taken.
+func (s *SampleSnapshot) Size() int { return len(s.values) }
+
+// Snapshot returns the snapshot.
+func (s *SampleSnapshot) Snapshot() Sample { return s }
+
+// StdDev returns the standard deviation of values at the time the snapshot was
+// taken.
+func (s *SampleSnapshot) StdDev() float64 { return SampleStdDev(s.values) }
+
+// Sum returns the sum of values at the time the snapshot was taken.
+func (s *SampleSnapshot) Sum() int64 { return SampleSum(s.values) }
+
+// Update panics.
+func (*SampleSnapshot) Update(int64) {
+	panic("Update called on a SampleSnapshot")
+}
+
+// Values returns a copy of the values in the sample.
+func (s *SampleSnapshot) Values() []int64 {
+	values := make([]int64, len(s.values))
+	copy(values, s.values)
+	return values
+}
+
+// Variance returns the variance of values at the time the snapshot was taken.
+func (s *SampleSnapshot) Variance() float64 { return SampleVariance(s.values) }
+
+// SampleStdDev returns the standard deviation of the slice of int64.
+func SampleStdDev(values []int64) float64 {
+	return math.Sqrt(SampleVariance(values))
+}
+
+// SampleSum returns the sum of the slice of int64.
+func SampleSum(values []int64) int64 {
+	var sum int64
+	for _, v := range values {
+		sum += v
+	}
+	return sum
+}
+
+// SampleVariance returns the variance of the slice of int64.
+func SampleVariance(values []int64) float64 {
+	if 0 == len(values) {
+		return 0.0
+	}
+	m := SampleMean(values)
+	var sum float64
+	for _, v := range values {
+		d := float64(v) - m
+		sum += d * d
+	}
+	return sum / float64(len(values))
+}
+
+// A uniform sample using Vitter's Algorithm R.
+//
+// <http://www.cs.umd.edu/~samir/498/vitter.pdf>
+type UniformSample struct {
+	count         int64
+	mutex         sync.Mutex
+	reservoirSize int
+	values        []int64
+}
+
+// NewUniformSample constructs a new uniform sample with the given reservoir
+// size.
+func NewUniformSample(reservoirSize int) Sample {
+	if UseNilMetrics {
+		return NilSample{}
+	}
+	return &UniformSample{
+		reservoirSize: reservoirSize,
+		values:        make([]int64, 0, reservoirSize),
+	}
+}
+
+// Clear clears all samples.
+func (s *UniformSample) Clear() {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	s.count = 0
+	s.values = make([]int64, 0, s.reservoirSize)
+}
+
+// Count returns the number of samples recorded, which may exceed the
+// reservoir size.
+func (s *UniformSample) Count() int64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return s.count
+}
+
+// Max returns the maximum value in the sample, which may not be the maximum
+// value ever to be part of the sample.
+func (s *UniformSample) Max() int64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return SampleMax(s.values)
+}
+
+// Mean returns the mean of the values in the sample.
+func (s *UniformSample) Mean() float64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return SampleMean(s.values)
+}
+
+// Min returns the minimum value in the sample, which may not be the minimum
+// value ever to be part of the sample.
+func (s *UniformSample) Min() int64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return SampleMin(s.values)
+}
+
+// Percentile returns an arbitrary percentile of values in the sample.
+func (s *UniformSample) Percentile(p float64) float64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return SamplePercentile(s.values, p)
+}
+
+// Percentiles returns a slice of arbitrary percentiles of values in the
+// sample.
+func (s *UniformSample) Percentiles(ps []float64) []float64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return SamplePercentiles(s.values, ps)
+}
+
+// Size returns the size of the sample, which is at most the reservoir size.
+func (s *UniformSample) Size() int {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return len(s.values)
+}
+
+// Snapshot returns a read-only copy of the sample.
+func (s *UniformSample) Snapshot() Sample {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	values := make([]int64, len(s.values))
+	copy(values, s.values)
+	return &SampleSnapshot{
+		count:  s.count,
+		values: values,
+	}
+}
+
+// StdDev returns the standard deviation of the values in the sample.
+func (s *UniformSample) StdDev() float64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return SampleStdDev(s.values)
+}
+
+// Sum returns the sum of the values in the sample.
+func (s *UniformSample) Sum() int64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return SampleSum(s.values)
+}
+
+// Update samples a new value.
+func (s *UniformSample) Update(v int64) {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	s.count++
+	if len(s.values) < s.reservoirSize {
+		s.values = append(s.values, v)
+	} else {
+		r := rand.Int63n(s.count)
+		if r < int64(len(s.values)) {
+			s.values[int(r)] = v
+		}
+	}
+}
+
+// Values returns a copy of the values in the sample.
+func (s *UniformSample) Values() []int64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	values := make([]int64, len(s.values))
+	copy(values, s.values)
+	return values
+}
+
+// Variance returns the variance of the values in the sample.
+func (s *UniformSample) Variance() float64 {
+	s.mutex.Lock()
+	defer s.mutex.Unlock()
+	return SampleVariance(s.values)
+}
+
+// expDecaySample represents an individual sample in a heap.
+type expDecaySample struct {
+	k float64
+	v int64
+}
+
+func newExpDecaySampleHeap(reservoirSize int) *expDecaySampleHeap {
+	return &expDecaySampleHeap{make([]expDecaySample, 0, reservoirSize)}
+}
+
+// expDecaySampleHeap is a min-heap of expDecaySamples.
+// The internal implementation is copied from the standard library's container/heap
+type expDecaySampleHeap struct {
+	s []expDecaySample
+}
+
+func (h *expDecaySampleHeap) Clear() {
+	h.s = h.s[:0]
+}
+
+func (h *expDecaySampleHeap) Push(s expDecaySample) {
+	n := len(h.s)
+	h.s = h.s[0 : n+1]
+	h.s[n] = s
+	h.up(n)
+}
+
+func (h *expDecaySampleHeap) Pop() expDecaySample {
+	n := len(h.s) - 1
+	h.s[0], h.s[n] = h.s[n], h.s[0]
+	h.down(0, n)
+
+	n = len(h.s)
+	s := h.s[n-1]
+	h.s = h.s[0 : n-1]
+	return s
+}
+
+func (h *expDecaySampleHeap) Size() int {
+	return len(h.s)
+}
+
+func (h *expDecaySampleHeap) Values() []expDecaySample {
+	return h.s
+}
+
+func (h *expDecaySampleHeap) up(j int) {
+	for {
+		i := (j - 1) / 2 // parent
+		if i == j || !(h.s[j].k < h.s[i].k) {
+			break
+		}
+		h.s[i], h.s[j] = h.s[j], h.s[i]
+		j = i
+	}
+}
+
+func (h *expDecaySampleHeap) down(i, n int) {
+	for {
+		j1 := 2*i + 1
+		if j1 >= n || j1 < 0 { // j1 < 0 after int overflow
+			break
+		}
+		j := j1 // left child
+		if j2 := j1 + 1; j2 < n && !(h.s[j1].k < h.s[j2].k) {
+			j = j2 // = 2*i + 2  // right child
+		}
+		if !(h.s[j].k < h.s[i].k) {
+			break
+		}
+		h.s[i], h.s[j] = h.s[j], h.s[i]
+		i = j
+	}
+}
+
+type int64Slice []int64
+
+func (p int64Slice) Len() int           { return len(p) }
+func (p int64Slice) Less(i, j int) bool { return p[i] < p[j] }
+func (p int64Slice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

Godeps/_workspace/src/github.com/rcrowley/go-metrics/stathat/stathat.go

@@ -0,0 +1,69 @@
+// Metrics output to StatHat.
+package stathat
+
+import (
+	"github.com/rcrowley/go-metrics"
+	"github.com/stathat/go"
+	"log"
+	"time"
+)
+
+func Stathat(r metrics.Registry, d time.Duration, userkey string) {
+	for {
+		if err := sh(r, userkey); nil != err {
+			log.Println(err)
+		}
+		time.Sleep(d)
+	}
+}
+
+func sh(r metrics.Registry, userkey string) error {
+	r.Each(func(name string, i interface{}) {
+		switch metric := i.(type) {
+		case metrics.Counter:
+			stathat.PostEZCount(name, userkey, int(metric.Count()))
+		case metrics.Gauge:
+			stathat.PostEZValue(name, userkey, float64(metric.Value()))
+		case metrics.GaugeFloat64:
+			stathat.PostEZValue(name, userkey, float64(metric.Value()))
+		case metrics.Histogram:
+			h := metric.Snapshot()
+			ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
+			stathat.PostEZCount(name+".count", userkey, int(h.Count()))
+			stathat.PostEZValue(name+".min", userkey, float64(h.Min()))
+			stathat.PostEZValue(name+".max", userkey, float64(h.Max()))
+			stathat.PostEZValue(name+".mean", userkey, float64(h.Mean()))
+			stathat.PostEZValue(name+".std-dev", userkey, float64(h.StdDev()))
+			stathat.PostEZValue(name+".50-percentile", userkey, float64(ps[0]))
+			stathat.PostEZValue(name+".75-percentile", userkey, float64(ps[1]))
+			stathat.PostEZValue(name+".95-percentile", userkey, float64(ps[2]))
+			stathat.PostEZValue(name+".99-percentile", userkey, float64(ps[3]))
+			stathat.PostEZValue(name+".999-percentile", userkey, float64(ps[4]))
+		case metrics.Meter:
+			m := metric.Snapshot()
+			stathat.PostEZCount(name+".count", userkey, int(m.Count()))
+			stathat.PostEZValue(name+".one-minute", userkey, float64(m.Rate1()))
+			stathat.PostEZValue(name+".five-minute", userkey, float64(m.Rate5()))
+			stathat.PostEZValue(name+".fifteen-minute", userkey, float64(m.Rate15()))
+			stathat.PostEZValue(name+".mean", userkey, float64(m.RateMean()))
+		case metrics.Timer:
+			t := metric.Snapshot()
+			ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
+			stathat.PostEZCount(name+".count", userkey, int(t.Count()))
+			stathat.PostEZValue(name+".min", userkey, float64(t.Min()))
+			stathat.PostEZValue(name+".max", userkey, float64(t.Max()))
+			stathat.PostEZValue(name+".mean", userkey, float64(t.Mean()))
+			stathat.PostEZValue(name+".std-dev", userkey, float64(t.StdDev()))
+			stathat.PostEZValue(name+".50-percentile", userkey, float64(ps[0]))
+			stathat.PostEZValue(name+".75-percentile", userkey, float64(ps[1]))
+			stathat.PostEZValue(name+".95-percentile", userkey, float64(ps[2]))
+			stathat.PostEZValue(name+".99-percentile", userkey, float64(ps[3]))
+			stathat.PostEZValue(name+".999-percentile", userkey, float64(ps[4]))
+			stathat.PostEZValue(name+".one-minute", userkey, float64(t.Rate1()))
+			stathat.PostEZValue(name+".five-minute", userkey, float64(t.Rate5()))
+			stathat.PostEZValue(name+".fifteen-minute", userkey, float64(t.Rate15()))
+			stathat.PostEZValue(name+".mean-rate", userkey, float64(t.RateMean()))
+		}
+	})
+	return nil
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/syslog.go

@@ -0,0 +1,78 @@
+// +build !windows
+
+package metrics
+
+import (
+	"fmt"
+	"log/syslog"
+	"time"
+)
+
+// Output each metric in the given registry to syslog periodically using
+// the given syslogger.
+func Syslog(r Registry, d time.Duration, w *syslog.Writer) {
+	for _ = range time.Tick(d) {
+		r.Each(func(name string, i interface{}) {
+			switch metric := i.(type) {
+			case Counter:
+				w.Info(fmt.Sprintf("counter %s: count: %d", name, metric.Count()))
+			case Gauge:
+				w.Info(fmt.Sprintf("gauge %s: value: %d", name, metric.Value()))
+			case GaugeFloat64:
+				w.Info(fmt.Sprintf("gauge %s: value: %f", name, metric.Value()))
+			case Healthcheck:
+				metric.Check()
+				w.Info(fmt.Sprintf("healthcheck %s: error: %v", name, metric.Error()))
+			case Histogram:
+				h := metric.Snapshot()
+				ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
+				w.Info(fmt.Sprintf(
+					"histogram %s: count: %d min: %d max: %d mean: %.2f stddev: %.2f median: %.2f 75%%: %.2f 95%%: %.2f 99%%: %.2f 99.9%%: %.2f",
+					name,
+					h.Count(),
+					h.Min(),
+					h.Max(),
+					h.Mean(),
+					h.StdDev(),
+					ps[0],
+					ps[1],
+					ps[2],
+					ps[3],
+					ps[4],
+				))
+			case Meter:
+				m := metric.Snapshot()
+				w.Info(fmt.Sprintf(
+					"meter %s: count: %d 1-min: %.2f 5-min: %.2f 15-min: %.2f mean: %.2f",
+					name,
+					m.Count(),
+					m.Rate1(),
+					m.Rate5(),
+					m.Rate15(),
+					m.RateMean(),
+				))
+			case Timer:
+				t := metric.Snapshot()
+				ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
+				w.Info(fmt.Sprintf(
+					"timer %s: count: %d min: %d max: %d mean: %.2f stddev: %.2f median: %.2f 75%%: %.2f 95%%: %.2f 99%%: %.2f 99.9%%: %.2f 1-min: %.2f 5-min: %.2f 15-min: %.2f mean-rate: %.2f",
+					name,
+					t.Count(),
+					t.Min(),
+					t.Max(),
+					t.Mean(),
+					t.StdDev(),
+					ps[0],
+					ps[1],
+					ps[2],
+					ps[3],
+					ps[4],
+					t.Rate1(),
+					t.Rate5(),
+					t.Rate15(),
+					t.RateMean(),
+				))
+			}
+		})
+	}
+}

Godeps/_workspace/src/github.com/rcrowley/go-metrics/timer.go

@@ -0,0 +1,311 @@
+package metrics
+
+import (
+	"sync"
+	"time"
+)
+
+// Timers capture the duration and rate of events.
+type Timer interface {
+	Count() int64
+	Max() int64
+	Mean() float64
+	Min() int64
+	Percentile(float64) float64
+	Percentiles([]float64) []float64
+	Rate1() float64
+	Rate5() float64
+	Rate15() float64
+	RateMean() float64
+	Snapshot() Timer
+	StdDev() float64
+	Sum() int64
+	Time(func())
+	Update(time.Duration)
+	UpdateSince(time.Time)
+	Variance() float64
+}
+
+// GetOrRegisterTimer returns an existing Timer or constructs and registers a
+// new StandardTimer.
+func GetOrRegisterTimer(name string, r Registry) Timer {
+	if nil == r {
+		r = DefaultRegistry
+	}
+	return r.GetOrRegister(name, NewTimer).(Timer)
+}
+
+// NewCustomTimer constructs a new StandardTimer from a Histogram and a Meter.
+func NewCustomTimer(h Histogram, m Meter) Timer {
+	if UseNilMetrics {
+		return NilTimer{}
+	}
+	return &StandardTimer{
+		histogram: h,
+		meter:     m,
+	}
+}
+
+// NewRegisteredTimer constructs and registers a new StandardTimer.
+func NewRegisteredTimer(name string, r Registry) Timer {
+	c := NewTimer()
+	if nil == r {
+		r = DefaultRegistry
+	}
+	r.Register(name, c)
+	return c
+}
+
+// NewTimer constructs a new StandardTimer using an exponentially-decaying
+// sample with the same reservoir size and alpha as UNIX load averages.
+func NewTimer() Timer {
+	if UseNilMetrics {
+		return NilTimer{}
+	}
+	return &StandardTimer{
+		histogram: NewHistogram(NewExpDecaySample(1028, 0.015)),
+		meter:     NewMeter(),
+	}
+}
+
+// NilTimer is a no-op Timer.
+type NilTimer struct {
+	h Histogram
+	m Meter
+}
+
+// Count is a no-op.
+func (NilTimer) Count() int64 { return 0 }
+
+// Max is a no-op.
+func (NilTimer) Max() int64 { return 0 }
+
+// Mean is a no-op.
+func (NilTimer) Mean() float64 { return 0.0 }
+
+// Min is a no-op.
+func (NilTimer) Min() int64 { return 0 }
+
+// Percentile is a no-op.
+func (NilTimer) Percentile(p float64) float64 { return 0.0 }
+
+// Percentiles is a no-op.
+func (NilTimer) Percentiles(ps []float64) []float64 {
+	return make([]float64, len(ps))
+}
+
+// Rate1 is a no-op.
+func (NilTimer) Rate1() float64 { return 0.0 }
+
+// Rate5 is a no-op.
+func (NilTimer) Rate5() float64 { return 0.0 }
+
+// Rate15 is a no-op.
+func (NilTimer) Rate15() float64 { return 0.0 }
+
+// RateMean is a no-op.
+func (NilTimer) RateMean() float64 { return 0.0 }
+
+// Snapshot is a no-op.
+func (NilTimer) Snapshot() Timer { return NilTimer{} }
+
+// StdDev is a no-op.
+func (NilTimer) StdDev() float64 { return 0.0 }
+
+// Sum is a no-op.
+func (NilTimer) Sum() int64 { return 0 }
+
+// Time is a no-op.
+func (NilTimer) Time(func()) {}
+
+// Update is a no-op.
+func (NilTimer) Update(time.Duration) {}
+
+// UpdateSince is a no-op.
+func (NilTimer) UpdateSince(time.Time) {}
+
+// Variance is a no-op.
+func (NilTimer) Variance() float64 { return 0.0 }
+
+// StandardTimer is the standard implementation of a Timer and uses a Histogram
+// and Meter.
+type StandardTimer struct {
+	histogram Histogram
+	meter     Meter
+	mutex     sync.Mutex
+}
+
+// Count returns the number of events recorded.
+func (t *StandardTimer) Count() int64 {
+	return t.histogram.Count()
+}
+
+// Max returns the maximum value in the sample.
+func (t *StandardTimer) Max() int64 {
+	return t.histogram.Max()
+}
+
+// Mean returns the mean of the values in the sample.
+func (t *StandardTimer) Mean() float64 {
+	return t.histogram.Mean()
+}
+
+// Min returns the minimum value in the sample.
+func (t *StandardTimer) Min() int64 {
+	return t.histogram.Min()
+}
+
+// Percentile returns an arbitrary percentile of the values in the sample.
+func (t *StandardTimer) Percentile(p float64) float64 {
+	return t.histogram.Percentile(p)
+}
+
+// Percentiles returns a slice of arbitrary percentiles of the values in the
+// sample.
+func (t *StandardTimer) Percentiles(ps []float64) []float64 {
+	return t.histogram.Percentiles(ps)
+}
+
+// Rate1 returns the one-minute moving average rate of events per second.
+func (t *StandardTimer) Rate1() float64 {
+	return t.meter.Rate1()
+}
+
+// Rate5 returns the five-minute moving average rate of events per second.
+func (t *StandardTimer) Rate5() float64 {
+	return t.meter.Rate5()
+}
+
+// Rate15 returns the fifteen-minute moving average rate of events per second.
+func (t *StandardTimer) Rate15() float64 {
+	return t.meter.Rate15()
+}
+
+// RateMean returns the meter's mean rate of events per second.
+func (t *StandardTimer) RateMean() float64 {
+	return t.meter.RateMean()
+}
+
+// Snapshot returns a read-only copy of the timer.
+func (t *StandardTimer) Snapshot() Timer {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+	return &TimerSnapshot{
+		histogram: t.histogram.Snapshot().(*HistogramSnapshot),
+		meter:     t.meter.Snapshot().(*MeterSnapshot),
+	}
+}
+
+// StdDev returns the standard deviation of the values in the sample.
+func (t *StandardTimer) StdDev() float64 {
+	return t.histogram.StdDev()
+}
+
+// Sum returns the sum in the sample.
+func (t *StandardTimer) Sum() int64 {
+	return t.histogram.Sum()
+}
+
+// Record the duration of the execution of the given function.
+func (t *StandardTimer) Time(f func()) {
+	ts := time.Now()
+	f()
+	t.Update(time.Since(ts))
+}
+
+// Record the duration of an event.
+func (t *StandardTimer) Update(d time.Duration) {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+	t.histogram.Update(int64(d))
+	t.meter.Mark(1)
+}
+
+// Record the duration of an event that started at a time and ends now.
+func (t *StandardTimer) UpdateSince(ts time.Time) {
+	t.mutex.Lock()
+	defer t.mutex.Unlock()
+	t.histogram.Update(int64(time.Since(ts)))
+	t.meter.Mark(1)
+}
+
+// Variance returns the variance of the values in the sample.
+func (t *StandardTimer) Variance() float64 {
+	return t.histogram.Variance()
+}
+
+// TimerSnapshot is a read-only copy of another Timer.
+type TimerSnapshot struct {
+	histogram *HistogramSnapshot
+	meter     *MeterSnapshot
+}
+
+// Count returns the number of events recorded at the time the snapshot was
+// taken.
+func (t *TimerSnapshot) Count() int64 { return t.histogram.Count() }
+
+// Max returns the maximum value at the time the snapshot was taken.
+func (t *TimerSnapshot) Max() int64 { return t.histogram.Max() }
+
+// Mean returns the mean value at the time the snapshot was taken.
+func (t *TimerSnapshot) Mean() float64 { return t.histogram.Mean() }
+
+// Min returns the minimum value at the time the snapshot was taken.
+func (t *TimerSnapshot) Min() int64 { return t.histogram.Min() }
+
+// Percentile returns an arbitrary percentile of sampled values at the time the
+// snapshot was taken.
+func (t *TimerSnapshot) Percentile(p float64) float64 {
+	return t.histogram.Percentile(p)
+}
+
+// Percentiles returns a slice of arbitrary percentiles of sampled values at
+// the time the snapshot was taken.
+func (t *TimerSnapshot) Percentiles(ps []float64) []float64 {
+	return t.histogram.Percentiles(ps)
+}
+
+// Rate1 returns the one-minute moving average rate of events per second at the
+// time the snapshot was taken.
+func (t *TimerSnapshot) Rate1() float64 { return t.meter.Rate1() }
+
+// Rate5 returns the five-minute moving average rate of events per second at
+// the time the snapshot was taken.
+func (t *TimerSnapshot) Rate5() float64 { return t.meter.Rate5() }
+
+// Rate15 returns the fifteen-minute moving average rate of events per second
+// at the time the snapshot was taken.
+func (t *TimerSnapshot) Rate15() float64 { return t.meter.Rate15() }
+
+// RateMean returns the meter's mean rate of events per second at the time the
+// snapshot was taken.
+func (t *TimerSnapshot) RateMean() float64 { return t.meter.RateMean() }
+
+// Snapshot returns the snapshot.
+func (t *TimerSnapshot) Snapshot() Timer { return t }
+
+// StdDev returns the standard deviation of the values at the time the snapshot
+// was taken.
+func (t *TimerSnapshot) StdDev() float64 { return t.histogram.StdDev() }
+
+// Sum returns the sum at the time the snapshot was taken.
+func (t *TimerSnapshot) Sum() int64 { return t.histogram.Sum() }
+
+// Time panics.
+func (*TimerSnapshot) Time(func()) {
+	panic("Time called on a TimerSnapshot")
+}
+
+// Update panics.
+func (*TimerSnapshot) Update(time.Duration) {
+	panic("Update called on a TimerSnapshot")
+}
+
+// UpdateSince panics.
+func (*TimerSnapshot) UpdateSince(time.Time) {
+	panic("UpdateSince called on a TimerSnapshot")
+}
+
+// Variance returns the variance of the values at the time the snapshot was
+// taken.
+func (t *TimerSnapshot) Variance() float64 { return t.histogram.Variance() }

Godeps/_workspace/src/github.com/rcrowley/go-metrics/validate.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+# check there are no formatting issues
+GOFMT_LINES=`gofmt -l . | wc -l | xargs`
+test $GOFMT_LINES -eq 0 || echo "gofmt needs to be run, ${GOFMT_LINES} files have issues"
+
+# run the tests for the root package
+go test .

Godeps/_workspace/src/github.com/rcrowley/go-metrics/writer.go

@@ -0,0 +1,100 @@
+package metrics
+
+import (
+	"fmt"
+	"io"
+	"sort"
+	"time"
+)
+
+// Write sorts writes each metric in the given registry periodically to the
+// given io.Writer.
+func Write(r Registry, d time.Duration, w io.Writer) {
+	for _ = range time.Tick(d) {
+		WriteOnce(r, w)
+	}
+}
+
+// WriteOnce sorts and writes metrics in the given registry to the given
+// io.Writer.
+func WriteOnce(r Registry, w io.Writer) {
+	var namedMetrics namedMetricSlice
+	r.Each(func(name string, i interface{}) {
+		namedMetrics = append(namedMetrics, namedMetric{name, i})
+	})
+
+	sort.Sort(namedMetrics)
+	for _, namedMetric := range namedMetrics {
+		switch metric := namedMetric.m.(type) {
+		case Counter:
+			fmt.Fprintf(w, "counter %s\n", namedMetric.name)
+			fmt.Fprintf(w, "  count:       %9d\n", metric.Count())
+		case Gauge:
+			fmt.Fprintf(w, "gauge %s\n", namedMetric.name)
+			fmt.Fprintf(w, "  value:       %9d\n", metric.Value())
+		case GaugeFloat64:
+			fmt.Fprintf(w, "gauge %s\n", namedMetric.name)
+			fmt.Fprintf(w, "  value:       %f\n", metric.Value())
+		case Healthcheck:
+			metric.Check()
+			fmt.Fprintf(w, "healthcheck %s\n", namedMetric.name)
+			fmt.Fprintf(w, "  error:       %v\n", metric.Error())
+		case Histogram:
+			h := metric.Snapshot()
+			ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
+			fmt.Fprintf(w, "histogram %s\n", namedMetric.name)
+			fmt.Fprintf(w, "  count:       %9d\n", h.Count())
+			fmt.Fprintf(w, "  min:         %9d\n", h.Min())
+			fmt.Fprintf(w, "  max:         %9d\n", h.Max())
+			fmt.Fprintf(w, "  mean:        %12.2f\n", h.Mean())
+			fmt.Fprintf(w, "  stddev:      %12.2f\n", h.StdDev())
+			fmt.Fprintf(w, "  median:      %12.2f\n", ps[0])
+			fmt.Fprintf(w, "  75%%:         %12.2f\n", ps[1])
+			fmt.Fprintf(w, "  95%%:         %12.2f\n", ps[2])
+			fmt.Fprintf(w, "  99%%:         %12.2f\n", ps[3])
+			fmt.Fprintf(w, "  99.9%%:       %12.2f\n", ps[4])
+		case Meter:
+			m := metric.Snapshot()
+			fmt.Fprintf(w, "meter %s\n", namedMetric.name)
+			fmt.Fprintf(w, "  count:       %9d\n", m.Count())
+			fmt.Fprintf(w, "  1-min rate:  %12.2f\n", m.Rate1())
+			fmt.Fprintf(w, "  5-min rate:  %12.2f\n", m.Rate5())
+			fmt.Fprintf(w, "  15-min rate: %12.2f\n", m.Rate15())
+			fmt.Fprintf(w, "  mean rate:   %12.2f\n", m.RateMean())
+		case Timer:
+			t := metric.Snapshot()
+			ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
+			fmt.Fprintf(w, "timer %s\n", namedMetric.name)
+			fmt.Fprintf(w, "  count:       %9d\n", t.Count())
+			fmt.Fprintf(w, "  min:         %9d\n", t.Min())
+			fmt.Fprintf(w, "  max:         %9d\n", t.Max())
+			fmt.Fprintf(w, "  mean:        %12.2f\n", t.Mean())
+			fmt.Fprintf(w, "  stddev:      %12.2f\n", t.StdDev())
+			fmt.Fprintf(w, "  median:      %12.2f\n", ps[0])
+			fmt.Fprintf(w, "  75%%:         %12.2f\n", ps[1])
+			fmt.Fprintf(w, "  95%%:         %12.2f\n", ps[2])
+			fmt.Fprintf(w, "  99%%:         %12.2f\n", ps[3])
+			fmt.Fprintf(w, "  99.9%%:       %12.2f\n", ps[4])
+			fmt.Fprintf(w, "  1-min rate:  %12.2f\n", t.Rate1())
+			fmt.Fprintf(w, "  5-min rate:  %12.2f\n", t.Rate5())
+			fmt.Fprintf(w, "  15-min rate: %12.2f\n", t.Rate15())
+			fmt.Fprintf(w, "  mean rate:   %12.2f\n", t.RateMean())
+		}
+	}
+}
+
+type namedMetric struct {
+	name string
+	m    interface{}
+}
+
+// namedMetricSlice is a slice of namedMetrics that implements sort.Interface.
+type namedMetricSlice []namedMetric
+
+func (nms namedMetricSlice) Len() int { return len(nms) }
+
+func (nms namedMetricSlice) Swap(i, j int) { nms[i], nms[j] = nms[j], nms[i] }
+
+func (nms namedMetricSlice) Less(i, j int) bool {
+	return nms[i].name < nms[j].name
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch_test.go

@@ -1,120 +0,0 @@
-// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package leveldb
-
-import (
-	"bytes"
-	"testing"
-
-	"github.com/syndtr/goleveldb/leveldb/comparer"
-	"github.com/syndtr/goleveldb/leveldb/memdb"
-)
-
-type tbRec struct {
-	kt         kType
-	key, value []byte
-}
-
-type testBatch struct {
-	rec []*tbRec
-}
-
-func (p *testBatch) Put(key, value []byte) {
-	p.rec = append(p.rec, &tbRec{ktVal, key, value})
-}
-
-func (p *testBatch) Delete(key []byte) {
-	p.rec = append(p.rec, &tbRec{ktDel, key, nil})
-}
-
-func compareBatch(t *testing.T, b1, b2 *Batch) {
-	if b1.seq != b2.seq {
-		t.Errorf("invalid seq number want %d, got %d", b1.seq, b2.seq)
-	}
-	if b1.Len() != b2.Len() {
-		t.Fatalf("invalid record length want %d, got %d", b1.Len(), b2.Len())
-	}
-	p1, p2 := new(testBatch), new(testBatch)
-	err := b1.Replay(p1)
-	if err != nil {
-		t.Fatal("error when replaying batch 1: ", err)
-	}
-	err = b2.Replay(p2)
-	if err != nil {
-		t.Fatal("error when replaying batch 2: ", err)
-	}
-	for i := range p1.rec {
-		r1, r2 := p1.rec[i], p2.rec[i]
-		if r1.kt != r2.kt {
-			t.Errorf("invalid type on record '%d' want %d, got %d", i, r1.kt, r2.kt)
-		}
-		if !bytes.Equal(r1.key, r2.key) {
-			t.Errorf("invalid key on record '%d' want %s, got %s", i, string(r1.key), string(r2.key))
-		}
-		if r1.kt == ktVal {
-			if !bytes.Equal(r1.value, r2.value) {
-				t.Errorf("invalid value on record '%d' want %s, got %s", i, string(r1.value), string(r2.value))
-			}
-		}
-	}
-}
-
-func TestBatch_EncodeDecode(t *testing.T) {
-	b1 := new(Batch)
-	b1.seq = 10009
-	b1.Put([]byte("key1"), []byte("value1"))
-	b1.Put([]byte("key2"), []byte("value2"))
-	b1.Delete([]byte("key1"))
-	b1.Put([]byte("k"), []byte(""))
-	b1.Put([]byte("zzzzzzzzzzz"), []byte("zzzzzzzzzzzzzzzzzzzzzzzz"))
-	b1.Delete([]byte("key10000"))
-	b1.Delete([]byte("k"))
-	buf := b1.encode()
-	b2 := new(Batch)
-	err := b2.decode(0, buf)
-	if err != nil {
-		t.Error("error when decoding batch: ", err)
-	}
-	compareBatch(t, b1, b2)
-}
-
-func TestBatch_Append(t *testing.T) {
-	b1 := new(Batch)
-	b1.seq = 10009
-	b1.Put([]byte("key1"), []byte("value1"))
-	b1.Put([]byte("key2"), []byte("value2"))
-	b1.Delete([]byte("key1"))
-	b1.Put([]byte("foo"), []byte("foovalue"))
-	b1.Put([]byte("bar"), []byte("barvalue"))
-	b2a := new(Batch)
-	b2a.seq = 10009
-	b2a.Put([]byte("key1"), []byte("value1"))
-	b2a.Put([]byte("key2"), []byte("value2"))
-	b2a.Delete([]byte("key1"))
-	b2b := new(Batch)
-	b2b.Put([]byte("foo"), []byte("foovalue"))
-	b2b.Put([]byte("bar"), []byte("barvalue"))
-	b2a.append(b2b)
-	compareBatch(t, b1, b2a)
-}
-
-func TestBatch_Size(t *testing.T) {
-	b := new(Batch)
-	for i := 0; i < 2; i++ {
-		b.Put([]byte("key1"), []byte("value1"))
-		b.Put([]byte("key2"), []byte("value2"))
-		b.Delete([]byte("key1"))
-		b.Put([]byte("foo"), []byte("foovalue"))
-		b.Put([]byte("bar"), []byte("barvalue"))
-		mem := memdb.New(&iComparer{comparer.DefaultComparer}, 0)
-		b.memReplay(mem)
-		if b.size() != mem.Size() {
-			t.Errorf("invalid batch size calculation, want=%d got=%d", mem.Size(), b.size())
-		}
-		b.Reset()
-	}
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench_test.go

@@ -1,464 +0,0 @@
-// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package leveldb
-
-import (
-	"bytes"
-	"fmt"
-	"math/rand"
-	"os"
-	"path/filepath"
-	"runtime"
-	"testing"
-
-	"github.com/syndtr/goleveldb/leveldb/iterator"
-	"github.com/syndtr/goleveldb/leveldb/opt"
-	"github.com/syndtr/goleveldb/leveldb/storage"
-)
-
-func randomString(r *rand.Rand, n int) []byte {
-	b := new(bytes.Buffer)
-	for i := 0; i < n; i++ {
-		b.WriteByte(' ' + byte(r.Intn(95)))
-	}
-	return b.Bytes()
-}
-
-func compressibleStr(r *rand.Rand, frac float32, n int) []byte {
-	nn := int(float32(n) * frac)
-	rb := randomString(r, nn)
-	b := make([]byte, 0, n+nn)
-	for len(b) < n {
-		b = append(b, rb...)
-	}
-	return b[:n]
-}
-
-type valueGen struct {
-	src []byte
-	pos int
-}
-
-func newValueGen(frac float32) *valueGen {
-	v := new(valueGen)
-	r := rand.New(rand.NewSource(301))
-	v.src = make([]byte, 0, 1048576+100)
-	for len(v.src) < 1048576 {
-		v.src = append(v.src, compressibleStr(r, frac, 100)...)
-	}
-	return v
-}
-
-func (v *valueGen) get(n int) []byte {
-	if v.pos+n > len(v.src) {
-		v.pos = 0
-	}
-	v.pos += n
-	return v.src[v.pos-n : v.pos]
-}
-
-var benchDB = filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbbench-%d", os.Getuid()))
-
-type dbBench struct {
-	b    *testing.B
-	stor storage.Storage
-	db   *DB
-
-	o  *opt.Options
-	ro *opt.ReadOptions
-	wo *opt.WriteOptions
-
-	keys, values [][]byte
-}
-
-func openDBBench(b *testing.B, noCompress bool) *dbBench {
-	_, err := os.Stat(benchDB)
-	if err == nil {
-		err = os.RemoveAll(benchDB)
-		if err != nil {
-			b.Fatal("cannot remove old db: ", err)
-		}
-	}
-
-	p := &dbBench{
-		b:  b,
-		o:  &opt.Options{},
-		ro: &opt.ReadOptions{},
-		wo: &opt.WriteOptions{},
-	}
-	p.stor, err = storage.OpenFile(benchDB)
-	if err != nil {
-		b.Fatal("cannot open stor: ", err)
-	}
-	if noCompress {
-		p.o.Compression = opt.NoCompression
-	}
-
-	p.db, err = Open(p.stor, p.o)
-	if err != nil {
-		b.Fatal("cannot open db: ", err)
-	}
-
-	runtime.GOMAXPROCS(runtime.NumCPU())
-	return p
-}
-
-func (p *dbBench) reopen() {
-	p.db.Close()
-	var err error
-	p.db, err = Open(p.stor, p.o)
-	if err != nil {
-		p.b.Fatal("Reopen: got error: ", err)
-	}
-}
-
-func (p *dbBench) populate(n int) {
-	p.keys, p.values = make([][]byte, n), make([][]byte, n)
-	v := newValueGen(0.5)
-	for i := range p.keys {
-		p.keys[i], p.values[i] = []byte(fmt.Sprintf("%016d", i)), v.get(100)
-	}
-}
-
-func (p *dbBench) randomize() {
-	m := len(p.keys)
-	times := m * 2
-	r1, r2 := rand.New(rand.NewSource(0xdeadbeef)), rand.New(rand.NewSource(0xbeefface))
-	for n := 0; n < times; n++ {
-		i, j := r1.Int()%m, r2.Int()%m
-		if i == j {
-			continue
-		}
-		p.keys[i], p.keys[j] = p.keys[j], p.keys[i]
-		p.values[i], p.values[j] = p.values[j], p.values[i]
-	}
-}
-
-func (p *dbBench) writes(perBatch int) {
-	b := p.b
-	db := p.db
-
-	n := len(p.keys)
-	m := n / perBatch
-	if n%perBatch > 0 {
-		m++
-	}
-	batches := make([]Batch, m)
-	j := 0
-	for i := range batches {
-		first := true
-		for ; j < n && ((j+1)%perBatch != 0 || first); j++ {
-			first = false
-			batches[i].Put(p.keys[j], p.values[j])
-		}
-	}
-	runtime.GC()
-
-	b.ResetTimer()
-	b.StartTimer()
-	for i := range batches {
-		err := db.Write(&(batches[i]), p.wo)
-		if err != nil {
-			b.Fatal("write failed: ", err)
-		}
-	}
-	b.StopTimer()
-	b.SetBytes(116)
-}
-
-func (p *dbBench) gc() {
-	p.keys, p.values = nil, nil
-	runtime.GC()
-}
-
-func (p *dbBench) puts() {
-	b := p.b
-	db := p.db
-
-	b.ResetTimer()
-	b.StartTimer()
-	for i := range p.keys {
-		err := db.Put(p.keys[i], p.values[i], p.wo)
-		if err != nil {
-			b.Fatal("put failed: ", err)
-		}
-	}
-	b.StopTimer()
-	b.SetBytes(116)
-}
-
-func (p *dbBench) fill() {
-	b := p.b
-	db := p.db
-
-	perBatch := 10000
-	batch := new(Batch)
-	for i, n := 0, len(p.keys); i < n; {
-		first := true
-		for ; i < n && ((i+1)%perBatch != 0 || first); i++ {
-			first = false
-			batch.Put(p.keys[i], p.values[i])
-		}
-		err := db.Write(batch, p.wo)
-		if err != nil {
-			b.Fatal("write failed: ", err)
-		}
-		batch.Reset()
-	}
-}
-
-func (p *dbBench) gets() {
-	b := p.b
-	db := p.db
-
-	b.ResetTimer()
-	for i := range p.keys {
-		_, err := db.Get(p.keys[i], p.ro)
-		if err != nil {
-			b.Error("got error: ", err)
-		}
-	}
-	b.StopTimer()
-}
-
-func (p *dbBench) seeks() {
-	b := p.b
-
-	iter := p.newIter()
-	defer iter.Release()
-	b.ResetTimer()
-	for i := range p.keys {
-		if !iter.Seek(p.keys[i]) {
-			b.Error("value not found for: ", string(p.keys[i]))
-		}
-	}
-	b.StopTimer()
-}
-
-func (p *dbBench) newIter() iterator.Iterator {
-	iter := p.db.NewIterator(nil, p.ro)
-	err := iter.Error()
-	if err != nil {
-		p.b.Fatal("cannot create iterator: ", err)
-	}
-	return iter
-}
-
-func (p *dbBench) close() {
-	if bp, err := p.db.GetProperty("leveldb.blockpool"); err == nil {
-		p.b.Log("Block pool stats: ", bp)
-	}
-	p.db.Close()
-	p.stor.Close()
-	os.RemoveAll(benchDB)
-	p.db = nil
-	p.keys = nil
-	p.values = nil
-	runtime.GC()
-	runtime.GOMAXPROCS(1)
-}
-
-func BenchmarkDBWrite(b *testing.B) {
-	p := openDBBench(b, false)
-	p.populate(b.N)
-	p.writes(1)
-	p.close()
-}
-
-func BenchmarkDBWriteBatch(b *testing.B) {
-	p := openDBBench(b, false)
-	p.populate(b.N)
-	p.writes(1000)
-	p.close()
-}
-
-func BenchmarkDBWriteUncompressed(b *testing.B) {
-	p := openDBBench(b, true)
-	p.populate(b.N)
-	p.writes(1)
-	p.close()
-}
-
-func BenchmarkDBWriteBatchUncompressed(b *testing.B) {
-	p := openDBBench(b, true)
-	p.populate(b.N)
-	p.writes(1000)
-	p.close()
-}
-
-func BenchmarkDBWriteRandom(b *testing.B) {
-	p := openDBBench(b, false)
-	p.populate(b.N)
-	p.randomize()
-	p.writes(1)
-	p.close()
-}
-
-func BenchmarkDBWriteRandomSync(b *testing.B) {
-	p := openDBBench(b, false)
-	p.wo.Sync = true
-	p.populate(b.N)
-	p.writes(1)
-	p.close()
-}
-
-func BenchmarkDBOverwrite(b *testing.B) {
-	p := openDBBench(b, false)
-	p.populate(b.N)
-	p.writes(1)
-	p.writes(1)
-	p.close()
-}
-
-func BenchmarkDBOverwriteRandom(b *testing.B) {
-	p := openDBBench(b, false)
-	p.populate(b.N)
-	p.writes(1)
-	p.randomize()
-	p.writes(1)
-	p.close()
-}
-
-func BenchmarkDBPut(b *testing.B) {
-	p := openDBBench(b, false)
-	p.populate(b.N)
-	p.puts()
-	p.close()
-}
-
-func BenchmarkDBRead(b *testing.B) {
-	p := openDBBench(b, false)
-	p.populate(b.N)
-	p.fill()
-	p.gc()
-
-	iter := p.newIter()
-	b.ResetTimer()
-	for iter.Next() {
-	}
-	iter.Release()
-	b.StopTimer()
-	b.SetBytes(116)
-	p.close()
-}
-
-func BenchmarkDBReadGC(b *testing.B) {
-	p := openDBBench(b, false)
-	p.populate(b.N)
-	p.fill()
-
-	iter := p.newIter()
-	b.ResetTimer()
-	for iter.Next() {
-	}
-	iter.Release()
-	b.StopTimer()
-	b.SetBytes(116)
-	p.close()
-}
-
-func BenchmarkDBReadUncompressed(b *testing.B) {
-	p := openDBBench(b, true)
-	p.populate(b.N)
-	p.fill()
-	p.gc()
-
-	iter := p.newIter()
-	b.ResetTimer()
-	for iter.Next() {
-	}
-	iter.Release()
-	b.StopTimer()
-	b.SetBytes(116)
-	p.close()
-}
-
-func BenchmarkDBReadTable(b *testing.B) {
-	p := openDBBench(b, false)
-	p.populate(b.N)
-	p.fill()
-	p.reopen()
-	p.gc()
-
-	iter := p.newIter()
-	b.ResetTimer()
-	for iter.Next() {
-	}
-	iter.Release()
-	b.StopTimer()
-	b.SetBytes(116)
-	p.close()
-}
-
-func BenchmarkDBReadReverse(b *testing.B) {
-	p := openDBBench(b, false)
-	p.populate(b.N)
-	p.fill()
-	p.gc()
-
-	iter := p.newIter()
-	b.ResetTimer()
-	iter.Last()
-	for iter.Prev() {
-	}
-	iter.Release()
-	b.StopTimer()
-	b.SetBytes(116)
-	p.close()
-}
-
-func BenchmarkDBReadReverseTable(b *testing.B) {
-	p := openDBBench(b, false)
-	p.populate(b.N)
-	p.fill()
-	p.reopen()
-	p.gc()
-
-	iter := p.newIter()
-	b.ResetTimer()
-	iter.Last()
-	for iter.Prev() {
-	}
-	iter.Release()
-	b.StopTimer()
-	b.SetBytes(116)
-	p.close()
-}
-
-func BenchmarkDBSeek(b *testing.B) {
-	p := openDBBench(b, false)
-	p.populate(b.N)
-	p.fill()
-	p.seeks()
-	p.close()
-}
-
-func BenchmarkDBSeekRandom(b *testing.B) {
-	p := openDBBench(b, false)
-	p.populate(b.N)
-	p.fill()
-	p.randomize()
-	p.seeks()
-	p.close()
-}
-
-func BenchmarkDBGet(b *testing.B) {
-	p := openDBBench(b, false)
-	p.populate(b.N)
-	p.fill()
-	p.gets()
-	p.close()
-}
-
-func BenchmarkDBGetRandom(b *testing.B) {
-	p := openDBBench(b, false)
-	p.populate(b.N)
-	p.fill()
-	p.randomize()
-	p.gets()
-	p.close()
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache.go

@@ -8,152 +8,669 @@
 package cache
 
 import (
+	"sync"
 	"sync/atomic"
+	"unsafe"
+
+	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
-// SetFunc is the function that will be called by Namespace.Get to create
-// a cache object, if charge is less than one than the cache object will
-// not be registered to cache tree, if value is nil then the cache object
-// will not be created.
-type SetFunc func() (charge int, value interface{})
-
-// DelFin is the function that will be called as the result of a delete operation.
-// Exist == true is indication that the object is exist, and pending == true is
-// indication of deletion already happen but haven't done yet (wait for all handles
-// to be released). And exist == false means the object doesn't exist.
-type DelFin func(exist, pending bool)
-
-// PurgeFin is the function that will be called as the result of a purge operation.
-type PurgeFin func(ns, key uint64)
-
-// Cache is a cache tree. A cache instance must be goroutine-safe.
-type Cache interface {
-	// SetCapacity sets cache tree capacity.
-	SetCapacity(capacity int)
-
-	// Capacity returns cache tree capacity.
+// Cacher provides interface to implements a caching functionality.
+// An implementation must be goroutine-safe.
+type Cacher interface {
+	// Capacity returns cache capacity.
 	Capacity() int
 
-	// Used returns used cache tree capacity.
-	Used() int
+	// SetCapacity sets cache capacity.
+	SetCapacity(capacity int)
 
-	// Size returns entire alive cache objects size.
-	Size() int
+	// Promote promotes the 'cache node'.
+	Promote(n *Node)
 
-	// NumObjects returns number of alive objects.
-	NumObjects() int
+	// Ban evicts the 'cache node' and prevent subsequent 'promote'.
+	Ban(n *Node)
 
-	// GetNamespace gets cache namespace with the given id.
-	// GetNamespace is never return nil.
-	GetNamespace(id uint64) Namespace
+	// Evict evicts the 'cache node'.
+	Evict(n *Node)
 
-	// PurgeNamespace purges cache namespace with the given id from this cache tree.
-	// Also read Namespace.Purge.
-	PurgeNamespace(id uint64, fin PurgeFin)
+	// EvictNS evicts 'cache node' with the given namespace.
+	EvictNS(ns uint64)
 
-	// ZapNamespace detaches cache namespace with the given id from this cache tree.
-	// Also read Namespace.Zap.
-	ZapNamespace(id uint64)
+	// EvictAll evicts all 'cache node'.
+	EvictAll()
 
-	// Purge purges all cache namespace from this cache tree.
-	// This is behave the same as calling Namespace.Purge method on all cache namespace.
-	Purge(fin PurgeFin)
-
-	// Zap detaches all cache namespace from this cache tree.
-	// This is behave the same as calling Namespace.Zap method on all cache namespace.
-	Zap()
+	// Close closes the 'cache tree'
+	Close() error
 }
 
-// Namespace is a cache namespace. A namespace instance must be goroutine-safe.
-type Namespace interface {
-	// Get gets cache object with the given key.
-	// If cache object is not found and setf is not nil, Get will atomically creates
-	// the cache object by calling setf. Otherwise Get will returns nil.
-	//
-	// The returned cache handle should be released after use by calling Release
-	// method.
-	Get(key uint64, setf SetFunc) Handle
+// Value is a 'cacheable object'. It may implements util.Releaser, if
+// so the the Release method will be called once object is released.
+type Value interface{}
 
-	// Delete removes cache object with the given key from cache tree.
-	// A deleted cache object will be released as soon as all of its handles have
-	// been released.
-	// Delete only happen once, subsequent delete will consider cache object doesn't
-	// exist, even if the cache object ins't released yet.
-	//
-	// If not nil, fin will be called if the cache object doesn't exist or when
-	// finally be released.
-	//
-	// Delete returns true if such cache object exist and never been deleted.
-	Delete(key uint64, fin DelFin) bool
-
-	// Purge removes all cache objects within this namespace from cache tree.
-	// This is the same as doing delete on all cache objects.
-	//
-	// If not nil, fin will be called on all cache objects when its finally be
-	// released.
-	Purge(fin PurgeFin)
-
-	// Zap detaches namespace from cache tree and release all its cache objects.
-	// A zapped namespace can never be filled again.
-	// Calling Get on zapped namespace will always return nil.
-	Zap()
+type CacheGetter struct {
+	Cache *Cache
+	NS    uint64
 }
 
-// Handle is a cache handle.
-type Handle interface {
-	// Release releases this cache handle. This method can be safely called mutiple
-	// times.
-	Release()
-
-	// Value returns value of this cache handle.
-	// Value will returns nil after this cache handle have be released.
-	Value() interface{}
+func (g *CacheGetter) Get(key uint64, setFunc func() (size int, value Value)) *Handle {
+	return g.Cache.Get(g.NS, key, setFunc)
 }
 
+// The hash tables implementation is based on:
+// "Dynamic-Sized Nonblocking Hash Tables", by Yujie Liu, Kunlong Zhang, and Michael Spear. ACM Symposium on Principles of Distributed Computing, Jul 2014.
+
 const (
-	DelNotExist = iota
-	DelExist
-	DelPendig
+	mInitialSize           = 1 << 4
+	mOverflowThreshold     = 1 << 5
+	mOverflowGrowThreshold = 1 << 7
 )
 
-// Namespace state.
-type nsState int
-
-const (
-	nsEffective nsState = iota
-	nsZapped
-)
-
-// Node state.
-type nodeState int
-
-const (
-	nodeZero nodeState = iota
-	nodeEffective
-	nodeEvicted
-	nodeDeleted
-)
-
-// Fake handle.
-type fakeHandle struct {
-	value interface{}
-	fin   func()
-	once  uint32
+type mBucket struct {
+	mu     sync.Mutex
+	node   []*Node
+	frozen bool
 }
 
-func (h *fakeHandle) Value() interface{} {
-	if atomic.LoadUint32(&h.once) == 0 {
-		return h.value
+func (b *mBucket) freeze() []*Node {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if !b.frozen {
+		b.frozen = true
+	}
+	return b.node
+}
+
+func (b *mBucket) get(r *Cache, h *mNode, hash uint32, ns, key uint64, noset bool) (done, added bool, n *Node) {
+	b.mu.Lock()
+
+	if b.frozen {
+		b.mu.Unlock()
+		return
+	}
+
+	// Scan the node.
+	for _, n := range b.node {
+		if n.hash == hash && n.ns == ns && n.key == key {
+			atomic.AddInt32(&n.ref, 1)
+			b.mu.Unlock()
+			return true, false, n
+		}
+	}
+
+	// Get only.
+	if noset {
+		b.mu.Unlock()
+		return true, false, nil
+	}
+
+	// Create node.
+	n = &Node{
+		r:    r,
+		hash: hash,
+		ns:   ns,
+		key:  key,
+		ref:  1,
+	}
+	// Add node to bucket.
+	b.node = append(b.node, n)
+	bLen := len(b.node)
+	b.mu.Unlock()
+
+	// Update counter.
+	grow := atomic.AddInt32(&r.nodes, 1) >= h.growThreshold
+	if bLen > mOverflowThreshold {
+		grow = grow || atomic.AddInt32(&h.overflow, 1) >= mOverflowGrowThreshold
+	}
+
+	// Grow.
+	if grow && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) {
+		nhLen := len(h.buckets) << 1
+		nh := &mNode{
+			buckets:         make([]unsafe.Pointer, nhLen),
+			mask:            uint32(nhLen) - 1,
+			pred:            unsafe.Pointer(h),
+			growThreshold:   int32(nhLen * mOverflowThreshold),
+			shrinkThreshold: int32(nhLen >> 1),
+		}
+		ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh))
+		if !ok {
+			panic("BUG: failed swapping head")
+		}
+		go nh.initBuckets()
+	}
+
+	return true, true, n
+}
+
+func (b *mBucket) delete(r *Cache, h *mNode, hash uint32, ns, key uint64) (done, deleted bool) {
+	b.mu.Lock()
+
+	if b.frozen {
+		b.mu.Unlock()
+		return
+	}
+
+	// Scan the node.
+	var (
+		n    *Node
+		bLen int
+	)
+	for i := range b.node {
+		n = b.node[i]
+		if n.ns == ns && n.key == key {
+			if atomic.LoadInt32(&n.ref) == 0 {
+				deleted = true
+
+				// Call releaser.
+				if n.value != nil {
+					if r, ok := n.value.(util.Releaser); ok {
+						r.Release()
+					}
+					n.value = nil
+				}
+
+				// Remove node from bucket.
+				b.node = append(b.node[:i], b.node[i+1:]...)
+				bLen = len(b.node)
+			}
+			break
+		}
+	}
+	b.mu.Unlock()
+
+	if deleted {
+		// Call OnDel.
+		for _, f := range n.onDel {
+			f()
+		}
+
+		// Update counter.
+		atomic.AddInt32(&r.size, int32(n.size)*-1)
+		shrink := atomic.AddInt32(&r.nodes, -1) < h.shrinkThreshold
+		if bLen >= mOverflowThreshold {
+			atomic.AddInt32(&h.overflow, -1)
+		}
+
+		// Shrink.
+		if shrink && len(h.buckets) > mInitialSize && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) {
+			nhLen := len(h.buckets) >> 1
+			nh := &mNode{
+				buckets:         make([]unsafe.Pointer, nhLen),
+				mask:            uint32(nhLen) - 1,
+				pred:            unsafe.Pointer(h),
+				growThreshold:   int32(nhLen * mOverflowThreshold),
+				shrinkThreshold: int32(nhLen >> 1),
+			}
+			ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh))
+			if !ok {
+				panic("BUG: failed swapping head")
+			}
+			go nh.initBuckets()
+		}
+	}
+
+	return true, deleted
+}
+
+type mNode struct {
+	buckets         []unsafe.Pointer // []*mBucket
+	mask            uint32
+	pred            unsafe.Pointer // *mNode
+	resizeInProgess int32
+
+	overflow        int32
+	growThreshold   int32
+	shrinkThreshold int32
+}
+
+func (n *mNode) initBucket(i uint32) *mBucket {
+	if b := (*mBucket)(atomic.LoadPointer(&n.buckets[i])); b != nil {
+		return b
+	}
+
+	p := (*mNode)(atomic.LoadPointer(&n.pred))
+	if p != nil {
+		var node []*Node
+		if n.mask > p.mask {
+			// Grow.
+			pb := (*mBucket)(atomic.LoadPointer(&p.buckets[i&p.mask]))
+			if pb == nil {
+				pb = p.initBucket(i & p.mask)
+			}
+			m := pb.freeze()
+			// Split nodes.
+			for _, x := range m {
+				if x.hash&n.mask == i {
+					node = append(node, x)
+				}
+			}
+		} else {
+			// Shrink.
+			pb0 := (*mBucket)(atomic.LoadPointer(&p.buckets[i]))
+			if pb0 == nil {
+				pb0 = p.initBucket(i)
+			}
+			pb1 := (*mBucket)(atomic.LoadPointer(&p.buckets[i+uint32(len(n.buckets))]))
+			if pb1 == nil {
+				pb1 = p.initBucket(i + uint32(len(n.buckets)))
+			}
+			m0 := pb0.freeze()
+			m1 := pb1.freeze()
+			// Merge nodes.
+			node = make([]*Node, 0, len(m0)+len(m1))
+			node = append(node, m0...)
+			node = append(node, m1...)
+		}
+		b := &mBucket{node: node}
+		if atomic.CompareAndSwapPointer(&n.buckets[i], nil, unsafe.Pointer(b)) {
+			if len(node) > mOverflowThreshold {
+				atomic.AddInt32(&n.overflow, int32(len(node)-mOverflowThreshold))
+			}
+			return b
+		}
+	}
+
+	return (*mBucket)(atomic.LoadPointer(&n.buckets[i]))
+}
+
+func (n *mNode) initBuckets() {
+	for i := range n.buckets {
+		n.initBucket(uint32(i))
+	}
+	atomic.StorePointer(&n.pred, nil)
+}
+
+// Cache is a 'cache map'.
+type Cache struct {
+	mu     sync.RWMutex
+	mHead  unsafe.Pointer // *mNode
+	nodes  int32
+	size   int32
+	cacher Cacher
+	closed bool
+}
+
+// NewCache creates a new 'cache map'. The cacher is optional and
+// may be nil.
+func NewCache(cacher Cacher) *Cache {
+	h := &mNode{
+		buckets:         make([]unsafe.Pointer, mInitialSize),
+		mask:            mInitialSize - 1,
+		growThreshold:   int32(mInitialSize * mOverflowThreshold),
+		shrinkThreshold: 0,
+	}
+	for i := range h.buckets {
+		h.buckets[i] = unsafe.Pointer(&mBucket{})
+	}
+	r := &Cache{
+		mHead:  unsafe.Pointer(h),
+		cacher: cacher,
+	}
+	return r
+}
+
+func (r *Cache) getBucket(hash uint32) (*mNode, *mBucket) {
+	h := (*mNode)(atomic.LoadPointer(&r.mHead))
+	i := hash & h.mask
+	b := (*mBucket)(atomic.LoadPointer(&h.buckets[i]))
+	if b == nil {
+		b = h.initBucket(i)
+	}
+	return h, b
+}
+
+func (r *Cache) delete(n *Node) bool {
+	for {
+		h, b := r.getBucket(n.hash)
+		done, deleted := b.delete(r, h, n.hash, n.ns, n.key)
+		if done {
+			return deleted
+		}
+	}
+	return false
+}
+
+// Nodes returns number of 'cache node' in the map.
+func (r *Cache) Nodes() int {
+	return int(atomic.LoadInt32(&r.nodes))
+}
+
+// Size returns sums of 'cache node' size in the map.
+func (r *Cache) Size() int {
+	return int(atomic.LoadInt32(&r.size))
+}
+
+// Capacity returns cache capacity.
+func (r *Cache) Capacity() int {
+	if r.cacher == nil {
+		return 0
+	}
+	return r.cacher.Capacity()
+}
+
+// SetCapacity sets cache capacity.
+func (r *Cache) SetCapacity(capacity int) {
+	if r.cacher != nil {
+		r.cacher.SetCapacity(capacity)
+	}
+}
+
+// Get gets 'cache node' with the given namespace and key.
+// If cache node is not found and setFunc is not nil, Get will atomically creates
+// the 'cache node' by calling setFunc. Otherwise Get will returns nil.
+//
+// The returned 'cache handle' should be released after use by calling Release
+// method.
+func (r *Cache) Get(ns, key uint64, setFunc func() (size int, value Value)) *Handle {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	if r.closed {
+		return nil
+	}
+
+	hash := murmur32(ns, key, 0xf00)
+	for {
+		h, b := r.getBucket(hash)
+		done, _, n := b.get(r, h, hash, ns, key, setFunc == nil)
+		if done {
+			if n != nil {
+				n.mu.Lock()
+				if n.value == nil {
+					if setFunc == nil {
+						n.mu.Unlock()
+						n.unref()
+						return nil
+					}
+
+					n.size, n.value = setFunc()
+					if n.value == nil {
+						n.size = 0
+						n.mu.Unlock()
+						n.unref()
+						return nil
+					}
+					atomic.AddInt32(&r.size, int32(n.size))
+				}
+				n.mu.Unlock()
+				if r.cacher != nil {
+					r.cacher.Promote(n)
+				}
+				return &Handle{unsafe.Pointer(n)}
+			}
+
+			break
+		}
 	}
 	return nil
 }
 
-func (h *fakeHandle) Release() {
-	if !atomic.CompareAndSwapUint32(&h.once, 0, 1) {
+// Delete removes and ban 'cache node' with the given namespace and key.
+// A banned 'cache node' will never inserted into the 'cache tree'. Ban
+// only attributed to the particular 'cache node', so when a 'cache node'
+// is recreated it will not be banned.
+//
+// If onDel is not nil, then it will be executed if such 'cache node'
+// doesn't exist or once the 'cache node' is released.
+//
+// Delete return true is such 'cache node' exist.
+func (r *Cache) Delete(ns, key uint64, onDel func()) bool {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	if r.closed {
+		return false
+	}
+
+	hash := murmur32(ns, key, 0xf00)
+	for {
+		h, b := r.getBucket(hash)
+		done, _, n := b.get(r, h, hash, ns, key, true)
+		if done {
+			if n != nil {
+				if onDel != nil {
+					n.mu.Lock()
+					n.onDel = append(n.onDel, onDel)
+					n.mu.Unlock()
+				}
+				if r.cacher != nil {
+					r.cacher.Ban(n)
+				}
+				n.unref()
+				return true
+			}
+
+			break
+		}
+	}
+
+	if onDel != nil {
+		onDel()
+	}
+
+	return false
+}
+
+// Evict evicts 'cache node' with the given namespace and key. This will
+// simply call Cacher.Evict.
+//
+// Evict return true is such 'cache node' exist.
+func (r *Cache) Evict(ns, key uint64) bool {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	if r.closed {
+		return false
+	}
+
+	hash := murmur32(ns, key, 0xf00)
+	for {
+		h, b := r.getBucket(hash)
+		done, _, n := b.get(r, h, hash, ns, key, true)
+		if done {
+			if n != nil {
+				if r.cacher != nil {
+					r.cacher.Evict(n)
+				}
+				n.unref()
+				return true
+			}
+
+			break
+		}
+	}
+
+	return false
+}
+
+// EvictNS evicts 'cache node' with the given namespace. This will
+// simply call Cacher.EvictNS.
+func (r *Cache) EvictNS(ns uint64) {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	if r.closed {
 		return
 	}
-	if h.fin != nil {
-		h.fin()
-		h.fin = nil
+
+	if r.cacher != nil {
+		r.cacher.EvictNS(ns)
 	}
 }
+
+// EvictAll evicts all 'cache node'. This will simply call Cacher.EvictAll.
+func (r *Cache) EvictAll() {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	if r.closed {
+		return
+	}
+
+	if r.cacher != nil {
+		r.cacher.EvictAll()
+	}
+}
+
+// Close closes the 'cache map' and releases all 'cache node'.
+func (r *Cache) Close() error {
+	r.mu.Lock()
+	if !r.closed {
+		r.closed = true
+
+		if r.cacher != nil {
+			if err := r.cacher.Close(); err != nil {
+				return err
+			}
+		}
+
+		h := (*mNode)(r.mHead)
+		h.initBuckets()
+
+		for i := range h.buckets {
+			b := (*mBucket)(h.buckets[i])
+			for _, n := range b.node {
+				// Call releaser.
+				if n.value != nil {
+					if r, ok := n.value.(util.Releaser); ok {
+						r.Release()
+					}
+					n.value = nil
+				}
+
+				// Call OnDel.
+				for _, f := range n.onDel {
+					f()
+				}
+			}
+		}
+	}
+	r.mu.Unlock()
+	return nil
+}
+
+// Node is a 'cache node'.
+type Node struct {
+	r *Cache
+
+	hash    uint32
+	ns, key uint64
+
+	mu    sync.Mutex
+	size  int
+	value Value
+
+	ref   int32
+	onDel []func()
+
+	CacheData unsafe.Pointer
+}
+
+// NS returns this 'cache node' namespace.
+func (n *Node) NS() uint64 {
+	return n.ns
+}
+
+// Key returns this 'cache node' key.
+func (n *Node) Key() uint64 {
+	return n.key
+}
+
+// Size returns this 'cache node' size.
+func (n *Node) Size() int {
+	return n.size
+}
+
+// Value returns this 'cache node' value.
+func (n *Node) Value() Value {
+	return n.value
+}
+
+// Ref returns this 'cache node' ref counter.
+func (n *Node) Ref() int32 {
+	return atomic.LoadInt32(&n.ref)
+}
+
+// GetHandle returns an handle for this 'cache node'.
+func (n *Node) GetHandle() *Handle {
+	if atomic.AddInt32(&n.ref, 1) <= 1 {
+		panic("BUG: Node.GetHandle on zero ref")
+	}
+	return &Handle{unsafe.Pointer(n)}
+}
+
+func (n *Node) unref() {
+	if atomic.AddInt32(&n.ref, -1) == 0 {
+		n.r.delete(n)
+	}
+}
+
+func (n *Node) unrefLocked() {
+	if atomic.AddInt32(&n.ref, -1) == 0 {
+		n.r.mu.RLock()
+		if !n.r.closed {
+			n.r.delete(n)
+		}
+		n.r.mu.RUnlock()
+	}
+}
+
+type Handle struct {
+	n unsafe.Pointer // *Node
+}
+
+func (h *Handle) Value() Value {
+	n := (*Node)(atomic.LoadPointer(&h.n))
+	if n != nil {
+		return n.value
+	}
+	return nil
+}
+
+func (h *Handle) Release() {
+	nPtr := atomic.LoadPointer(&h.n)
+	if nPtr != nil && atomic.CompareAndSwapPointer(&h.n, nPtr, nil) {
+		n := (*Node)(nPtr)
+		n.unrefLocked()
+	}
+}
+
+func murmur32(ns, key uint64, seed uint32) uint32 {
+	const (
+		m = uint32(0x5bd1e995)
+		r = 24
+	)
+
+	k1 := uint32(ns >> 32)
+	k2 := uint32(ns)
+	k3 := uint32(key >> 32)
+	k4 := uint32(key)
+
+	k1 *= m
+	k1 ^= k1 >> r
+	k1 *= m
+
+	k2 *= m
+	k2 ^= k2 >> r
+	k2 *= m
+
+	k3 *= m
+	k3 ^= k3 >> r
+	k3 *= m
+
+	k4 *= m
+	k4 ^= k4 >> r
+	k4 *= m
+
+	h := seed
+
+	h *= m
+	h ^= k1
+	h *= m
+	h ^= k2
+	h *= m
+	h ^= k3
+	h *= m
+	h ^= k4
+
+	h ^= h >> 13
+	h *= m
+	h ^= h >> 15
+
+	return h
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go

@@ -1,655 +0,0 @@
-// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package cache
-
-import (
-	"math/rand"
-	"runtime"
-	"sync"
-	"sync/atomic"
-	"testing"
-	"time"
-)
-
-type releaserFunc struct {
-	fn    func()
-	value interface{}
-}
-
-func (r releaserFunc) Release() {
-	if r.fn != nil {
-		r.fn()
-	}
-}
-
-func set(ns Namespace, key uint64, value interface{}, charge int, relf func()) Handle {
-	return ns.Get(key, func() (int, interface{}) {
-		if relf != nil {
-			return charge, releaserFunc{relf, value}
-		} else {
-			return charge, value
-		}
-	})
-}
-
-func TestCache_HitMiss(t *testing.T) {
-	cases := []struct {
-		key   uint64
-		value string
-	}{
-		{1, "vvvvvvvvv"},
-		{100, "v1"},
-		{0, "v2"},
-		{12346, "v3"},
-		{777, "v4"},
-		{999, "v5"},
-		{7654, "v6"},
-		{2, "v7"},
-		{3, "v8"},
-		{9, "v9"},
-	}
-
-	setfin := 0
-	c := NewLRUCache(1000)
-	ns := c.GetNamespace(0)
-	for i, x := range cases {
-		set(ns, x.key, x.value, len(x.value), func() {
-			setfin++
-		}).Release()
-		for j, y := range cases {
-			h := ns.Get(y.key, nil)
-			if j <= i {
-				// should hit
-				if h == nil {
-					t.Errorf("case '%d' iteration '%d' is miss", i, j)
-				} else {
-					if x := h.Value().(releaserFunc).value.(string); x != y.value {
-						t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, x, y.value)
-					}
-				}
-			} else {
-				// should miss
-				if h != nil {
-					t.Errorf("case '%d' iteration '%d' is hit , value '%s'", i, j, h.Value().(releaserFunc).value.(string))
-				}
-			}
-			if h != nil {
-				h.Release()
-			}
-		}
-	}
-
-	for i, x := range cases {
-		finalizerOk := false
-		ns.Delete(x.key, func(exist, pending bool) {
-			finalizerOk = true
-		})
-
-		if !finalizerOk {
-			t.Errorf("case %d delete finalizer not executed", i)
-		}
-
-		for j, y := range cases {
-			h := ns.Get(y.key, nil)
-			if j > i {
-				// should hit
-				if h == nil {
-					t.Errorf("case '%d' iteration '%d' is miss", i, j)
-				} else {
-					if x := h.Value().(releaserFunc).value.(string); x != y.value {
-						t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, x, y.value)
-					}
-				}
-			} else {
-				// should miss
-				if h != nil {
-					t.Errorf("case '%d' iteration '%d' is hit, value '%s'", i, j, h.Value().(releaserFunc).value.(string))
-				}
-			}
-			if h != nil {
-				h.Release()
-			}
-		}
-	}
-
-	if setfin != len(cases) {
-		t.Errorf("some set finalizer may not be executed, want=%d got=%d", len(cases), setfin)
-	}
-}
-
-func TestLRUCache_Eviction(t *testing.T) {
-	c := NewLRUCache(12)
-	ns := c.GetNamespace(0)
-	o1 := set(ns, 1, 1, 1, nil)
-	set(ns, 2, 2, 1, nil).Release()
-	set(ns, 3, 3, 1, nil).Release()
-	set(ns, 4, 4, 1, nil).Release()
-	set(ns, 5, 5, 1, nil).Release()
-	if h := ns.Get(2, nil); h != nil { // 1,3,4,5,2
-		h.Release()
-	}
-	set(ns, 9, 9, 10, nil).Release() // 5,2,9
-
-	for _, key := range []uint64{9, 2, 5, 1} {
-		h := ns.Get(key, nil)
-		if h == nil {
-			t.Errorf("miss for key '%d'", key)
-		} else {
-			if x := h.Value().(int); x != int(key) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
-			}
-			h.Release()
-		}
-	}
-	o1.Release()
-	for _, key := range []uint64{1, 2, 5} {
-		h := ns.Get(key, nil)
-		if h == nil {
-			t.Errorf("miss for key '%d'", key)
-		} else {
-			if x := h.Value().(int); x != int(key) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
-			}
-			h.Release()
-		}
-	}
-	for _, key := range []uint64{3, 4, 9} {
-		h := ns.Get(key, nil)
-		if h != nil {
-			t.Errorf("hit for key '%d'", key)
-			if x := h.Value().(int); x != int(key) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
-			}
-			h.Release()
-		}
-	}
-}
-
-func TestLRUCache_SetGet(t *testing.T) {
-	c := NewLRUCache(13)
-	ns := c.GetNamespace(0)
-	for i := 0; i < 200; i++ {
-		n := uint64(rand.Intn(99999) % 20)
-		set(ns, n, n, 1, nil).Release()
-		if h := ns.Get(n, nil); h != nil {
-			if h.Value() == nil {
-				t.Errorf("key '%d' contains nil value", n)
-			} else {
-				if x := h.Value().(uint64); x != n {
-					t.Errorf("invalid value for key '%d' want '%d', got '%d'", n, n, x)
-				}
-			}
-			h.Release()
-		} else {
-			t.Errorf("key '%d' doesn't exist", n)
-		}
-	}
-}
-
-func TestLRUCache_Purge(t *testing.T) {
-	c := NewLRUCache(3)
-	ns1 := c.GetNamespace(0)
-	o1 := set(ns1, 1, 1, 1, nil)
-	o2 := set(ns1, 2, 2, 1, nil)
-	ns1.Purge(nil)
-	set(ns1, 3, 3, 1, nil).Release()
-	for _, key := range []uint64{1, 2, 3} {
-		h := ns1.Get(key, nil)
-		if h == nil {
-			t.Errorf("miss for key '%d'", key)
-		} else {
-			if x := h.Value().(int); x != int(key) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
-			}
-			h.Release()
-		}
-	}
-	o1.Release()
-	o2.Release()
-	for _, key := range []uint64{1, 2} {
-		h := ns1.Get(key, nil)
-		if h != nil {
-			t.Errorf("hit for key '%d'", key)
-			if x := h.Value().(int); x != int(key) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
-			}
-			h.Release()
-		}
-	}
-}
-
-type testingCacheObjectCounter struct {
-	created  uint
-	released uint
-}
-
-func (c *testingCacheObjectCounter) createOne() {
-	c.created++
-}
-
-func (c *testingCacheObjectCounter) releaseOne() {
-	c.released++
-}
-
-type testingCacheObject struct {
-	t   *testing.T
-	cnt *testingCacheObjectCounter
-
-	ns, key uint64
-
-	releaseCalled bool
-}
-
-func (x *testingCacheObject) Release() {
-	if !x.releaseCalled {
-		x.releaseCalled = true
-		x.cnt.releaseOne()
-	} else {
-		x.t.Errorf("duplicate setfin NS#%d KEY#%d", x.ns, x.key)
-	}
-}
-
-func TestLRUCache_ConcurrentSetGet(t *testing.T) {
-	runtime.GOMAXPROCS(runtime.NumCPU())
-
-	seed := time.Now().UnixNano()
-	t.Logf("seed=%d", seed)
-
-	const (
-		N = 2000000
-		M = 4000
-		C = 3
-	)
-
-	var set, get uint32
-
-	wg := &sync.WaitGroup{}
-	c := NewLRUCache(M / 4)
-	for ni := uint64(0); ni < C; ni++ {
-		r0 := rand.New(rand.NewSource(seed + int64(ni)))
-		r1 := rand.New(rand.NewSource(seed + int64(ni) + 1))
-		ns := c.GetNamespace(ni)
-
-		wg.Add(2)
-		go func(ns Namespace, r *rand.Rand) {
-			for i := 0; i < N; i++ {
-				x := uint64(r.Int63n(M))
-				o := ns.Get(x, func() (int, interface{}) {
-					atomic.AddUint32(&set, 1)
-					return 1, x
-				})
-				if v := o.Value().(uint64); v != x {
-					t.Errorf("#%d invalid value, got=%d", x, v)
-				}
-				o.Release()
-			}
-			wg.Done()
-		}(ns, r0)
-		go func(ns Namespace, r *rand.Rand) {
-			for i := 0; i < N; i++ {
-				x := uint64(r.Int63n(M))
-				o := ns.Get(x, nil)
-				if o != nil {
-					atomic.AddUint32(&get, 1)
-					if v := o.Value().(uint64); v != x {
-						t.Errorf("#%d invalid value, got=%d", x, v)
-					}
-					o.Release()
-				}
-			}
-			wg.Done()
-		}(ns, r1)
-	}
-
-	wg.Wait()
-
-	t.Logf("set=%d get=%d", set, get)
-}
-
-func TestLRUCache_Finalizer(t *testing.T) {
-	const (
-		capacity   = 100
-		goroutines = 100
-		iterations = 10000
-		keymax     = 8000
-	)
-
-	cnt := &testingCacheObjectCounter{}
-
-	c := NewLRUCache(capacity)
-
-	type instance struct {
-		seed       int64
-		rnd        *rand.Rand
-		nsid       uint64
-		ns         Namespace
-		effective  int
-		handles    []Handle
-		handlesMap map[uint64]int
-
-		delete          bool
-		purge           bool
-		zap             bool
-		wantDel         int
-		delfinCalled    int
-		delfinCalledAll int
-		delfinCalledEff int
-		purgefinCalled  int
-	}
-
-	instanceGet := func(p *instance, key uint64) {
-		h := p.ns.Get(key, func() (charge int, value interface{}) {
-			to := &testingCacheObject{
-				t: t, cnt: cnt,
-				ns:  p.nsid,
-				key: key,
-			}
-			p.effective++
-			cnt.createOne()
-			return 1, releaserFunc{func() {
-				to.Release()
-				p.effective--
-			}, to}
-		})
-		p.handles = append(p.handles, h)
-		p.handlesMap[key] = p.handlesMap[key] + 1
-	}
-	instanceRelease := func(p *instance, i int) {
-		h := p.handles[i]
-		key := h.Value().(releaserFunc).value.(*testingCacheObject).key
-		if n := p.handlesMap[key]; n == 0 {
-			t.Fatal("key ref == 0")
-		} else if n > 1 {
-			p.handlesMap[key] = n - 1
-		} else {
-			delete(p.handlesMap, key)
-		}
-		h.Release()
-		p.handles = append(p.handles[:i], p.handles[i+1:]...)
-		p.handles[len(p.handles) : len(p.handles)+1][0] = nil
-	}
-
-	seed := time.Now().UnixNano()
-	t.Logf("seed=%d", seed)
-
-	instances := make([]*instance, goroutines)
-	for i := range instances {
-		p := &instance{}
-		p.handlesMap = make(map[uint64]int)
-		p.seed = seed + int64(i)
-		p.rnd = rand.New(rand.NewSource(p.seed))
-		p.nsid = uint64(i)
-		p.ns = c.GetNamespace(p.nsid)
-		p.delete = i%6 == 0
-		p.purge = i%8 == 0
-		p.zap = i%12 == 0 || i%3 == 0
-		instances[i] = p
-	}
-
-	runr := rand.New(rand.NewSource(seed - 1))
-	run := func(rnd *rand.Rand, x []*instance, init func(p *instance) bool, fn func(p *instance, i int) bool) {
-		var (
-			rx []*instance
-			rn []int
-		)
-		if init == nil {
-			rx = append([]*instance{}, x...)
-			rn = make([]int, len(x))
-		} else {
-			for _, p := range x {
-				if init(p) {
-					rx = append(rx, p)
-					rn = append(rn, 0)
-				}
-			}
-		}
-		for len(rx) > 0 {
-			i := rand.Intn(len(rx))
-			if fn(rx[i], rn[i]) {
-				rn[i]++
-			} else {
-				rx = append(rx[:i], rx[i+1:]...)
-				rn = append(rn[:i], rn[i+1:]...)
-			}
-		}
-	}
-
-	// Get and release.
-	run(runr, instances, nil, func(p *instance, i int) bool {
-		if i < iterations {
-			if len(p.handles) == 0 || p.rnd.Int()%2 == 0 {
-				instanceGet(p, uint64(p.rnd.Intn(keymax)))
-			} else {
-				instanceRelease(p, p.rnd.Intn(len(p.handles)))
-			}
-			return true
-		} else {
-			return false
-		}
-	})
-
-	if used, cap := c.Used(), c.Capacity(); used > cap {
-		t.Errorf("Used > capacity, used=%d cap=%d", used, cap)
-	}
-
-	// Check effective objects.
-	for i, p := range instances {
-		if int(p.effective) < len(p.handlesMap) {
-			t.Errorf("#%d effective objects < acquired handle, eo=%d ah=%d", i, p.effective, len(p.handlesMap))
-		}
-	}
-
-	if want := int(cnt.created - cnt.released); c.Size() != want {
-		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
-	}
-
-	// First delete.
-	run(runr, instances, func(p *instance) bool {
-		p.wantDel = p.effective
-		return p.delete
-	}, func(p *instance, i int) bool {
-		key := uint64(i)
-		if key < keymax {
-			_, wantExist := p.handlesMap[key]
-			gotExist := p.ns.Delete(key, func(exist, pending bool) {
-				p.delfinCalledAll++
-				if exist {
-					p.delfinCalledEff++
-				}
-			})
-			if !gotExist && wantExist {
-				t.Errorf("delete on NS#%d KEY#%d not found", p.nsid, key)
-			}
-			return true
-		} else {
-			return false
-		}
-	})
-
-	// Second delete.
-	run(runr, instances, func(p *instance) bool {
-		p.delfinCalled = 0
-		return p.delete
-	}, func(p *instance, i int) bool {
-		key := uint64(i)
-		if key < keymax {
-			gotExist := p.ns.Delete(key, func(exist, pending bool) {
-				if exist && !pending {
-					t.Errorf("delete fin on NS#%d KEY#%d exist and not pending for deletion", p.nsid, key)
-				}
-				p.delfinCalled++
-			})
-			if gotExist {
-				t.Errorf("delete on NS#%d KEY#%d found", p.nsid, key)
-			}
-			return true
-		} else {
-			if p.delfinCalled != keymax {
-				t.Errorf("(2) NS#%d not all delete fin called, diff=%d", p.nsid, keymax-p.delfinCalled)
-			}
-			return false
-		}
-	})
-
-	// Purge.
-	run(runr, instances, func(p *instance) bool {
-		return p.purge
-	}, func(p *instance, i int) bool {
-		p.ns.Purge(func(ns, key uint64) {
-			p.purgefinCalled++
-		})
-		return false
-	})
-
-	if want := int(cnt.created - cnt.released); c.Size() != want {
-		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
-	}
-
-	// Release.
-	run(runr, instances, func(p *instance) bool {
-		return !p.zap
-	}, func(p *instance, i int) bool {
-		if len(p.handles) > 0 {
-			instanceRelease(p, len(p.handles)-1)
-			return true
-		} else {
-			return false
-		}
-	})
-
-	if want := int(cnt.created - cnt.released); c.Size() != want {
-		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
-	}
-
-	// Zap.
-	run(runr, instances, func(p *instance) bool {
-		return p.zap
-	}, func(p *instance, i int) bool {
-		p.ns.Zap()
-		p.handles = nil
-		p.handlesMap = nil
-		return false
-	})
-
-	if want := int(cnt.created - cnt.released); c.Size() != want {
-		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
-	}
-
-	if notrel, used := int(cnt.created-cnt.released), c.Used(); notrel != used {
-		t.Errorf("Invalid used value, want=%d got=%d", notrel, used)
-	}
-
-	c.Purge(nil)
-
-	for _, p := range instances {
-		if p.delete {
-			if p.delfinCalledAll != keymax {
-				t.Errorf("#%d not all delete fin called, purge=%v zap=%v diff=%d", p.nsid, p.purge, p.zap, keymax-p.delfinCalledAll)
-			}
-			if p.delfinCalledEff != p.wantDel {
-				t.Errorf("#%d not all effective delete fin called, diff=%d", p.nsid, p.wantDel-p.delfinCalledEff)
-			}
-			if p.purge && p.purgefinCalled > 0 {
-				t.Errorf("#%d some purge fin called, delete=%v zap=%v n=%d", p.nsid, p.delete, p.zap, p.purgefinCalled)
-			}
-		} else {
-			if p.purge {
-				if p.purgefinCalled != p.wantDel {
-					t.Errorf("#%d not all purge fin called, delete=%v zap=%v diff=%d", p.nsid, p.delete, p.zap, p.wantDel-p.purgefinCalled)
-				}
-			}
-		}
-	}
-
-	if cnt.created != cnt.released {
-		t.Errorf("Some cache object weren't released, created=%d released=%d", cnt.created, cnt.released)
-	}
-}
-
-func BenchmarkLRUCache_Set(b *testing.B) {
-	c := NewLRUCache(0)
-	ns := c.GetNamespace(0)
-	b.ResetTimer()
-	for i := uint64(0); i < uint64(b.N); i++ {
-		set(ns, i, "", 1, nil)
-	}
-}
-
-func BenchmarkLRUCache_Get(b *testing.B) {
-	c := NewLRUCache(0)
-	ns := c.GetNamespace(0)
-	b.ResetTimer()
-	for i := uint64(0); i < uint64(b.N); i++ {
-		set(ns, i, "", 1, nil)
-	}
-	b.ResetTimer()
-	for i := uint64(0); i < uint64(b.N); i++ {
-		ns.Get(i, nil)
-	}
-}
-
-func BenchmarkLRUCache_Get2(b *testing.B) {
-	c := NewLRUCache(0)
-	ns := c.GetNamespace(0)
-	b.ResetTimer()
-	for i := uint64(0); i < uint64(b.N); i++ {
-		set(ns, i, "", 1, nil)
-	}
-	b.ResetTimer()
-	for i := uint64(0); i < uint64(b.N); i++ {
-		ns.Get(i, func() (charge int, value interface{}) {
-			return 0, nil
-		})
-	}
-}
-
-func BenchmarkLRUCache_Release(b *testing.B) {
-	c := NewLRUCache(0)
-	ns := c.GetNamespace(0)
-	handles := make([]Handle, b.N)
-	for i := uint64(0); i < uint64(b.N); i++ {
-		handles[i] = set(ns, i, "", 1, nil)
-	}
-	b.ResetTimer()
-	for _, h := range handles {
-		h.Release()
-	}
-}
-
-func BenchmarkLRUCache_SetRelease(b *testing.B) {
-	capacity := b.N / 100
-	if capacity <= 0 {
-		capacity = 10
-	}
-	c := NewLRUCache(capacity)
-	ns := c.GetNamespace(0)
-	b.ResetTimer()
-	for i := uint64(0); i < uint64(b.N); i++ {
-		set(ns, i, "", 1, nil).Release()
-	}
-}
-
-func BenchmarkLRUCache_SetReleaseTwice(b *testing.B) {
-	capacity := b.N / 100
-	if capacity <= 0 {
-		capacity = 10
-	}
-	c := NewLRUCache(capacity)
-	ns := c.GetNamespace(0)
-	b.ResetTimer()
-
-	na := b.N / 2
-	nb := b.N - na
-
-	for i := uint64(0); i < uint64(na); i++ {
-		set(ns, i, "", 1, nil).Release()
-	}
-
-	for i := uint64(0); i < uint64(nb); i++ {
-		set(ns, i, "", 1, nil).Release()
-	}
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru.go

@@ -0,0 +1,195 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package cache
+
+import (
+	"sync"
+	"unsafe"
+)
+
+type lruNode struct {
+	n   *Node
+	h   *Handle
+	ban bool
+
+	next, prev *lruNode
+}
+
+func (n *lruNode) insert(at *lruNode) {
+	x := at.next
+	at.next = n
+	n.prev = at
+	n.next = x
+	x.prev = n
+}
+
+func (n *lruNode) remove() {
+	if n.prev != nil {
+		n.prev.next = n.next
+		n.next.prev = n.prev
+		n.prev = nil
+		n.next = nil
+	} else {
+		panic("BUG: removing removed node")
+	}
+}
+
+type lru struct {
+	mu       sync.Mutex
+	capacity int
+	used     int
+	recent   lruNode
+}
+
+func (r *lru) reset() {
+	r.recent.next = &r.recent
+	r.recent.prev = &r.recent
+	r.used = 0
+}
+
+func (r *lru) Capacity() int {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return r.capacity
+}
+
+func (r *lru) SetCapacity(capacity int) {
+	var evicted []*lruNode
+
+	r.mu.Lock()
+	r.capacity = capacity
+	for r.used > r.capacity {
+		rn := r.recent.prev
+		if rn == nil {
+			panic("BUG: invalid LRU used or capacity counter")
+		}
+		rn.remove()
+		rn.n.CacheData = nil
+		r.used -= rn.n.Size()
+		evicted = append(evicted, rn)
+	}
+	r.mu.Unlock()
+
+	for _, rn := range evicted {
+		rn.h.Release()
+	}
+}
+
+func (r *lru) Promote(n *Node) {
+	var evicted []*lruNode
+
+	r.mu.Lock()
+	if n.CacheData == nil {
+		if n.Size() <= r.capacity {
+			rn := &lruNode{n: n, h: n.GetHandle()}
+			rn.insert(&r.recent)
+			n.CacheData = unsafe.Pointer(rn)
+			r.used += n.Size()
+
+			for r.used > r.capacity {
+				rn := r.recent.prev
+				if rn == nil {
+					panic("BUG: invalid LRU used or capacity counter")
+				}
+				rn.remove()
+				rn.n.CacheData = nil
+				r.used -= rn.n.Size()
+				evicted = append(evicted, rn)
+			}
+		}
+	} else {
+		rn := (*lruNode)(n.CacheData)
+		if !rn.ban {
+			rn.remove()
+			rn.insert(&r.recent)
+		}
+	}
+	r.mu.Unlock()
+
+	for _, rn := range evicted {
+		rn.h.Release()
+	}
+}
+
+func (r *lru) Ban(n *Node) {
+	r.mu.Lock()
+	if n.CacheData == nil {
+		n.CacheData = unsafe.Pointer(&lruNode{n: n, ban: true})
+	} else {
+		rn := (*lruNode)(n.CacheData)
+		if !rn.ban {
+			rn.remove()
+			rn.ban = true
+			r.used -= rn.n.Size()
+			r.mu.Unlock()
+
+			rn.h.Release()
+			rn.h = nil
+			return
+		}
+	}
+	r.mu.Unlock()
+}
+
+func (r *lru) Evict(n *Node) {
+	r.mu.Lock()
+	rn := (*lruNode)(n.CacheData)
+	if rn == nil || rn.ban {
+		r.mu.Unlock()
+		return
+	}
+	n.CacheData = nil
+	r.mu.Unlock()
+
+	rn.h.Release()
+}
+
+func (r *lru) EvictNS(ns uint64) {
+	var evicted []*lruNode
+
+	r.mu.Lock()
+	for e := r.recent.prev; e != &r.recent; {
+		rn := e
+		e = e.prev
+		if rn.n.NS() == ns {
+			rn.remove()
+			rn.n.CacheData = nil
+			r.used -= rn.n.Size()
+			evicted = append(evicted, rn)
+		}
+	}
+	r.mu.Unlock()
+
+	for _, rn := range evicted {
+		rn.h.Release()
+	}
+}
+
+func (r *lru) EvictAll() {
+	r.mu.Lock()
+	back := r.recent.prev
+	for rn := back; rn != &r.recent; rn = rn.prev {
+		rn.n.CacheData = nil
+	}
+	r.reset()
+	r.mu.Unlock()
+
+	for rn := back; rn != &r.recent; rn = rn.prev {
+		rn.h.Release()
+	}
+}
+
+func (r *lru) Close() error {
+	return nil
+}
+
+// NewLRU create a new LRU-cache.
+func NewLRU(capacity int) Cacher {
+	r := &lru{capacity: capacity}
+	r.reset()
+	return r
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru_cache.go

@@ -1,622 +0,0 @@
-// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package cache
-
-import (
-	"sync"
-	"sync/atomic"
-
-	"github.com/syndtr/goleveldb/leveldb/util"
-)
-
-// The LLRB implementation were taken from https://github.com/petar/GoLLRB.
-// Which contains the following header:
-//
-// Copyright 2010 Petar Maymounkov. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// lruCache represent a LRU cache state.
-type lruCache struct {
-	mu                sync.Mutex
-	recent            lruNode
-	table             map[uint64]*lruNs
-	capacity          int
-	used, size, alive int
-}
-
-// NewLRUCache creates a new initialized LRU cache with the given capacity.
-func NewLRUCache(capacity int) Cache {
-	c := &lruCache{
-		table:    make(map[uint64]*lruNs),
-		capacity: capacity,
-	}
-	c.recent.rNext = &c.recent
-	c.recent.rPrev = &c.recent
-	return c
-}
-
-func (c *lruCache) Capacity() int {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	return c.capacity
-}
-
-func (c *lruCache) Used() int {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	return c.used
-}
-
-func (c *lruCache) Size() int {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	return c.size
-}
-
-func (c *lruCache) NumObjects() int {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-	return c.alive
-}
-
-// SetCapacity set cache capacity.
-func (c *lruCache) SetCapacity(capacity int) {
-	c.mu.Lock()
-	c.capacity = capacity
-	c.evict()
-	c.mu.Unlock()
-}
-
-// GetNamespace return namespace object for given id.
-func (c *lruCache) GetNamespace(id uint64) Namespace {
-	c.mu.Lock()
-	defer c.mu.Unlock()
-
-	if ns, ok := c.table[id]; ok {
-		return ns
-	}
-
-	ns := &lruNs{lru: c, id: id}
-	c.table[id] = ns
-	return ns
-}
-
-func (c *lruCache) ZapNamespace(id uint64) {
-	c.mu.Lock()
-	if ns, exist := c.table[id]; exist {
-		ns.zapNB()
-		delete(c.table, id)
-	}
-	c.mu.Unlock()
-}
-
-func (c *lruCache) PurgeNamespace(id uint64, fin PurgeFin) {
-	c.mu.Lock()
-	if ns, exist := c.table[id]; exist {
-		ns.purgeNB(fin)
-	}
-	c.mu.Unlock()
-}
-
-// Purge purge entire cache.
-func (c *lruCache) Purge(fin PurgeFin) {
-	c.mu.Lock()
-	for _, ns := range c.table {
-		ns.purgeNB(fin)
-	}
-	c.mu.Unlock()
-}
-
-func (c *lruCache) Zap() {
-	c.mu.Lock()
-	for _, ns := range c.table {
-		ns.zapNB()
-	}
-	c.table = make(map[uint64]*lruNs)
-	c.mu.Unlock()
-}
-
-func (c *lruCache) evict() {
-	top := &c.recent
-	for n := c.recent.rPrev; c.used > c.capacity && n != top; {
-		if n.state != nodeEffective {
-			panic("evicting non effective node")
-		}
-		n.state = nodeEvicted
-		n.rRemove()
-		n.derefNB()
-		c.used -= n.charge
-		n = c.recent.rPrev
-	}
-}
-
-type lruNs struct {
-	lru    *lruCache
-	id     uint64
-	rbRoot *lruNode
-	state  nsState
-}
-
-func (ns *lruNs) rbGetOrCreateNode(h *lruNode, key uint64) (hn, n *lruNode) {
-	if h == nil {
-		n = &lruNode{ns: ns, key: key}
-		return n, n
-	}
-
-	if key < h.key {
-		hn, n = ns.rbGetOrCreateNode(h.rbLeft, key)
-		if hn != nil {
-			h.rbLeft = hn
-		} else {
-			return nil, n
-		}
-	} else if key > h.key {
-		hn, n = ns.rbGetOrCreateNode(h.rbRight, key)
-		if hn != nil {
-			h.rbRight = hn
-		} else {
-			return nil, n
-		}
-	} else {
-		return nil, h
-	}
-
-	if rbIsRed(h.rbRight) && !rbIsRed(h.rbLeft) {
-		h = rbRotLeft(h)
-	}
-	if rbIsRed(h.rbLeft) && rbIsRed(h.rbLeft.rbLeft) {
-		h = rbRotRight(h)
-	}
-	if rbIsRed(h.rbLeft) && rbIsRed(h.rbRight) {
-		rbFlip(h)
-	}
-	return h, n
-}
-
-func (ns *lruNs) getOrCreateNode(key uint64) *lruNode {
-	hn, n := ns.rbGetOrCreateNode(ns.rbRoot, key)
-	if hn != nil {
-		ns.rbRoot = hn
-		ns.rbRoot.rbBlack = true
-	}
-	return n
-}
-
-func (ns *lruNs) rbGetNode(key uint64) *lruNode {
-	h := ns.rbRoot
-	for h != nil {
-		switch {
-		case key < h.key:
-			h = h.rbLeft
-		case key > h.key:
-			h = h.rbRight
-		default:
-			return h
-		}
-	}
-	return nil
-}
-
-func (ns *lruNs) getNode(key uint64) *lruNode {
-	return ns.rbGetNode(key)
-}
-
-func (ns *lruNs) rbDeleteNode(h *lruNode, key uint64) *lruNode {
-	if h == nil {
-		return nil
-	}
-
-	if key < h.key {
-		if h.rbLeft == nil { // key not present. Nothing to delete
-			return h
-		}
-		if !rbIsRed(h.rbLeft) && !rbIsRed(h.rbLeft.rbLeft) {
-			h = rbMoveLeft(h)
-		}
-		h.rbLeft = ns.rbDeleteNode(h.rbLeft, key)
-	} else {
-		if rbIsRed(h.rbLeft) {
-			h = rbRotRight(h)
-		}
-		// If @key equals @h.key and no right children at @h
-		if h.key == key && h.rbRight == nil {
-			return nil
-		}
-		if h.rbRight != nil && !rbIsRed(h.rbRight) && !rbIsRed(h.rbRight.rbLeft) {
-			h = rbMoveRight(h)
-		}
-		// If @key equals @h.key, and (from above) 'h.Right != nil'
-		if h.key == key {
-			var x *lruNode
-			h.rbRight, x = rbDeleteMin(h.rbRight)
-			if x == nil {
-				panic("logic")
-			}
-			x.rbLeft, h.rbLeft = h.rbLeft, nil
-			x.rbRight, h.rbRight = h.rbRight, nil
-			x.rbBlack = h.rbBlack
-			h = x
-		} else { // Else, @key is bigger than @h.key
-			h.rbRight = ns.rbDeleteNode(h.rbRight, key)
-		}
-	}
-
-	return rbFixup(h)
-}
-
-func (ns *lruNs) deleteNode(key uint64) {
-	ns.rbRoot = ns.rbDeleteNode(ns.rbRoot, key)
-	if ns.rbRoot != nil {
-		ns.rbRoot.rbBlack = true
-	}
-}
-
-func (ns *lruNs) rbIterateNodes(h *lruNode, pivot uint64, iter func(n *lruNode) bool) bool {
-	if h == nil {
-		return true
-	}
-	if h.key >= pivot {
-		if !ns.rbIterateNodes(h.rbLeft, pivot, iter) {
-			return false
-		}
-		if !iter(h) {
-			return false
-		}
-	}
-	return ns.rbIterateNodes(h.rbRight, pivot, iter)
-}
-
-func (ns *lruNs) iterateNodes(iter func(n *lruNode) bool) {
-	ns.rbIterateNodes(ns.rbRoot, 0, iter)
-}
-
-func (ns *lruNs) Get(key uint64, setf SetFunc) Handle {
-	ns.lru.mu.Lock()
-	defer ns.lru.mu.Unlock()
-
-	if ns.state != nsEffective {
-		return nil
-	}
-
-	var n *lruNode
-	if setf == nil {
-		n = ns.getNode(key)
-		if n == nil {
-			return nil
-		}
-	} else {
-		n = ns.getOrCreateNode(key)
-	}
-	switch n.state {
-	case nodeZero:
-		charge, value := setf()
-		if value == nil {
-			ns.deleteNode(key)
-			return nil
-		}
-		if charge < 0 {
-			charge = 0
-		}
-
-		n.value = value
-		n.charge = charge
-		n.state = nodeEvicted
-
-		ns.lru.size += charge
-		ns.lru.alive++
-
-		fallthrough
-	case nodeEvicted:
-		if n.charge == 0 {
-			break
-		}
-
-		// Insert to recent list.
-		n.state = nodeEffective
-		n.ref++
-		ns.lru.used += n.charge
-		ns.lru.evict()
-
-		fallthrough
-	case nodeEffective:
-		// Bump to front.
-		n.rRemove()
-		n.rInsert(&ns.lru.recent)
-	case nodeDeleted:
-		// Do nothing.
-	default:
-		panic("invalid state")
-	}
-	n.ref++
-
-	return &lruHandle{node: n}
-}
-
-func (ns *lruNs) Delete(key uint64, fin DelFin) bool {
-	ns.lru.mu.Lock()
-	defer ns.lru.mu.Unlock()
-
-	if ns.state != nsEffective {
-		if fin != nil {
-			fin(false, false)
-		}
-		return false
-	}
-
-	n := ns.getNode(key)
-	if n == nil {
-		if fin != nil {
-			fin(false, false)
-		}
-		return false
-
-	}
-
-	switch n.state {
-	case nodeEffective:
-		ns.lru.used -= n.charge
-		n.state = nodeDeleted
-		n.delfin = fin
-		n.rRemove()
-		n.derefNB()
-	case nodeEvicted:
-		n.state = nodeDeleted
-		n.delfin = fin
-	case nodeDeleted:
-		if fin != nil {
-			fin(true, true)
-		}
-		return false
-	default:
-		panic("invalid state")
-	}
-
-	return true
-}
-
-func (ns *lruNs) purgeNB(fin PurgeFin) {
-	if ns.state == nsEffective {
-		var nodes []*lruNode
-		ns.iterateNodes(func(n *lruNode) bool {
-			nodes = append(nodes, n)
-			return true
-		})
-		for _, n := range nodes {
-			switch n.state {
-			case nodeEffective:
-				ns.lru.used -= n.charge
-				n.state = nodeDeleted
-				n.purgefin = fin
-				n.rRemove()
-				n.derefNB()
-			case nodeEvicted:
-				n.state = nodeDeleted
-				n.purgefin = fin
-			case nodeDeleted:
-			default:
-				panic("invalid state")
-			}
-		}
-	}
-}
-
-func (ns *lruNs) Purge(fin PurgeFin) {
-	ns.lru.mu.Lock()
-	ns.purgeNB(fin)
-	ns.lru.mu.Unlock()
-}
-
-func (ns *lruNs) zapNB() {
-	if ns.state == nsEffective {
-		ns.state = nsZapped
-
-		ns.iterateNodes(func(n *lruNode) bool {
-			if n.state == nodeEffective {
-				ns.lru.used -= n.charge
-				n.rRemove()
-			}
-			ns.lru.size -= n.charge
-			n.state = nodeDeleted
-			n.fin()
-
-			return true
-		})
-		ns.rbRoot = nil
-	}
-}
-
-func (ns *lruNs) Zap() {
-	ns.lru.mu.Lock()
-	ns.zapNB()
-	delete(ns.lru.table, ns.id)
-	ns.lru.mu.Unlock()
-}
-
-type lruNode struct {
-	ns *lruNs
-
-	rNext, rPrev    *lruNode
-	rbLeft, rbRight *lruNode
-	rbBlack         bool
-
-	key      uint64
-	value    interface{}
-	charge   int
-	ref      int
-	state    nodeState
-	delfin   DelFin
-	purgefin PurgeFin
-}
-
-func (n *lruNode) rInsert(at *lruNode) {
-	x := at.rNext
-	at.rNext = n
-	n.rPrev = at
-	n.rNext = x
-	x.rPrev = n
-}
-
-func (n *lruNode) rRemove() bool {
-	if n.rPrev == nil {
-		return false
-	}
-
-	n.rPrev.rNext = n.rNext
-	n.rNext.rPrev = n.rPrev
-	n.rPrev = nil
-	n.rNext = nil
-
-	return true
-}
-
-func (n *lruNode) fin() {
-	if r, ok := n.value.(util.Releaser); ok {
-		r.Release()
-	}
-	if n.purgefin != nil {
-		if n.delfin != nil {
-			panic("conflicting delete and purge fin")
-		}
-		n.purgefin(n.ns.id, n.key)
-		n.purgefin = nil
-	} else if n.delfin != nil {
-		n.delfin(true, false)
-		n.delfin = nil
-	}
-}
-
-func (n *lruNode) derefNB() {
-	n.ref--
-	if n.ref == 0 {
-		if n.ns.state == nsEffective {
-			// Remove elemement.
-			n.ns.deleteNode(n.key)
-			n.ns.lru.size -= n.charge
-			n.ns.lru.alive--
-			n.fin()
-		}
-		n.value = nil
-	} else if n.ref < 0 {
-		panic("leveldb/cache: lruCache: negative node reference")
-	}
-}
-
-func (n *lruNode) deref() {
-	n.ns.lru.mu.Lock()
-	n.derefNB()
-	n.ns.lru.mu.Unlock()
-}
-
-type lruHandle struct {
-	node *lruNode
-	once uint32
-}
-
-func (h *lruHandle) Value() interface{} {
-	if atomic.LoadUint32(&h.once) == 0 {
-		return h.node.value
-	}
-	return nil
-}
-
-func (h *lruHandle) Release() {
-	if !atomic.CompareAndSwapUint32(&h.once, 0, 1) {
-		return
-	}
-	h.node.deref()
-	h.node = nil
-}
-
-func rbIsRed(h *lruNode) bool {
-	if h == nil {
-		return false
-	}
-	return !h.rbBlack
-}
-
-func rbRotLeft(h *lruNode) *lruNode {
-	x := h.rbRight
-	if x.rbBlack {
-		panic("rotating a black link")
-	}
-	h.rbRight = x.rbLeft
-	x.rbLeft = h
-	x.rbBlack = h.rbBlack
-	h.rbBlack = false
-	return x
-}
-
-func rbRotRight(h *lruNode) *lruNode {
-	x := h.rbLeft
-	if x.rbBlack {
-		panic("rotating a black link")
-	}
-	h.rbLeft = x.rbRight
-	x.rbRight = h
-	x.rbBlack = h.rbBlack
-	h.rbBlack = false
-	return x
-}
-
-func rbFlip(h *lruNode) {
-	h.rbBlack = !h.rbBlack
-	h.rbLeft.rbBlack = !h.rbLeft.rbBlack
-	h.rbRight.rbBlack = !h.rbRight.rbBlack
-}
-
-func rbMoveLeft(h *lruNode) *lruNode {
-	rbFlip(h)
-	if rbIsRed(h.rbRight.rbLeft) {
-		h.rbRight = rbRotRight(h.rbRight)
-		h = rbRotLeft(h)
-		rbFlip(h)
-	}
-	return h
-}
-
-func rbMoveRight(h *lruNode) *lruNode {
-	rbFlip(h)
-	if rbIsRed(h.rbLeft.rbLeft) {
-		h = rbRotRight(h)
-		rbFlip(h)
-	}
-	return h
-}
-
-func rbFixup(h *lruNode) *lruNode {
-	if rbIsRed(h.rbRight) {
-		h = rbRotLeft(h)
-	}
-
-	if rbIsRed(h.rbLeft) && rbIsRed(h.rbLeft.rbLeft) {
-		h = rbRotRight(h)
-	}
-
-	if rbIsRed(h.rbLeft) && rbIsRed(h.rbRight) {
-		rbFlip(h)
-	}
-
-	return h
-}
-
-func rbDeleteMin(h *lruNode) (hn, n *lruNode) {
-	if h == nil {
-		return nil, nil
-	}
-	if h.rbLeft == nil {
-		return nil, h
-	}
-
-	if !rbIsRed(h.rbLeft) && !rbIsRed(h.rbLeft.rbLeft) {
-		h = rbMoveLeft(h)
-	}
-
-	h.rbLeft, n = rbDeleteMin(h.rbLeft)
-
-	return rbFixup(h), n
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/corrupt_test.go

@@ -1,502 +0,0 @@
-// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package leveldb
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"math/rand"
-	"testing"
-
-	"github.com/syndtr/goleveldb/leveldb/cache"
-	"github.com/syndtr/goleveldb/leveldb/filter"
-	"github.com/syndtr/goleveldb/leveldb/opt"
-	"github.com/syndtr/goleveldb/leveldb/storage"
-)
-
-const ctValSize = 1000
-
-type dbCorruptHarness struct {
-	dbHarness
-}
-
-func newDbCorruptHarnessWopt(t *testing.T, o *opt.Options) *dbCorruptHarness {
-	h := new(dbCorruptHarness)
-	h.init(t, o)
-	return h
-}
-
-func newDbCorruptHarness(t *testing.T) *dbCorruptHarness {
-	return newDbCorruptHarnessWopt(t, &opt.Options{
-		BlockCache: cache.NewLRUCache(100),
-		Strict:     opt.StrictJournalChecksum,
-	})
-}
-
-func (h *dbCorruptHarness) recover() {
-	p := &h.dbHarness
-	t := p.t
-
-	var err error
-	p.db, err = Recover(h.stor, h.o)
-	if err != nil {
-		t.Fatal("Repair: got error: ", err)
-	}
-}
-
-func (h *dbCorruptHarness) build(n int) {
-	p := &h.dbHarness
-	t := p.t
-	db := p.db
-
-	batch := new(Batch)
-	for i := 0; i < n; i++ {
-		batch.Reset()
-		batch.Put(tkey(i), tval(i, ctValSize))
-		err := db.Write(batch, p.wo)
-		if err != nil {
-			t.Fatal("write error: ", err)
-		}
-	}
-}
-
-func (h *dbCorruptHarness) buildShuffled(n int, rnd *rand.Rand) {
-	p := &h.dbHarness
-	t := p.t
-	db := p.db
-
-	batch := new(Batch)
-	for i := range rnd.Perm(n) {
-		batch.Reset()
-		batch.Put(tkey(i), tval(i, ctValSize))
-		err := db.Write(batch, p.wo)
-		if err != nil {
-			t.Fatal("write error: ", err)
-		}
-	}
-}
-
-func (h *dbCorruptHarness) deleteRand(n, max int, rnd *rand.Rand) {
-	p := &h.dbHarness
-	t := p.t
-	db := p.db
-
-	batch := new(Batch)
-	for i := 0; i < n; i++ {
-		batch.Reset()
-		batch.Delete(tkey(rnd.Intn(max)))
-		err := db.Write(batch, p.wo)
-		if err != nil {
-			t.Fatal("write error: ", err)
-		}
-	}
-}
-
-func (h *dbCorruptHarness) corrupt(ft storage.FileType, fi, offset, n int) {
-	p := &h.dbHarness
-	t := p.t
-
-	ff, _ := p.stor.GetFiles(ft)
-	sff := files(ff)
-	sff.sort()
-	if fi < 0 {
-		fi = len(sff) - 1
-	}
-	if fi >= len(sff) {
-		t.Fatalf("no such file with type %q with index %d", ft, fi)
-	}
-
-	file := sff[fi]
-
-	r, err := file.Open()
-	if err != nil {
-		t.Fatal("cannot open file: ", err)
-	}
-	x, err := r.Seek(0, 2)
-	if err != nil {
-		t.Fatal("cannot query file size: ", err)
-	}
-	m := int(x)
-	if _, err := r.Seek(0, 0); err != nil {
-		t.Fatal(err)
-	}
-
-	if offset < 0 {
-		if -offset > m {
-			offset = 0
-		} else {
-			offset = m + offset
-		}
-	}
-	if offset > m {
-		offset = m
-	}
-	if offset+n > m {
-		n = m - offset
-	}
-
-	buf := make([]byte, m)
-	_, err = io.ReadFull(r, buf)
-	if err != nil {
-		t.Fatal("cannot read file: ", err)
-	}
-	r.Close()
-
-	for i := 0; i < n; i++ {
-		buf[offset+i] ^= 0x80
-	}
-
-	err = file.Remove()
-	if err != nil {
-		t.Fatal("cannot remove old file: ", err)
-	}
-	w, err := file.Create()
-	if err != nil {
-		t.Fatal("cannot create new file: ", err)
-	}
-	_, err = w.Write(buf)
-	if err != nil {
-		t.Fatal("cannot write new file: ", err)
-	}
-	w.Close()
-}
-
-func (h *dbCorruptHarness) removeAll(ft storage.FileType) {
-	ff, err := h.stor.GetFiles(ft)
-	if err != nil {
-		h.t.Fatal("get files: ", err)
-	}
-	for _, f := range ff {
-		if err := f.Remove(); err != nil {
-			h.t.Error("remove file: ", err)
-		}
-	}
-}
-
-func (h *dbCorruptHarness) removeOne(ft storage.FileType) {
-	ff, err := h.stor.GetFiles(ft)
-	if err != nil {
-		h.t.Fatal("get files: ", err)
-	}
-	f := ff[rand.Intn(len(ff))]
-	h.t.Logf("removing file @%d", f.Num())
-	if err := f.Remove(); err != nil {
-		h.t.Error("remove file: ", err)
-	}
-}
-
-func (h *dbCorruptHarness) check(min, max int) {
-	p := &h.dbHarness
-	t := p.t
-	db := p.db
-
-	var n, badk, badv, missed, good int
-	iter := db.NewIterator(nil, p.ro)
-	for iter.Next() {
-		k := 0
-		fmt.Sscanf(string(iter.Key()), "%d", &k)
-		if k < n {
-			badk++
-			continue
-		}
-		missed += k - n
-		n = k + 1
-		if !bytes.Equal(iter.Value(), tval(k, ctValSize)) {
-			badv++
-		} else {
-			good++
-		}
-	}
-	err := iter.Error()
-	iter.Release()
-	t.Logf("want=%d..%d got=%d badkeys=%d badvalues=%d missed=%d, err=%v",
-		min, max, good, badk, badv, missed, err)
-	if good < min || good > max {
-		t.Errorf("good entries number not in range")
-	}
-}
-
-func TestCorruptDB_Journal(t *testing.T) {
-	h := newDbCorruptHarness(t)
-
-	h.build(100)
-	h.check(100, 100)
-	h.closeDB()
-	h.corrupt(storage.TypeJournal, -1, 19, 1)
-	h.corrupt(storage.TypeJournal, -1, 32*1024+1000, 1)
-
-	h.openDB()
-	h.check(36, 36)
-
-	h.close()
-}
-
-func TestCorruptDB_Table(t *testing.T) {
-	h := newDbCorruptHarness(t)
-
-	h.build(100)
-	h.compactMem()
-	h.compactRangeAt(0, "", "")
-	h.compactRangeAt(1, "", "")
-	h.closeDB()
-	h.corrupt(storage.TypeTable, -1, 100, 1)
-
-	h.openDB()
-	h.check(99, 99)
-
-	h.close()
-}
-
-func TestCorruptDB_TableIndex(t *testing.T) {
-	h := newDbCorruptHarness(t)
-
-	h.build(10000)
-	h.compactMem()
-	h.closeDB()
-	h.corrupt(storage.TypeTable, -1, -2000, 500)
-
-	h.openDB()
-	h.check(5000, 9999)
-
-	h.close()
-}
-
-func TestCorruptDB_MissingManifest(t *testing.T) {
-	rnd := rand.New(rand.NewSource(0x0badda7a))
-	h := newDbCorruptHarnessWopt(t, &opt.Options{
-		BlockCache:  cache.NewLRUCache(100),
-		Strict:      opt.StrictJournalChecksum,
-		WriteBuffer: 1000 * 60,
-	})
-
-	h.build(1000)
-	h.compactMem()
-	h.buildShuffled(1000, rnd)
-	h.compactMem()
-	h.deleteRand(500, 1000, rnd)
-	h.compactMem()
-	h.buildShuffled(1000, rnd)
-	h.compactMem()
-	h.deleteRand(500, 1000, rnd)
-	h.compactMem()
-	h.buildShuffled(1000, rnd)
-	h.compactMem()
-	h.closeDB()
-
-	h.stor.SetIgnoreOpenErr(storage.TypeManifest)
-	h.removeAll(storage.TypeManifest)
-	h.openAssert(false)
-	h.stor.SetIgnoreOpenErr(0)
-
-	h.recover()
-	h.check(1000, 1000)
-	h.build(1000)
-	h.compactMem()
-	h.compactRange("", "")
-	h.closeDB()
-
-	h.recover()
-	h.check(1000, 1000)
-
-	h.close()
-}
-
-func TestCorruptDB_SequenceNumberRecovery(t *testing.T) {
-	h := newDbCorruptHarness(t)
-
-	h.put("foo", "v1")
-	h.put("foo", "v2")
-	h.put("foo", "v3")
-	h.put("foo", "v4")
-	h.put("foo", "v5")
-	h.closeDB()
-
-	h.recover()
-	h.getVal("foo", "v5")
-	h.put("foo", "v6")
-	h.getVal("foo", "v6")
-
-	h.reopenDB()
-	h.getVal("foo", "v6")
-
-	h.close()
-}
-
-func TestCorruptDB_SequenceNumberRecoveryTable(t *testing.T) {
-	h := newDbCorruptHarness(t)
-
-	h.put("foo", "v1")
-	h.put("foo", "v2")
-	h.put("foo", "v3")
-	h.compactMem()
-	h.put("foo", "v4")
-	h.put("foo", "v5")
-	h.compactMem()
-	h.closeDB()
-
-	h.recover()
-	h.getVal("foo", "v5")
-	h.put("foo", "v6")
-	h.getVal("foo", "v6")
-
-	h.reopenDB()
-	h.getVal("foo", "v6")
-
-	h.close()
-}
-
-func TestCorruptDB_CorruptedManifest(t *testing.T) {
-	h := newDbCorruptHarness(t)
-
-	h.put("foo", "hello")
-	h.compactMem()
-	h.compactRange("", "")
-	h.closeDB()
-	h.corrupt(storage.TypeManifest, -1, 0, 1000)
-	h.openAssert(false)
-
-	h.recover()
-	h.getVal("foo", "hello")
-
-	h.close()
-}
-
-func TestCorruptDB_CompactionInputError(t *testing.T) {
-	h := newDbCorruptHarness(t)
-
-	h.build(10)
-	h.compactMem()
-	h.closeDB()
-	h.corrupt(storage.TypeTable, -1, 100, 1)
-
-	h.openDB()
-	h.check(9, 9)
-
-	h.build(10000)
-	h.check(10000, 10000)
-
-	h.close()
-}
-
-func TestCorruptDB_UnrelatedKeys(t *testing.T) {
-	h := newDbCorruptHarness(t)
-
-	h.build(10)
-	h.compactMem()
-	h.closeDB()
-	h.corrupt(storage.TypeTable, -1, 100, 1)
-
-	h.openDB()
-	h.put(string(tkey(1000)), string(tval(1000, ctValSize)))
-	h.getVal(string(tkey(1000)), string(tval(1000, ctValSize)))
-	h.compactMem()
-	h.getVal(string(tkey(1000)), string(tval(1000, ctValSize)))
-
-	h.close()
-}
-
-func TestCorruptDB_Level0NewerFileHasOlderSeqnum(t *testing.T) {
-	h := newDbCorruptHarness(t)
-
-	h.put("a", "v1")
-	h.put("b", "v1")
-	h.compactMem()
-	h.put("a", "v2")
-	h.put("b", "v2")
-	h.compactMem()
-	h.put("a", "v3")
-	h.put("b", "v3")
-	h.compactMem()
-	h.put("c", "v0")
-	h.put("d", "v0")
-	h.compactMem()
-	h.compactRangeAt(1, "", "")
-	h.closeDB()
-
-	h.recover()
-	h.getVal("a", "v3")
-	h.getVal("b", "v3")
-	h.getVal("c", "v0")
-	h.getVal("d", "v0")
-
-	h.close()
-}
-
-func TestCorruptDB_RecoverInvalidSeq_Issue53(t *testing.T) {
-	h := newDbCorruptHarness(t)
-
-	h.put("a", "v1")
-	h.put("b", "v1")
-	h.compactMem()
-	h.put("a", "v2")
-	h.put("b", "v2")
-	h.compactMem()
-	h.put("a", "v3")
-	h.put("b", "v3")
-	h.compactMem()
-	h.put("c", "v0")
-	h.put("d", "v0")
-	h.compactMem()
-	h.compactRangeAt(0, "", "")
-	h.closeDB()
-
-	h.recover()
-	h.getVal("a", "v3")
-	h.getVal("b", "v3")
-	h.getVal("c", "v0")
-	h.getVal("d", "v0")
-
-	h.close()
-}
-
-func TestCorruptDB_MissingTableFiles(t *testing.T) {
-	h := newDbCorruptHarness(t)
-
-	h.put("a", "v1")
-	h.put("b", "v1")
-	h.compactMem()
-	h.put("c", "v2")
-	h.put("d", "v2")
-	h.compactMem()
-	h.put("e", "v3")
-	h.put("f", "v3")
-	h.closeDB()
-
-	h.removeOne(storage.TypeTable)
-	h.openAssert(false)
-
-	h.close()
-}
-
-func TestCorruptDB_RecoverTable(t *testing.T) {
-	h := newDbCorruptHarnessWopt(t, &opt.Options{
-		WriteBuffer:         112 * opt.KiB,
-		CompactionTableSize: 90 * opt.KiB,
-		Filter:              filter.NewBloomFilter(10),
-	})
-
-	h.build(1000)
-	h.compactMem()
-	h.compactRangeAt(0, "", "")
-	h.compactRangeAt(1, "", "")
-	seq := h.db.seq
-	h.closeDB()
-	h.corrupt(storage.TypeTable, 0, 1000, 1)
-	h.corrupt(storage.TypeTable, 3, 10000, 1)
-	// Corrupted filter shouldn't affect recovery.
-	h.corrupt(storage.TypeTable, 3, 113888, 10)
-	h.corrupt(storage.TypeTable, -1, 20000, 1)
-
-	h.recover()
-	if h.db.seq != seq {
-		t.Errorf("invalid seq, want=%d got=%d", seq, h.db.seq)
-	}
-	h.check(985, 985)
-
-	h.close()
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go

@@ -63,13 +63,14 @@ type DB struct {
 	journalAckC  chan error
 
 	// Compaction.
-	tcompCmdC   chan cCmd
-	tcompPauseC chan chan<- struct{}
-	mcompCmdC   chan cCmd
-	compErrC    chan error
-	compPerErrC chan error
-	compErrSetC chan error
-	compStats   []cStats
+	tcompCmdC        chan cCmd
+	tcompPauseC      chan chan<- struct{}
+	mcompCmdC        chan cCmd
+	compErrC         chan error
+	compPerErrC      chan error
+	compErrSetC      chan error
+	compWriteLocking bool
+	compStats        []cStats
 
 	// Close.
 	closeW sync.WaitGroup
@@ -108,28 +109,44 @@ func openDB(s *session) (*DB, error) {
 		closeC: make(chan struct{}),
 	}
 
-	if err := db.recoverJournal(); err != nil {
-		return nil, err
-	}
+	// Read-only mode.
+	readOnly := s.o.GetReadOnly()
 
-	// Remove any obsolete files.
-	if err := db.checkAndCleanFiles(); err != nil {
-		// Close journal.
-		if db.journal != nil {
-			db.journal.Close()
-			db.journalWriter.Close()
+	if readOnly {
+		// Recover journals (read-only mode).
+		if err := db.recoverJournalRO(); err != nil {
+			return nil, err
 		}
-		return nil, err
+	} else {
+		// Recover journals.
+		if err := db.recoverJournal(); err != nil {
+			return nil, err
+		}
+
+		// Remove any obsolete files.
+		if err := db.checkAndCleanFiles(); err != nil {
+			// Close journal.
+			if db.journal != nil {
+				db.journal.Close()
+				db.journalWriter.Close()
+			}
+			return nil, err
+		}
+
 	}
 
 	// Doesn't need to be included in the wait group.
 	go db.compactionError()
 	go db.mpoolDrain()
 
-	db.closeW.Add(3)
-	go db.tCompaction()
-	go db.mCompaction()
-	go db.jWriter()
+	if readOnly {
+		db.SetReadOnly()
+	} else {
+		db.closeW.Add(3)
+		go db.tCompaction()
+		go db.mCompaction()
+		go db.jWriter()
+	}
 
 	s.logf("db@open done T·%v", time.Since(start))
 
@@ -274,8 +291,9 @@ func recoverTable(s *session, o *opt.Options) error {
 
 		// We will drop corrupted table.
 		strict = o.GetStrict(opt.StrictRecovery)
+		noSync = o.GetNoSync()
 
-		rec   = &sessionRecord{numLevel: o.GetNumLevel()}
+		rec   = &sessionRecord{}
 		bpool = util.NewBufferPool(o.GetBlockSize() + 5)
 	)
 	buildTable := func(iter iterator.Iterator) (tmp storage.File, size int64, err error) {
@@ -311,9 +329,11 @@ func recoverTable(s *session, o *opt.Options) error {
 		if err != nil {
 			return
 		}
-		err = writer.Sync()
-		if err != nil {
-			return
+		if !noSync {
+			err = writer.Sync()
+			if err != nil {
+				return
+			}
 		}
 		size = int64(tw.BytesLen())
 		return
@@ -347,12 +367,14 @@ func recoverTable(s *session, o *opt.Options) error {
 			return err
 		}
 		iter := tr.NewIterator(nil, nil)
-		iter.(iterator.ErrorCallbackSetter).SetErrorCallback(func(err error) {
-			if errors.IsCorrupted(err) {
-				s.logf("table@recovery block corruption @%d %q", file.Num(), err)
-				tcorruptedBlock++
-			}
-		})
+		if itererr, ok := iter.(iterator.ErrorCallbackSetter); ok {
+			itererr.SetErrorCallback(func(err error) {
+				if errors.IsCorrupted(err) {
+					s.logf("table@recovery block corruption @%d %q", file.Num(), err)
+					tcorruptedBlock++
+				}
+			})
+		}
 
 		// Scan the table.
 		for iter.Next() {
@@ -448,132 +470,136 @@ func recoverTable(s *session, o *opt.Options) error {
 }
 
 func (db *DB) recoverJournal() error {
-	// Get all tables and sort it by file number.
-	journalFiles_, err := db.s.getFiles(storage.TypeJournal)
+	// Get all journals and sort it by file number.
+	allJournalFiles, err := db.s.getFiles(storage.TypeJournal)
 	if err != nil {
 		return err
 	}
-	journalFiles := files(journalFiles_)
-	journalFiles.sort()
+	files(allJournalFiles).sort()
 
-	// Discard older journal.
-	prev := -1
-	for i, file := range journalFiles {
-		if file.Num() >= db.s.stJournalNum {
-			if prev >= 0 {
-				i--
-				journalFiles[i] = journalFiles[prev]
-			}
-			journalFiles = journalFiles[i:]
-			break
-		} else if file.Num() == db.s.stPrevJournalNum {
-			prev = i
+	// Journals that will be recovered.
+	var recJournalFiles []storage.File
+	for _, jf := range allJournalFiles {
+		if jf.Num() >= db.s.stJournalNum || jf.Num() == db.s.stPrevJournalNum {
+			recJournalFiles = append(recJournalFiles, jf)
 		}
 	}
 
-	var jr *journal.Reader
-	var of storage.File
-	var mem *memdb.DB
-	batch := new(Batch)
-	cm := newCMem(db.s)
-	buf := new(util.Buffer)
-	// Options.
-	strict := db.s.o.GetStrict(opt.StrictJournal)
-	checksum := db.s.o.GetStrict(opt.StrictJournalChecksum)
-	writeBuffer := db.s.o.GetWriteBuffer()
-	recoverJournal := func(file storage.File) error {
-		db.logf("journal@recovery recovering @%d", file.Num())
-		reader, err := file.Open()
-		if err != nil {
-			return err
-		}
-		defer reader.Close()
+	var (
+		of  storage.File // Obsolete file.
+		rec = &sessionRecord{}
+	)
 
-		// Create/reset journal reader instance.
-		if jr == nil {
-			jr = journal.NewReader(reader, dropper{db.s, file}, strict, checksum)
-		} else {
-			jr.Reset(reader, dropper{db.s, file}, strict, checksum)
-		}
-
-		// Flush memdb and remove obsolete journal file.
-		if of != nil {
-			if mem.Len() > 0 {
-				if err := cm.flush(mem, 0); err != nil {
-					return err
-				}
-			}
-			if err := cm.commit(file.Num(), db.seq); err != nil {
-				return err
-			}
-			cm.reset()
-			of.Remove()
-			of = nil
-		}
-
-		// Replay journal to memdb.
-		mem.Reset()
-		for {
-			r, err := jr.Next()
-			if err != nil {
-				if err == io.EOF {
-					break
-				}
-				return errors.SetFile(err, file)
-			}
-
-			buf.Reset()
-			if _, err := buf.ReadFrom(r); err != nil {
-				if err == io.ErrUnexpectedEOF {
-					// This is error returned due to corruption, with strict == false.
-					continue
-				} else {
-					return errors.SetFile(err, file)
-				}
-			}
-			if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mem); err != nil {
-				if strict || !errors.IsCorrupted(err) {
-					return errors.SetFile(err, file)
-				} else {
-					db.s.logf("journal error: %v (skipped)", err)
-					// We won't apply sequence number as it might be corrupted.
-					continue
-				}
-			}
-
-			// Save sequence number.
-			db.seq = batch.seq + uint64(batch.Len())
-
-			// Flush it if large enough.
-			if mem.Size() >= writeBuffer {
-				if err := cm.flush(mem, 0); err != nil {
-					return err
-				}
-				mem.Reset()
-			}
-		}
-
-		of = file
-		return nil
-	}
-
-	// Recover all journals.
-	if len(journalFiles) > 0 {
-		db.logf("journal@recovery F·%d", len(journalFiles))
+	// Recover journals.
+	if len(recJournalFiles) > 0 {
+		db.logf("journal@recovery F·%d", len(recJournalFiles))
 
 		// Mark file number as used.
-		db.s.markFileNum(journalFiles[len(journalFiles)-1].Num())
+		db.s.markFileNum(recJournalFiles[len(recJournalFiles)-1].Num())
 
-		mem = memdb.New(db.s.icmp, writeBuffer)
-		for _, file := range journalFiles {
-			if err := recoverJournal(file); err != nil {
+		var (
+			// Options.
+			strict      = db.s.o.GetStrict(opt.StrictJournal)
+			checksum    = db.s.o.GetStrict(opt.StrictJournalChecksum)
+			writeBuffer = db.s.o.GetWriteBuffer()
+
+			jr    *journal.Reader
+			mdb   = memdb.New(db.s.icmp, writeBuffer)
+			buf   = &util.Buffer{}
+			batch = &Batch{}
+		)
+
+		for _, jf := range recJournalFiles {
+			db.logf("journal@recovery recovering @%d", jf.Num())
+
+			fr, err := jf.Open()
+			if err != nil {
 				return err
 			}
+
+			// Create or reset journal reader instance.
+			if jr == nil {
+				jr = journal.NewReader(fr, dropper{db.s, jf}, strict, checksum)
+			} else {
+				jr.Reset(fr, dropper{db.s, jf}, strict, checksum)
+			}
+
+			// Flush memdb and remove obsolete journal file.
+			if of != nil {
+				if mdb.Len() > 0 {
+					if _, err := db.s.flushMemdb(rec, mdb, -1); err != nil {
+						fr.Close()
+						return err
+					}
+				}
+
+				rec.setJournalNum(jf.Num())
+				rec.setSeqNum(db.seq)
+				if err := db.s.commit(rec); err != nil {
+					fr.Close()
+					return err
+				}
+				rec.resetAddedTables()
+
+				of.Remove()
+				of = nil
+			}
+
+			// Replay journal to memdb.
+			mdb.Reset()
+			for {
+				r, err := jr.Next()
+				if err != nil {
+					if err == io.EOF {
+						break
+					}
+
+					fr.Close()
+					return errors.SetFile(err, jf)
+				}
+
+				buf.Reset()
+				if _, err := buf.ReadFrom(r); err != nil {
+					if err == io.ErrUnexpectedEOF {
+						// This is error returned due to corruption, with strict == false.
+						continue
+					}
+
+					fr.Close()
+					return errors.SetFile(err, jf)
+				}
+				if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mdb); err != nil {
+					if !strict && errors.IsCorrupted(err) {
+						db.s.logf("journal error: %v (skipped)", err)
+						// We won't apply sequence number as it might be corrupted.
+						continue
+					}
+
+					fr.Close()
+					return errors.SetFile(err, jf)
+				}
+
+				// Save sequence number.
+				db.seq = batch.seq + uint64(batch.Len())
+
+				// Flush it if large enough.
+				if mdb.Size() >= writeBuffer {
+					if _, err := db.s.flushMemdb(rec, mdb, 0); err != nil {
+						fr.Close()
+						return err
+					}
+
+					mdb.Reset()
+				}
+			}
+
+			fr.Close()
+			of = jf
 		}
 
-		// Flush the last journal.
-		if mem.Len() > 0 {
-			if err := cm.flush(mem, 0); err != nil {
+		// Flush the last memdb.
+		if mdb.Len() > 0 {
+			if _, err := db.s.flushMemdb(rec, mdb, 0); err != nil {
 				return err
 			}
 		}
@@ -585,8 +611,10 @@ func (db *DB) recoverJournal() error {
 	}
 
 	// Commit.
-	if err := cm.commit(db.journalFile.Num(), db.seq); err != nil {
-		// Close journal.
+	rec.setJournalNum(db.journalFile.Num())
+	rec.setSeqNum(db.seq)
+	if err := db.s.commit(rec); err != nil {
+		// Close journal on error.
 		if db.journal != nil {
 			db.journal.Close()
 			db.journalWriter.Close()
@@ -602,6 +630,103 @@ func (db *DB) recoverJournal() error {
 	return nil
 }
 
+func (db *DB) recoverJournalRO() error {
+	// Get all journals and sort it by file number.
+	allJournalFiles, err := db.s.getFiles(storage.TypeJournal)
+	if err != nil {
+		return err
+	}
+	files(allJournalFiles).sort()
+
+	// Journals that will be recovered.
+	var recJournalFiles []storage.File
+	for _, jf := range allJournalFiles {
+		if jf.Num() >= db.s.stJournalNum || jf.Num() == db.s.stPrevJournalNum {
+			recJournalFiles = append(recJournalFiles, jf)
+		}
+	}
+
+	var (
+		// Options.
+		strict      = db.s.o.GetStrict(opt.StrictJournal)
+		checksum    = db.s.o.GetStrict(opt.StrictJournalChecksum)
+		writeBuffer = db.s.o.GetWriteBuffer()
+
+		mdb = memdb.New(db.s.icmp, writeBuffer)
+	)
+
+	// Recover journals.
+	if len(recJournalFiles) > 0 {
+		db.logf("journal@recovery RO·Mode F·%d", len(recJournalFiles))
+
+		var (
+			jr    *journal.Reader
+			buf   = &util.Buffer{}
+			batch = &Batch{}
+		)
+
+		for _, jf := range recJournalFiles {
+			db.logf("journal@recovery recovering @%d", jf.Num())
+
+			fr, err := jf.Open()
+			if err != nil {
+				return err
+			}
+
+			// Create or reset journal reader instance.
+			if jr == nil {
+				jr = journal.NewReader(fr, dropper{db.s, jf}, strict, checksum)
+			} else {
+				jr.Reset(fr, dropper{db.s, jf}, strict, checksum)
+			}
+
+			// Replay journal to memdb.
+			for {
+				r, err := jr.Next()
+				if err != nil {
+					if err == io.EOF {
+						break
+					}
+
+					fr.Close()
+					return errors.SetFile(err, jf)
+				}
+
+				buf.Reset()
+				if _, err := buf.ReadFrom(r); err != nil {
+					if err == io.ErrUnexpectedEOF {
+						// This is error returned due to corruption, with strict == false.
+						continue
+					}
+
+					fr.Close()
+					return errors.SetFile(err, jf)
+				}
+				if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mdb); err != nil {
+					if !strict && errors.IsCorrupted(err) {
+						db.s.logf("journal error: %v (skipped)", err)
+						// We won't apply sequence number as it might be corrupted.
+						continue
+					}
+
+					fr.Close()
+					return errors.SetFile(err, jf)
+				}
+
+				// Save sequence number.
+				db.seq = batch.seq + uint64(batch.Len())
+			}
+
+			fr.Close()
+		}
+	}
+
+	// Set memDB.
+	db.mem = &memDB{db: db, DB: mdb, ref: 1}
+
+	return nil
+}
+
 func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, err error) {
 	ikey := newIkey(key, seq, ktSeek)
 
@@ -612,7 +737,7 @@ func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, er
 		}
 		defer m.decref()
 
-		mk, mv, me := m.mdb.Find(ikey)
+		mk, mv, me := m.Find(ikey)
 		if me == nil {
 			ukey, _, kt, kerr := parseIkey(mk)
 			if kerr != nil {
@@ -650,7 +775,7 @@ func (db *DB) has(key []byte, seq uint64, ro *opt.ReadOptions) (ret bool, err er
 		}
 		defer m.decref()
 
-		mk, _, me := m.mdb.Find(ikey)
+		mk, _, me := m.Find(ikey)
 		if me == nil {
 			ukey, _, kt, kerr := parseIkey(mk)
 			if kerr != nil {
@@ -782,7 +907,7 @@ func (db *DB) GetProperty(name string) (value string, err error) {
 
 	const prefix = "leveldb."
 	if !strings.HasPrefix(name, prefix) {
-		return "", errors.New("leveldb: GetProperty: unknown property: " + name)
+		return "", ErrNotFound
 	}
 	p := name[len(prefix):]
 
@@ -796,7 +921,7 @@ func (db *DB) GetProperty(name string) (value string, err error) {
 		var rest string
 		n, _ := fmt.Sscanf(p[len(numFilesPrefix):], "%d%s", &level, &rest)
 		if n != 1 || int(level) >= db.s.o.GetNumLevel() {
-			err = errors.New("leveldb: GetProperty: invalid property: " + name)
+			err = ErrNotFound
 		} else {
 			value = fmt.Sprint(v.tLen(int(level)))
 		}
@@ -823,8 +948,8 @@ func (db *DB) GetProperty(name string) (value string, err error) {
 	case p == "blockpool":
 		value = fmt.Sprintf("%v", db.s.tops.bpool)
 	case p == "cachedblock":
-		if bc := db.s.o.GetBlockCache(); bc != nil {
-			value = fmt.Sprintf("%d", bc.Size())
+		if db.s.tops.bcache != nil {
+			value = fmt.Sprintf("%d", db.s.tops.bcache.Size())
 		} else {
 			value = "<nil>"
 		}
@@ -835,7 +960,7 @@ func (db *DB) GetProperty(name string) (value string, err error) {
 	case p == "aliveiters":
 		value = fmt.Sprintf("%d", atomic.LoadInt32(&db.aliveIters))
 	default:
-		err = errors.New("leveldb: GetProperty: unknown property: " + name)
+		err = ErrNotFound
 	}
 
 	return
@@ -898,6 +1023,9 @@ func (db *DB) Close() error {
 	var err error
 	select {
 	case err = <-db.compErrC:
+		if err == ErrReadOnly {
+			err = nil
+		}
 	default:
 	}