Translation update

UPnP API Additions to address outstanding parts of #432

.gitignore

@@ -4,7 +4,9 @@ syncthing.exe
 *.zip
 *.asc
 *.sublime*
-discosrv
 .jshintrc
 coverage.out
 files/pidx
+bin
+perfstats*.csv
+coverage.xml

.travis.yml

@@ -1,20 +0,0 @@
-language: go
-
-go:
-   - tip
-
-install:
-   - export PATH=$PATH:$HOME/gopath/bin
-   - ./build.sh setup
-   - go get code.google.com/p/go.tools/cmd/cover
-   - go get github.com/mattn/goveralls
-
-script:
-   - ./build.sh test-cov
-
-after_success:
-   - goveralls -coverprofile=coverage.out -service=travis-ci -package=calmh/syncthing -repotoken="$COVERALS_TOKEN"
-
-env:
-   global:
-      secure: "zEV2h2XtKHNLVdXJjM4LA/VjMfLVydm6goF+ARit+nOSGxGoH7f7jIdzJzhxgh7shKG93q61eLO1Tug+WBMYB2EpBuYnTB5AIMYhCDwNI8C4uBV6c3brHfcrie7MASNao8TID2QScASKNFFWvjv/i1Ccn5ztxdcQuhSsNjGZp8A="

CONTRIBUTING.md

@@ -1,14 +1,62 @@
+## Reporting Bugs
+
+Please file bugs in the [Github Issue
+Tracker](https://github.com/syncthing/syncthing/issues). Include at
+least the following:
+
+ - What happened
+
+ - What did you expect to happen instead of what *did* happen, if it's
+   not crazy obvious
+
+ - What operating system, operating system version and version of
+   Syncthing you are running
+
+ - The same for other connected devices, where relevant
+
+ - Screenshot if the issue concerns something visible in the GUI
+
+ - Console log entries, where possible and relevant
+
+If you're not sure whether something is relevant, erring on the side of
+too much information will never get you yelled at. :)
+
+## Contributing Translations
+
+All translations are done via
+[Transifex](https://www.transifex.com/projects/p/syncthing/). If you
+wish to contribute to a translation, just head over there and sign up.
+Before every release, the language resources are updated from the
+latest info on Transifex.
+
+## Contributing Code
+
 Please do contribute! If you want to contribute but are unsure where to
 start, the [Contributions Needed
-topic](http://discourse.syncthing.net/t/contributions-needed/49)
-lists areas in need of attention.
+topic](http://discourse.syncthing.net/t/49) lists areas in need of
+attention. In general, any open issues are fair game! Be prepared for a
+[certain amount of
+review](https://discourse.syncthing.net/t/733); it's all in the name of
+quality. :)
+
+You may be asked to "rebase" or "squash" your pull request before it's
+accepted. This means to make sure that the pull request:
+
+ - Is based on the latest commit on `master`.
+
+ - Only contains a single commit (exceptions to be made where
+   appropriate).
+
+ - Has a descriptive commit message, ending in `(fixes #123)` if it
+   resolves an existing issue.
 
 ## Licensing
 
-All contributions are made under the same MIT License as the rest of the
-project, except documentation which is licensed under the Creative
-Commons Attribution 4.0 International License. You retain the copyright
-to code you have written.
+All contributions are made under the same GPL license as the rest of the
+project, except documentation, user interface text and translation
+strings which are licensed under the Creative Commons Attribution 4.0
+International License. You retain the copyright to code you have
+written.
 
 When accepting your first contribution, the maintainer of the project
 will ensure that you are added to the CONTRIBUTORS file. You are welcome
@@ -16,8 +64,8 @@ to add yourself as a separate commit in your first pull request.
 
 ## Building
 
-[See the
-documentation](http://discourse.syncthing.net/t/building-syncthing/44)
+[See the documentation](http://discourse.syncthing.net/t/44) on how to
+get started with a build environment.
 
 ## Branches
 
@@ -44,7 +92,9 @@ Yes please!
 
 ## Style
 
-`go fmt`
+ - `go fmt`
+
+ - Unix line breaks
 
 ## Documentation
 
@@ -52,5 +102,4 @@ Yes please!
 
 ## License
 
-MIT
-
+GPLv3

CONTRIBUTORS

@@ -1,10 +1,24 @@
 Aaron Bieber <qbit@deftly.net>
+Alexander Graf <register-github@alex-graf.de>
 Andrew Dunham <andrew@du.nham.ca>
-Arthur Axel fREW Schmidt <frew@afoolishmanifesto.com>
+Audrius Butkevicius <audrius.butkevicius@gmail.com>
+Arthur Axel fREW Schmidt <frew@afoolishmanifesto.com> <frioux@gmail.com>
 Ben Sidhom <bsidhom@gmail.com>
 Brandon Philips <brandon@ifup.org>
-James Patterson <jamespatterson@operamail.com>
-Jens Diemer <github.com@jensdiemer.de>
+Caleb Callaway <enlightened.despot@gmail.com>
+Chris Joel <chris@scriptolo.gy>
+Daniel Martí <mvdan@mvdan.cc>
+Felix Ableitner <me@nutomic.com>
+Felix Unterpaintner <bigbear2nd@gmail.com>
+Gilli Sigurdsson <gilli@vx.is>
+James Patterson <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
+Jens Diemer <github.com@jensdiemer.de> <git@jensdiemer.de>
+Jochen Voss <voss@seehuhn.de>
+Lode Hoste <zillode@zillode.be>
+Marcin Dziadus <dziadus.marcin@gmail.com>
+Michael Tilli <pyfisch@gmail.com>
 Philippe Schommers <philippe@schommers.be>
+Phill Luby <phill.luby@newredo.com>
 Ryan Sullivan <kayoticsully@gmail.com>
+Tully Robinson <tully@tojr.org>
 Veeti Paananen <veeti.paananen@rojekti.fi>

Godeps/Godeps.json

@@ -1,36 +1,29 @@
 {
-	"ImportPath": "github.com/calmh/syncthing",
-	"GoVersion": "go1.3",
+	"ImportPath": "github.com/syncthing/syncthing",
+	"GoVersion": "go1.3.1",
 	"Packages": [
-		"./cmd/syncthing",
-		"./cmd/assets",
-		"./discover/cmd/discosrv"
+		"./cmd/..."
 	],
 	"Deps": [
-		{
-			"ImportPath": "bitbucket.org/kardianos/osext",
-			"Comment": "null-13",
-			"Rev": "5d3ddcf53a508cc2f7404eaebf546ef2cb5cdb6e"
-		},
 		{
 			"ImportPath": "code.google.com/p/go.crypto/bcrypt",
-			"Comment": "null-212",
-			"Rev": "1064b89a6fb591df0dd65422295b8498916b092f"
+			"Comment": "null-216",
+			"Rev": "41cd4647fccc72b0b79ef1bd1fe6735e718257cd"
 		},
 		{
 			"ImportPath": "code.google.com/p/go.crypto/blowfish",
-			"Comment": "null-212",
-			"Rev": "1064b89a6fb591df0dd65422295b8498916b092f"
+			"Comment": "null-216",
+			"Rev": "41cd4647fccc72b0b79ef1bd1fe6735e718257cd"
 		},
 		{
 			"ImportPath": "code.google.com/p/go.text/transform",
-			"Comment": "null-87",
-			"Rev": "c59e4f2f93824f81213799e64c3eead7be24660a"
+			"Comment": "null-90",
+			"Rev": "d65bffbc88a153d23a6d2a864531e6e7c2cde59b"
 		},
 		{
 			"ImportPath": "code.google.com/p/go.text/unicode/norm",
-			"Comment": "null-87",
-			"Rev": "c59e4f2f93824f81213799e64c3eead7be24660a"
+			"Comment": "null-90",
+			"Rev": "d65bffbc88a153d23a6d2a864531e6e7c2cde59b"
 		},
 		{
 			"ImportPath": "code.google.com/p/snappy-go/snappy",
@@ -38,8 +31,20 @@
 			"Rev": "12e4b4183793ac4b061921e7980845e750679fd0"
 		},
 		{
-			"ImportPath": "github.com/golang/groupcache/lru",
-			"Rev": "a531d51b7f9f3dd13c1c2b50d42d739b70442dbb"
+			"ImportPath": "github.com/AudriusButkevicius/lfu-go",
+			"Rev": "164bcecceb92fd6037f4d18a8d97b495ec6ef669"
+		},
+		{
+			"ImportPath": "github.com/bkaradzic/go-lz4",
+			"Rev": "93a831dcee242be64a9cc9803dda84af25932de7"
+		},
+		{
+			"ImportPath": "github.com/calmh/osext",
+			"Rev": "9bf61584e5f1f172e8766ddc9022d9c401faaa5e"
+		},
+		{
+			"ImportPath": "github.com/calmh/xdr",
+			"Rev": "a597b63b87d6140f79084c8aab214b4d533833a1"
 		},
 		{
 			"ImportPath": "github.com/juju/ratelimit",
@@ -47,7 +52,7 @@
 		},
 		{
 			"ImportPath": "github.com/syndtr/goleveldb/leveldb",
-			"Rev": "e1f2d2bdccd7c62f4d4a29aaf081bf1fc4404f91"
+			"Rev": "0d8857b7ec571b0a6c9677d8e6c0a4ceeabd1d71"
 		},
 		{
 			"ImportPath": "github.com/vitrun/qart/coding",

Godeps/_workspace/src/code.google.com/p/go.crypto/blowfish/block.go

@@ -4,6 +4,22 @@
 
 package blowfish
 
+// getNextWord returns the next big-endian uint32 value from the byte slice
+// at the given position in a circular manner, updating the position.
+func getNextWord(b []byte, pos *int) uint32 {
+	var w uint32
+	j := *pos
+	for i := 0; i < 4; i++ {
+		w = w<<8 | uint32(b[j])
+		j++
+		if j >= len(b) {
+			j = 0
+		}
+	}
+	*pos = j
+	return w
+}
+
 // ExpandKey performs a key expansion on the given *Cipher. Specifically, it
 // performs the Blowfish algorithm's key schedule which sets up the *Cipher's
 // pi and substitution tables for calls to Encrypt. This is used, primarily,
@@ -12,6 +28,7 @@ package blowfish
 func ExpandKey(key []byte, c *Cipher) {
 	j := 0
 	for i := 0; i < 18; i++ {
+		// Using inlined getNextWord for performance.
 		var d uint32
 		for k := 0; k < 4; k++ {
 			d = d<<8 | uint32(key[j])
@@ -54,86 +71,44 @@ func ExpandKey(key []byte, c *Cipher) {
 func expandKeyWithSalt(key []byte, salt []byte, c *Cipher) {
 	j := 0
 	for i := 0; i < 18; i++ {
-		var d uint32
-		for k := 0; k < 4; k++ {
-			d = d<<8 | uint32(key[j])
-			j++
-			if j >= len(key) {
-				j = 0
-			}
-		}
-		c.p[i] ^= d
+		c.p[i] ^= getNextWord(key, &j)
 	}
 
 	j = 0
-	var expandedSalt [4]uint32
-	for i := range expandedSalt {
-		var d uint32
-		for k := 0; k < 4; k++ {
-			d = d<<8 | uint32(salt[j])
-			j++
-			if j >= len(salt) {
-				j = 0
-			}
-		}
-		expandedSalt[i] = d
-	}
-
 	var l, r uint32
 	for i := 0; i < 18; i += 2 {
-		l ^= expandedSalt[i&2]
-		r ^= expandedSalt[(i&2)+1]
+		l ^= getNextWord(salt, &j)
+		r ^= getNextWord(salt, &j)
 		l, r = encryptBlock(l, r, c)
 		c.p[i], c.p[i+1] = l, r
 	}
 
-	for i := 0; i < 256; i += 4 {
-		l ^= expandedSalt[2]
-		r ^= expandedSalt[3]
+	for i := 0; i < 256; i += 2 {
+		l ^= getNextWord(salt, &j)
+		r ^= getNextWord(salt, &j)
 		l, r = encryptBlock(l, r, c)
 		c.s0[i], c.s0[i+1] = l, r
-
-		l ^= expandedSalt[0]
-		r ^= expandedSalt[1]
-		l, r = encryptBlock(l, r, c)
-		c.s0[i+2], c.s0[i+3] = l, r
-
 	}
 
-	for i := 0; i < 256; i += 4 {
-		l ^= expandedSalt[2]
-		r ^= expandedSalt[3]
+	for i := 0; i < 256; i += 2 {
+		l ^= getNextWord(salt, &j)
+		r ^= getNextWord(salt, &j)
 		l, r = encryptBlock(l, r, c)
 		c.s1[i], c.s1[i+1] = l, r
-
-		l ^= expandedSalt[0]
-		r ^= expandedSalt[1]
-		l, r = encryptBlock(l, r, c)
-		c.s1[i+2], c.s1[i+3] = l, r
 	}
 
-	for i := 0; i < 256; i += 4 {
-		l ^= expandedSalt[2]
-		r ^= expandedSalt[3]
+	for i := 0; i < 256; i += 2 {
+		l ^= getNextWord(salt, &j)
+		r ^= getNextWord(salt, &j)
 		l, r = encryptBlock(l, r, c)
 		c.s2[i], c.s2[i+1] = l, r
-
-		l ^= expandedSalt[0]
-		r ^= expandedSalt[1]
-		l, r = encryptBlock(l, r, c)
-		c.s2[i+2], c.s2[i+3] = l, r
 	}
 
-	for i := 0; i < 256; i += 4 {
-		l ^= expandedSalt[2]
-		r ^= expandedSalt[3]
+	for i := 0; i < 256; i += 2 {
+		l ^= getNextWord(salt, &j)
+		r ^= getNextWord(salt, &j)
 		l, r = encryptBlock(l, r, c)
 		c.s3[i], c.s3[i+1] = l, r
-
-		l ^= expandedSalt[0]
-		r ^= expandedSalt[1]
-		l, r = encryptBlock(l, r, c)
-		c.s3[i+2], c.s3[i+3] = l, r
 	}
 }
 
@@ -182,9 +157,3 @@ func decryptBlock(l, r uint32, c *Cipher) (uint32, uint32) {
 	xr ^= c.p[0]
 	return xr, xl
 }
-
-func zero(x []uint32) {
-	for i := range x {
-		x[i] = 0
-	}
-}

Godeps/_workspace/src/code.google.com/p/go.crypto/blowfish/blowfish_test.go

@@ -4,9 +4,7 @@
 
 package blowfish
 
-import (
-	"testing"
-)
+import "testing"
 
 type CryptTest struct {
 	key []byte
@@ -202,3 +200,75 @@ func TestSaltedCipherKeyLength(t *testing.T) {
 		t.Errorf("NewSaltedCipher with long key, gave error %#v", err)
 	}
 }
+
+// Test vectors generated with Blowfish from OpenSSH.
+var saltedVectors = [][8]byte{
+	{0x0c, 0x82, 0x3b, 0x7b, 0x8d, 0x01, 0x4b, 0x7e},
+	{0xd1, 0xe1, 0x93, 0xf0, 0x70, 0xa6, 0xdb, 0x12},
+	{0xfc, 0x5e, 0xba, 0xde, 0xcb, 0xf8, 0x59, 0xad},
+	{0x8a, 0x0c, 0x76, 0xe7, 0xdd, 0x2c, 0xd3, 0xa8},
+	{0x2c, 0xcb, 0x7b, 0xee, 0xac, 0x7b, 0x7f, 0xf8},
+	{0xbb, 0xf6, 0x30, 0x6f, 0xe1, 0x5d, 0x62, 0xbf},
+	{0x97, 0x1e, 0xc1, 0x3d, 0x3d, 0xe0, 0x11, 0xe9},
+	{0x06, 0xd7, 0x4d, 0xb1, 0x80, 0xa3, 0xb1, 0x38},
+	{0x67, 0xa1, 0xa9, 0x75, 0x0e, 0x5b, 0xc6, 0xb4},
+	{0x51, 0x0f, 0x33, 0x0e, 0x4f, 0x67, 0xd2, 0x0c},
+	{0xf1, 0x73, 0x7e, 0xd8, 0x44, 0xea, 0xdb, 0xe5},
+	{0x14, 0x0e, 0x16, 0xce, 0x7f, 0x4a, 0x9c, 0x7b},
+	{0x4b, 0xfe, 0x43, 0xfd, 0xbf, 0x36, 0x04, 0x47},
+	{0xb1, 0xeb, 0x3e, 0x15, 0x36, 0xa7, 0xbb, 0xe2},
+	{0x6d, 0x0b, 0x41, 0xdd, 0x00, 0x98, 0x0b, 0x19},
+	{0xd3, 0xce, 0x45, 0xce, 0x1d, 0x56, 0xb7, 0xfc},
+	{0xd9, 0xf0, 0xfd, 0xda, 0xc0, 0x23, 0xb7, 0x93},
+	{0x4c, 0x6f, 0xa1, 0xe4, 0x0c, 0xa8, 0xca, 0x57},
+	{0xe6, 0x2f, 0x28, 0xa7, 0x0c, 0x94, 0x0d, 0x08},
+	{0x8f, 0xe3, 0xf0, 0xb6, 0x29, 0xe3, 0x44, 0x03},
+	{0xff, 0x98, 0xdd, 0x04, 0x45, 0xb4, 0x6d, 0x1f},
+	{0x9e, 0x45, 0x4d, 0x18, 0x40, 0x53, 0xdb, 0xef},
+	{0xb7, 0x3b, 0xef, 0x29, 0xbe, 0xa8, 0x13, 0x71},
+	{0x02, 0x54, 0x55, 0x41, 0x8e, 0x04, 0xfc, 0xad},
+	{0x6a, 0x0a, 0xee, 0x7c, 0x10, 0xd9, 0x19, 0xfe},
+	{0x0a, 0x22, 0xd9, 0x41, 0xcc, 0x23, 0x87, 0x13},
+	{0x6e, 0xff, 0x1f, 0xff, 0x36, 0x17, 0x9c, 0xbe},
+	{0x79, 0xad, 0xb7, 0x40, 0xf4, 0x9f, 0x51, 0xa6},
+	{0x97, 0x81, 0x99, 0xa4, 0xde, 0x9e, 0x9f, 0xb6},
+	{0x12, 0x19, 0x7a, 0x28, 0xd0, 0xdc, 0xcc, 0x92},
+	{0x81, 0xda, 0x60, 0x1e, 0x0e, 0xdd, 0x65, 0x56},
+	{0x7d, 0x76, 0x20, 0xb2, 0x73, 0xc9, 0x9e, 0xee},
+}
+
+func TestSaltedCipher(t *testing.T) {
+	var key, salt [32]byte
+	for i := range key {
+		key[i] = byte(i)
+		salt[i] = byte(i + 32)
+	}
+	for i, v := range saltedVectors {
+		c, err := NewSaltedCipher(key[:], salt[:i])
+		if err != nil {
+			t.Fatal(err)
+		}
+		var buf [8]byte
+		c.Encrypt(buf[:], buf[:])
+		if v != buf {
+			t.Errorf("%d: expected %x, got %x", i, v, buf)
+		}
+	}
+}
+
+func BenchmarkExpandKeyWithSalt(b *testing.B) {
+	key := make([]byte, 32)
+	salt := make([]byte, 16)
+	c, _ := NewCipher(key)
+	for i := 0; i < b.N; i++ {
+		expandKeyWithSalt(key, salt, c)
+	}
+}
+
+func BenchmarkExpandKey(b *testing.B) {
+	key := make([]byte, 32)
+	c, _ := NewCipher(key)
+	for i := 0; i < b.N; i++ {
+		ExpandKey(key, c)
+	}
+}

Godeps/_workspace/src/code.google.com/p/go.crypto/blowfish/cipher.go

@@ -40,8 +40,11 @@ func NewCipher(key []byte) (*Cipher, error) {
 // NewSaltedCipher creates a returns a Cipher that folds a salt into its key
 // schedule. For most purposes, NewCipher, instead of NewSaltedCipher, is
 // sufficient and desirable. For bcrypt compatiblity, the key can be over 56
-// bytes. Only the first 16 bytes of salt are used.
+// bytes.
 func NewSaltedCipher(key, salt []byte) (*Cipher, error) {
+	if len(salt) == 0 {
+		return NewCipher(key)
+	}
 	var result Cipher
 	if k := len(key); k < 1 {
 		return nil, KeySizeError(k)

Godeps/_workspace/src/code.google.com/p/go.text/transform/transform.go

@@ -9,6 +9,7 @@
 package transform
 
 import (
+	"bytes"
 	"errors"
 	"io"
 	"unicode/utf8"
@@ -127,7 +128,7 @@ func (r *Reader) Read(p []byte) (int, error) {
 				// cannot read more bytes into src.
 				r.transformComplete = r.err != nil
 				continue
-			case err == ErrShortDst && r.dst1 != 0:
+			case err == ErrShortDst && (r.dst1 != 0 || n != 0):
 				// Make room in dst by copying out, and try again.
 				continue
 			case err == ErrShortSrc && r.src1-r.src0 != len(r.src) && r.err == nil:
@@ -210,7 +211,7 @@ func (w *Writer) Write(data []byte) (n int, err error) {
 			n += nSrc
 		}
 		switch {
-		case err == ErrShortDst && nDst > 0:
+		case err == ErrShortDst && (nDst > 0 || nSrc > 0):
 		case err == ErrShortSrc && len(src) < len(w.src):
 			m := copy(w.src, src)
 			// If w.n > 0, bytes from data were already copied to w.src and n
@@ -467,30 +468,125 @@ func (t removeF) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err err
 	return
 }
 
-// Bytes returns a new byte slice with the result of converting b using t.
-// If any unrecoverable error occurs it returns nil.
-func Bytes(t Transformer, b []byte) []byte {
-	out := make([]byte, len(b))
-	n := 0
-	for {
-		nDst, nSrc, err := t.Transform(out[n:], b, true)
-		n += nDst
-		if err == nil {
-			return out[:n]
-		} else if err != ErrShortDst {
-			return nil
-		}
-		b = b[nSrc:]
+// grow returns a new []byte that is longer than b, and copies the first n bytes
+// of b to the start of the new slice.
+func grow(b []byte, n int) []byte {
+	m := len(b)
+	if m <= 256 {
+		m *= 2
+	} else {
+		m += m >> 1
+	}
+	buf := make([]byte, m)
+	copy(buf, b[:n])
+	return buf
+}
 
-		// Grow the destination buffer.
-		sz := len(out)
-		if sz <= 256 {
-			sz *= 2
-		} else {
-			sz += sz >> 1
+const initialBufSize = 128
+
+// String returns a string with the result of converting s[:n] using t, where
+// n <= len(s). If err == nil, n will be len(s).
+func String(t Transformer, s string) (result string, n int, err error) {
+	if s == "" {
+		return "", 0, nil
+	}
+
+	// Allocate only once. Note that both dst and src escape when passed to
+	// Transform.
+	buf := [2 * initialBufSize]byte{}
+	dst := buf[:initialBufSize:initialBufSize]
+	src := buf[initialBufSize : 2*initialBufSize]
+
+	// Avoid allocation if the transformed string is identical to the original.
+	// After this loop, pDst will point to the furthest point in s for which it
+	// could be detected that t gives equal results, src[:nSrc] will
+	// indicated the last processed chunk of s for which the output is not equal
+	// and dst[:nDst] will be the transform of this chunk.
+	var nDst, nSrc int
+	pDst := 0 // Used as index in both src and dst in this loop.
+	for {
+		n := copy(src, s[pDst:])
+		nDst, nSrc, err = t.Transform(dst, src[:n], pDst+n == len(s))
+
+		// Note 1: we will not enter the loop with pDst == len(s) and we will
+		// not end the loop with it either. So if nSrc is 0, this means there is
+		// some kind of error from which we cannot recover given the current
+		// buffer sizes. We will give up in this case.
+		// Note 2: it is not entirely correct to simply do a bytes.Equal as
+		// a Transformer may buffer internally. It will work in most cases,
+		// though, and no harm is done if it doesn't work.
+		// TODO:  let transformers implement an optional Spanner interface, akin
+		// to norm's QuickSpan. This would even allow us to avoid any allocation.
+		if nSrc == 0 || !bytes.Equal(dst[:nDst], src[:nSrc]) {
+			break
+		}
+
+		if pDst += nDst; pDst == len(s) {
+			return s, pDst, nil
+		}
+	}
+
+	// Move the bytes seen so far to dst.
+	pSrc := pDst + nSrc
+	if pDst+nDst <= initialBufSize {
+		copy(dst[pDst:], dst[:nDst])
+	} else {
+		b := make([]byte, len(s)+nDst-nSrc)
+		copy(b[pDst:], dst[:nDst])
+		dst = b
+	}
+	copy(dst, s[:pDst])
+	pDst += nDst
+
+	if err != nil && err != ErrShortDst && err != ErrShortSrc {
+		return string(dst[:pDst]), pSrc, err
+	}
+
+	// Complete the string with the remainder.
+	for {
+		n := copy(src, s[pSrc:])
+		nDst, nSrc, err = t.Transform(dst[pDst:], src[:n], pSrc+n == len(s))
+		pDst += nDst
+		pSrc += nSrc
+
+		switch err {
+		case nil:
+			if pSrc == len(s) {
+				return string(dst[:pDst]), pSrc, nil
+			}
+		case ErrShortDst:
+			// Do not grow as long as we can make progress. This may avoid
+			// excessive allocations.
+			if nDst == 0 {
+				dst = grow(dst, pDst)
+			}
+		case ErrShortSrc:
+			if nSrc == 0 {
+				src = grow(src, 0)
+			}
+		default:
+			return string(dst[:pDst]), pSrc, err
+		}
+	}
+}
+
+// Bytes returns a new byte slice with the result of converting b[:n] using t,
+// where n <= len(b). If err == nil, n will be len(b).
+func Bytes(t Transformer, b []byte) (result []byte, n int, err error) {
+	dst := make([]byte, len(b))
+	pDst, pSrc := 0, 0
+	for {
+		nDst, nSrc, err := t.Transform(dst[pDst:], b[pSrc:], true)
+		pDst += nDst
+		pSrc += nSrc
+		if err != ErrShortDst {
+			return dst[:pDst], pSrc, err
+		}
+
+		// Grow the destination buffer, but do not grow as long as we can make
+		// progress. This may avoid excessive allocations.
+		if nDst == 0 {
+			dst = grow(dst, pDst)
 		}
-		out2 := make([]byte, sz)
-		copy(out2, out[:n])
-		out = out2
 	}
 }

Godeps/_workspace/src/code.google.com/p/go.text/transform/transform_test.go

@@ -12,6 +12,7 @@ import (
 	"strconv"
 	"strings"
 	"testing"
+	"time"
 	"unicode/utf8"
 )
 
@@ -132,6 +133,43 @@ func (e rleEncode) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err e
 	return nDst, nSrc, nil
 }
 
+// trickler consumes all input bytes, but writes a single byte at a time to dst.
+type trickler []byte
+
+func (t *trickler) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	*t = append(*t, src...)
+	if len(*t) == 0 {
+		return 0, 0, nil
+	}
+	if len(dst) == 0 {
+		return 0, len(src), ErrShortDst
+	}
+	dst[0] = (*t)[0]
+	*t = (*t)[1:]
+	if len(*t) > 0 {
+		err = ErrShortDst
+	}
+	return 1, len(src), err
+}
+
+// delayedTrickler is like trickler, but delays writing output to dst. This is
+// highly unlikely to be relevant in practice, but it seems like a good idea
+// to have some tolerance as long as progress can be detected.
+type delayedTrickler []byte
+
+func (t *delayedTrickler) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	if len(*t) > 0 && len(dst) > 0 {
+		dst[0] = (*t)[0]
+		*t = (*t)[1:]
+		nDst = 1
+	}
+	*t = append(*t, src...)
+	if len(*t) > 0 {
+		err = ErrShortDst
+	}
+	return nDst, len(src), err
+}
+
 type testCase struct {
 	desc     string
 	t        Transformer
@@ -170,6 +208,15 @@ func (c chain) String() string {
 }
 
 var testCases = []testCase{
+	{
+		desc:    "empty",
+		t:       lowerCaseASCII{},
+		src:     "",
+		dstSize: 100,
+		srcSize: 100,
+		wantStr: "",
+	},
+
 	{
 		desc:    "basic",
 		t:       lowerCaseASCII{},
@@ -378,6 +425,24 @@ var testCases = []testCase{
 		ioSize:  10,
 		wantStr: "4a6b2b4c4d1d",
 	},
+
+	{
+		desc:    "trickler",
+		t:       &trickler{},
+		src:     "abcdefghijklm",
+		dstSize: 3,
+		srcSize: 15,
+		wantStr: "abcdefghijklm",
+	},
+
+	{
+		desc:    "delayedTrickler",
+		t:       &delayedTrickler{},
+		src:     "abcdefghijklm",
+		dstSize: 3,
+		srcSize: 15,
+		wantStr: "abcdefghijklm",
+	},
 }
 
 func TestReader(t *testing.T) {
@@ -685,7 +750,7 @@ func doTransform(tc testCase) (res string, iter int, err error) {
 		switch {
 		case err == nil && len(in) != 0:
 		case err == ErrShortSrc && nSrc > 0:
-		case err == ErrShortDst && nDst > 0:
+		case err == ErrShortDst && (nDst > 0 || nSrc > 0):
 		default:
 			return string(out), iter, err
 		}
@@ -875,27 +940,136 @@ func TestRemoveFunc(t *testing.T) {
 	}
 }
 
-func TestBytes(t *testing.T) {
+func testString(t *testing.T, f func(Transformer, string) (string, int, error)) {
 	for _, tt := range append(testCases, chainTests()...) {
 		if tt.desc == "allowStutter = true" {
 			// We don't have control over the buffer size, so we eliminate tests
 			// that depend on a specific buffer size being set.
 			continue
 		}
-		got := Bytes(tt.t, []byte(tt.src))
-		if tt.wantErr != nil {
-			if tt.wantErr != ErrShortDst && tt.wantErr != ErrShortSrc {
-				// Bytes should return nil for non-recoverable errors.
-				if g, w := (got == nil), (tt.wantErr != nil); g != w {
-					t.Errorf("%s:error: got %v; want %v", tt.desc, g, w)
-				}
-			}
-			// The output strings in the tests that expect an error will
-			// almost certainly not be the same as the result of Bytes.
+		reset(tt.t)
+		if tt.wantErr == ErrShortDst || tt.wantErr == ErrShortSrc {
+			// The result string will be different.
 			continue
 		}
-		if string(got) != tt.wantStr {
+		got, n, err := f(tt.t, tt.src)
+		if tt.wantErr != err {
+			t.Errorf("%s:error: got %v; want %v", tt.desc, err, tt.wantErr)
+		}
+		if got, want := err == nil, n == len(tt.src); got != want {
+			t.Errorf("%s:n: got %v; want %v", tt.desc, got, want)
+		}
+		if got != tt.wantStr {
 			t.Errorf("%s:string: got %q; want %q", tt.desc, got, tt.wantStr)
 		}
 	}
 }
+
+func TestBytes(t *testing.T) {
+	testString(t, func(z Transformer, s string) (string, int, error) {
+		b, n, err := Bytes(z, []byte(s))
+		return string(b), n, err
+	})
+}
+
+func TestString(t *testing.T) {
+	testString(t, String)
+
+	// Overrun the internal destination buffer.
+	for i, s := range []string{
+		strings.Repeat("a", initialBufSize-1),
+		strings.Repeat("a", initialBufSize+0),
+		strings.Repeat("a", initialBufSize+1),
+		strings.Repeat("A", initialBufSize-1),
+		strings.Repeat("A", initialBufSize+0),
+		strings.Repeat("A", initialBufSize+1),
+		strings.Repeat("A", 2*initialBufSize-1),
+		strings.Repeat("A", 2*initialBufSize+0),
+		strings.Repeat("A", 2*initialBufSize+1),
+		strings.Repeat("a", initialBufSize-2) + "A",
+		strings.Repeat("a", initialBufSize-1) + "A",
+		strings.Repeat("a", initialBufSize+0) + "A",
+		strings.Repeat("a", initialBufSize+1) + "A",
+	} {
+		got, _, _ := String(lowerCaseASCII{}, s)
+		if want := strings.ToLower(s); got != want {
+			t.Errorf("%d:dst buffer test: got %s (%d); want %s (%d)", i, got, len(got), want, len(want))
+		}
+	}
+
+	// Overrun the internal source buffer.
+	for i, s := range []string{
+		strings.Repeat("a", initialBufSize-1),
+		strings.Repeat("a", initialBufSize+0),
+		strings.Repeat("a", initialBufSize+1),
+		strings.Repeat("a", 2*initialBufSize+1),
+		strings.Repeat("a", 2*initialBufSize+0),
+		strings.Repeat("a", 2*initialBufSize+1),
+	} {
+		got, _, _ := String(rleEncode{}, s)
+		if want := fmt.Sprintf("%da", len(s)); got != want {
+			t.Errorf("%d:src buffer test: got %s (%d); want %s (%d)", i, got, len(got), want, len(want))
+		}
+	}
+
+	// Test allocations for non-changing strings.
+	// Note we still need to allocate a single buffer.
+	for i, s := range []string{
+		"",
+		"123",
+		"123456789",
+		strings.Repeat("a", initialBufSize),
+		strings.Repeat("a", 10*initialBufSize),
+	} {
+		if n := testing.AllocsPerRun(5, func() { String(&lowerCaseASCII{}, s) }); n > 1 {
+			t.Errorf("%d: #allocs was %f; want 1", i, n)
+		}
+	}
+}
+
+// TestBytesAllocation tests that buffer growth stays limited with the trickler
+// transformer, which behaves oddly but within spec. In case buffer growth is
+// not correctly handled, the test will either panic with a failed allocation or
+// thrash. To ensure the tests terminate under the last condition, we time out
+// after some sufficiently long period of time.
+func TestBytesAllocation(t *testing.T) {
+	done := make(chan bool)
+	go func() {
+		in := bytes.Repeat([]byte{'a'}, 1000)
+		tr := trickler(make([]byte, 1))
+		Bytes(&tr, in)
+		done <- true
+	}()
+	select {
+	case <-done:
+	case <-time.After(3 * time.Second):
+		t.Error("time out, likely due to excessive allocation")
+	}
+}
+
+// TestStringAllocation tests that buffer growth stays limited with the trickler
+// transformer, which behaves oddly but within spec. In case buffer growth is
+// not correctly handled, the test will either panic with a failed allocation or
+// thrash. To ensure the tests terminate under the last condition, we time out
+// after some sufficiently long period of time.
+func TestStringAllocation(t *testing.T) {
+	done := make(chan bool)
+	go func() {
+		in := strings.Repeat("a", 1000)
+		tr := trickler(make([]byte, 1))
+		String(&tr, in)
+		done <- true
+	}()
+	select {
+	case <-done:
+	case <-time.After(3 * time.Second):
+		t.Error("time out, likely due to excessive allocation")
+	}
+}
+
+func BenchmarkStringLower(b *testing.B) {
+	in := strings.Repeat("a", 4096)
+	for i := 0; i < b.N; i++ {
+		String(&lowerCaseASCII{}, in)
+	}
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/maketables.go

@@ -11,7 +11,6 @@
 package main
 
 import (
-	"bufio"
 	"bytes"
 	"flag"
 	"fmt"
@@ -24,6 +23,8 @@ import (
 	"strconv"
 	"strings"
 	"unicode"
+
+	"code.google.com/p/go.text/internal/ucd"
 )
 
 func main() {
@@ -63,31 +64,7 @@ var localFiles = flag.Bool("local",
 
 var logger = log.New(os.Stderr, "", log.Lshortfile)
 
-// UnicodeData.txt has form:
-//	0037;DIGIT SEVEN;Nd;0;EN;;7;7;7;N;;;;;
-//	007A;LATIN SMALL LETTER Z;Ll;0;L;;;;;N;;;005A;;005A
-// See http://unicode.org/reports/tr44/ for full explanation
-// The fields:
-const (
-	FCodePoint = iota
-	FName
-	FGeneralCategory
-	FCanonicalCombiningClass
-	FBidiClass
-	FDecompMapping
-	FDecimalValue
-	FDigitValue
-	FNumericValue
-	FBidiMirrored
-	FUnicode1Name
-	FISOComment
-	FSimpleUppercaseMapping
-	FSimpleLowercaseMapping
-	FSimpleTitlecaseMapping
-	NumField
-
-	MaxChar = 0x10FFFF // anything above this shouldn't exist
-)
+const MaxChar = 0x10FFFF // anything above this shouldn't exist
 
 // Quick Check properties of runes allow us to quickly
 // determine whether a rune may occur in a normal form.
@@ -232,7 +209,7 @@ func openReader(file string) (input io.ReadCloser) {
 	return
 }
 
-func parseDecomposition(s string, skipfirst bool) (a []rune, e error) {
+func parseDecomposition(s string, skipfirst bool) (a []rune, err error) {
 	decomp := strings.Split(s, " ")
 	if len(decomp) > 0 && skipfirst {
 		decomp = decomp[1:]
@@ -247,56 +224,31 @@ func parseDecomposition(s string, skipfirst bool) (a []rune, e error) {
 	return a, nil
 }
 
-func parseCharacter(line string) {
-	field := strings.Split(line, ";")
-	if len(field) != NumField {
-		logger.Fatalf("%5s: %d fields (expected %d)\n", line, len(field), NumField)
-	}
-	x, err := strconv.ParseUint(field[FCodePoint], 16, 64)
-	point := int(x)
-	if err != nil {
-		logger.Fatalf("%.5s...: %s", line, err)
-	}
-	if point == 0 {
-		return // not interesting and we use 0 as unset
-	}
-	if point > MaxChar {
-		logger.Fatalf("%5s: Rune %X > MaxChar (%X)", line, point, MaxChar)
-		return
-	}
-	state := SNormal
-	switch {
-	case strings.Index(field[FName], ", First>") > 0:
-		state = SFirst
-	case strings.Index(field[FName], ", Last>") > 0:
-		state = SLast
-	}
-	firstChar := lastChar + 1
-	lastChar = rune(point)
-	if state != SLast {
-		firstChar = lastChar
-	}
-	x, err = strconv.ParseUint(field[FCanonicalCombiningClass], 10, 64)
-	if err != nil {
-		logger.Fatalf("%U: bad ccc field: %s", int(x), err)
-	}
-	ccc := uint8(x)
-	decmap := field[FDecompMapping]
-	exp, e := parseDecomposition(decmap, false)
-	isCompat := false
-	if e != nil {
-		if len(decmap) > 0 {
-			exp, e = parseDecomposition(decmap, true)
-			if e != nil {
-				logger.Fatalf(`%U: bad decomp |%v|: "%s"`, int(x), decmap, e)
+func loadUnicodeData() {
+	f := openReader("UnicodeData.txt")
+	defer f.Close()
+	p := ucd.New(f)
+	for p.Next() {
+		r := p.Rune(ucd.CodePoint)
+		char := &chars[r]
+
+		char.ccc = uint8(p.Uint(ucd.CanonicalCombiningClass))
+		decmap := p.String(ucd.DecompMapping)
+
+		exp, err := parseDecomposition(decmap, false)
+		isCompat := false
+		if err != nil {
+			if len(decmap) > 0 {
+				exp, err = parseDecomposition(decmap, true)
+				if err != nil {
+					logger.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err)
+				}
+				isCompat = true
 			}
-			isCompat = true
 		}
-	}
-	for i := firstChar; i <= lastChar; i++ {
-		char := &chars[i]
-		char.name = field[FName]
-		char.codePoint = i
+
+		char.name = p.String(ucd.Name)
+		char.codePoint = r
 		char.forms[FCompatibility].decomp = exp
 		if !isCompat {
 			char.forms[FCanonical].decomp = exp
@@ -306,24 +258,9 @@ func parseCharacter(line string) {
 		if len(decmap) > 0 {
 			char.forms[FCompatibility].decomp = exp
 		}
-		char.ccc = ccc
-		char.state = SMissing
-		if i == lastChar {
-			char.state = state
-		}
 	}
-	return
-}
-
-func loadUnicodeData() {
-	f := openReader("UnicodeData.txt")
-	defer f.Close()
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		parseCharacter(scanner.Text())
-	}
-	if scanner.Err() != nil {
-		logger.Fatal(scanner.Err())
+	if err := p.Err(); err != nil {
+		logger.Fatal(err)
 	}
 }
 
@@ -354,47 +291,22 @@ func compactCCC() {
 	}
 }
 
-var singlePointRe = regexp.MustCompile(`^([0-9A-F]+) *$`)
-
 // CompositionExclusions.txt has form:
 // 0958    # ...
 // See http://unicode.org/reports/tr44/ for full explanation
-func parseExclusion(line string) int {
-	comment := strings.Index(line, "#")
-	if comment >= 0 {
-		line = line[0:comment]
-	}
-	if len(line) == 0 {
-		return 0
-	}
-	matches := singlePointRe.FindStringSubmatch(line)
-	if len(matches) != 2 {
-		logger.Fatalf("%s: %d matches (expected 1)\n", line, len(matches))
-	}
-	point, err := strconv.ParseUint(matches[1], 16, 64)
-	if err != nil {
-		logger.Fatalf("%.5s...: %s", line, err)
-	}
-	return int(point)
-}
-
 func loadCompositionExclusions() {
 	f := openReader("CompositionExclusions.txt")
 	defer f.Close()
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		point := parseExclusion(scanner.Text())
-		if point == 0 {
-			continue
-		}
-		c := &chars[point]
+	p := ucd.New(f)
+	for p.Next() {
+		c := &chars[p.Rune(0)]
 		if c.excludeInComp {
 			logger.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint)
 		}
 		c.excludeInComp = true
 	}
-	if scanner.Err() != nil {
-		log.Fatal(scanner.Err())
+	if e := p.Err(); e != nil {
+		logger.Fatal(e)
 	}
 }
 
@@ -988,8 +900,6 @@ func verifyComputed() {
 	}
 }
 
-var qcRe = regexp.MustCompile(`([0-9A-F\.]+) *; (NF.*_QC); ([YNM]) #.*`)
-
 // Use values in DerivedNormalizationProps.txt to compare against the
 // values we computed.
 // DerivedNormalizationProps.txt has form:
@@ -999,27 +909,13 @@ var qcRe = regexp.MustCompile(`([0-9A-F\.]+) *; (NF.*_QC); ([YNM]) #.*`)
 func testDerived() {
 	f := openReader("DerivedNormalizationProps.txt")
 	defer f.Close()
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		line := scanner.Text()
-		qc := qcRe.FindStringSubmatch(line)
-		if qc == nil {
-			continue
-		}
-		rng := strings.Split(qc[1], "..")
-		i, err := strconv.ParseUint(rng[0], 16, 64)
-		if err != nil {
-			log.Fatal(err)
-		}
-		j := i
-		if len(rng) > 1 {
-			j, err = strconv.ParseUint(rng[1], 16, 64)
-			if err != nil {
-				log.Fatal(err)
-			}
-		}
+	p := ucd.New(f)
+	for p.Next() {
+		r := p.Rune(0)
+		c := &chars[r]
+
 		var ftype, mode int
-		qt := strings.TrimSpace(qc[2])
+		qt := p.String(1)
 		switch qt {
 		case "NFC_QC":
 			ftype, mode = FCanonical, MComposed
@@ -1030,10 +926,10 @@ func testDerived() {
 		case "NFKD_QC":
 			ftype, mode = FCompatibility, MDecomposed
 		default:
-			log.Fatalf(`Unexpected quick check type "%s"`, qt)
+			continue
 		}
 		var qr QCResult
-		switch qc[3] {
+		switch p.String(2) {
 		case "Y":
 			qr = QCYes
 		case "N":
@@ -1041,27 +937,15 @@ func testDerived() {
 		case "M":
 			qr = QCMaybe
 		default:
-			log.Fatalf(`Unexpected quick check value "%s"`, qc[3])
+			log.Fatalf(`Unexpected quick check value "%s"`, p.String(2))
 		}
-		var lastFailed bool
-		// Verify current
-		for ; i <= j; i++ {
-			c := &chars[int(i)]
-			c.forms[ftype].verified[mode] = true
-			curqr := c.forms[ftype].quickCheck[mode]
-			if curqr != qr {
-				if !lastFailed {
-					logger.Printf("%s: %.4X..%.4X -- %s\n",
-						qt, int(i), int(j), line[0:50])
-				}
-				logger.Printf("%U: FAILED %s (was %v need %v)\n",
-					int(i), qt, curqr, qr)
-				lastFailed = true
-			}
+		if got := c.forms[ftype].quickCheck[mode]; got != qr {
+			logger.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr)
 		}
+		c.forms[ftype].verified[mode] = true
 	}
-	if scanner.Err() != nil {
-		logger.Fatal(scanner.Err())
+	if err := p.Err(); err != nil {
+		logger.Fatal(err)
 	}
 	// Any unspecified value must be QCYes. Verify this.
 	for i, c := range chars {

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2012 Dave Grijalva
+
+Permission is hereby granted, free of charge, to any person obtaining a 
+copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation 
+the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the 
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included 
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 
+THE SOFTWARE.

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/README.md

@@ -0,0 +1,19 @@
+A simple LFU cache for golang.  Based on the paper [An O(1) algorithm for implementing the LFU cache eviction scheme](http://dhruvbird.com/lfu.pdf).
+
+Usage:
+
+```go
+import "github.com/dgrijalva/lfu-go"
+
+// Make a new thing
+c := lfu.New()
+
+// Set some values
+c.Set("myKey", myValue)
+
+// Retrieve some values
+myValue = c.Get("myKey")
+
+// Evict some values
+c.Evict(1)
+```

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/lfu.go

@@ -0,0 +1,156 @@
+package lfu
+
+import (
+	"container/list"
+	"sync"
+)
+
+type Eviction struct {
+	Key   string
+	Value interface{}
+}
+
+type Cache struct {
+	// If len > UpperBound, cache will automatically evict
+	// down to LowerBound.  If either value is 0, this behavior
+	// is disabled.
+	UpperBound      int
+	LowerBound      int
+	values          map[string]*cacheEntry
+	freqs           *list.List
+	len             int
+	lock            *sync.Mutex
+	EvictionChannel chan<- Eviction
+}
+
+type cacheEntry struct {
+	key      string
+	value    interface{}
+	freqNode *list.Element
+}
+
+type listEntry struct {
+	entries map[*cacheEntry]byte
+	freq    int
+}
+
+func New() *Cache {
+	c := new(Cache)
+	c.values = make(map[string]*cacheEntry)
+	c.freqs = list.New()
+	c.lock = new(sync.Mutex)
+	return c
+}
+
+func (c *Cache) Get(key string) interface{} {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	if e, ok := c.values[key]; ok {
+		c.increment(e)
+		return e.value
+	}
+	return nil
+}
+
+func (c *Cache) Set(key string, value interface{}) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	if e, ok := c.values[key]; ok {
+		// value already exists for key.  overwrite
+		e.value = value
+		c.increment(e)
+	} else {
+		// value doesn't exist.  insert
+		e := new(cacheEntry)
+		e.key = key
+		e.value = value
+		c.values[key] = e
+		c.increment(e)
+		c.len++
+		// bounds mgmt
+		if c.UpperBound > 0 && c.LowerBound > 0 {
+			if c.len > c.UpperBound {
+				c.evict(c.len - c.LowerBound)
+			}
+		}
+	}
+}
+
+func (c *Cache) Len() int {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return c.len
+}
+
+func (c *Cache) Evict(count int) int {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return c.evict(count)
+}
+
+func (c *Cache) evict(count int) int {
+	// No lock here so it can be called
+	// from within the lock (during Set)
+	var evicted int
+	for i := 0; i < count; {
+		if place := c.freqs.Front(); place != nil {
+			for entry, _ := range place.Value.(*listEntry).entries {
+				if i < count {
+					if c.EvictionChannel != nil {
+						c.EvictionChannel <- Eviction{
+							Key:   entry.key,
+							Value: entry.value,
+						}
+					}
+					delete(c.values, entry.key)
+					c.remEntry(place, entry)
+					evicted++
+					c.len--
+					i++
+				}
+			}
+		}
+	}
+	return evicted
+}
+
+func (c *Cache) increment(e *cacheEntry) {
+	currentPlace := e.freqNode
+	var nextFreq int
+	var nextPlace *list.Element
+	if currentPlace == nil {
+		// new entry
+		nextFreq = 1
+		nextPlace = c.freqs.Front()
+	} else {
+		// move up
+		nextFreq = currentPlace.Value.(*listEntry).freq + 1
+		nextPlace = currentPlace.Next()
+	}
+
+	if nextPlace == nil || nextPlace.Value.(*listEntry).freq != nextFreq {
+		// create a new list entry
+		li := new(listEntry)
+		li.freq = nextFreq
+		li.entries = make(map[*cacheEntry]byte)
+		if currentPlace != nil {
+			nextPlace = c.freqs.InsertAfter(li, currentPlace)
+		} else {
+			nextPlace = c.freqs.PushFront(li)
+		}
+	}
+	e.freqNode = nextPlace
+	nextPlace.Value.(*listEntry).entries[e] = 1
+	if currentPlace != nil {
+		// remove from current position
+		c.remEntry(currentPlace, e)
+	}
+}
+
+func (c *Cache) remEntry(place *list.Element, entry *cacheEntry) {
+	entries := place.Value.(*listEntry).entries
+	delete(entries, entry)
+	if len(entries) == 0 {
+		c.freqs.Remove(place)
+	}
+}

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/lfu_test.go

@@ -0,0 +1,68 @@
+package lfu
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestLFU(t *testing.T) {
+	c := New()
+	c.Set("a", "a")
+	if v := c.Get("a"); v != "a" {
+		t.Errorf("Value was not saved: %v != 'a'", v)
+	}
+	if l := c.Len(); l != 1 {
+		t.Errorf("Length was not updated: %v != 1", l)
+	}
+
+	c.Set("b", "b")
+	if v := c.Get("b"); v != "b" {
+		t.Errorf("Value was not saved: %v != 'b'", v)
+	}
+	if l := c.Len(); l != 2 {
+		t.Errorf("Length was not updated: %v != 2", l)
+	}
+
+	c.Get("a")
+	evicted := c.Evict(1)
+	if v := c.Get("a"); v != "a" {
+		t.Errorf("Value was improperly evicted: %v != 'a'", v)
+	}
+	if v := c.Get("b"); v != nil {
+		t.Errorf("Value was not evicted: %v", v)
+	}
+	if l := c.Len(); l != 1 {
+		t.Errorf("Length was not updated: %v != 1", l)
+	}
+	if evicted != 1 {
+		t.Errorf("Number of evicted items is wrong: %v != 1", evicted)
+	}
+}
+
+func TestBoundsMgmt(t *testing.T) {
+	c := New()
+	c.UpperBound = 10
+	c.LowerBound = 5
+
+	for i := 0; i < 100; i++ {
+		c.Set(fmt.Sprintf("%v", i), i)
+	}
+	if c.Len() > 10 {
+		t.Errorf("Bounds management failed to evict properly: %v", c.Len())
+	}
+}
+
+func TestEviction(t *testing.T) {
+	ch := make(chan Eviction, 1)
+
+	c := New()
+	c.EvictionChannel = ch
+	c.Set("a", "b")
+	c.Evict(1)
+
+	ev := <-ch
+
+	if ev.Key != "a" || ev.Value.(string) != "b" {
+		t.Error("Incorrect item")
+	}
+}

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/.gitignore

@@ -0,0 +1 @@
+/lz4-example/lz4-example

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/.travis.yml

@@ -0,0 +1,7 @@
+language: go
+
+go:
+  - 1.1
+  - 1.2
+  - 1.3
+  - tip

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/LICENSE

@@ -0,0 +1,24 @@
+Copyright 2011-2012 Branimir Karadzic. All rights reserved.
+Copyright 2013 Damian Gryski. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice, this
+      list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
+

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/README.md

@@ -0,0 +1,71 @@
+go-lz4
+======
+
+go-lz4 is port of LZ4 lossless compression algorithm to Go. The original C code
+is located at:
+
+https://code.google.com/p/lz4/
+
+Status
+------
+[![Build Status](https://secure.travis-ci.org/bkaradzic/go-lz4.png)](http://travis-ci.org/bkaradzic/go-lz4)  
+[![GoDoc](https://godoc.org/github.com/bkaradzic/go-lz4?status.png)](https://godoc.org/github.com/bkaradzic/go-lz4)
+
+Usage
+-----
+
+    go get github.com/bkaradzic/go-lz4
+
+    import "github.com/bkaradzic/go-lz4"
+
+The package name is `lz4`
+
+Notes
+-----
+
+* go-lz4 saves a uint32 with the original uncompressed length at the beginning
+  of the encoded buffer.  They may get in the way of interoperability with
+  other implementations.
+
+Contributors
+------------
+
+Damian Gryski ([@dgryski](https://github.com/dgryski))  
+Dustin Sallings ([@dustin](https://github.com/dustin))
+
+Contact
+-------
+
+[@bkaradzic](https://twitter.com/bkaradzic)  
+http://www.stuckingeometry.com
+
+Project page  
+https://github.com/bkaradzic/go-lz4
+
+License
+-------
+
+Copyright 2011-2012 Branimir Karadzic. All rights reserved.  
+Copyright 2013 Damian Gryski. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+   1. Redistributions of source code must retain the above copyright notice, this
+      list of conditions and the following disclaimer.
+
+   2. Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
+IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
+

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/fuzzer/main.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"math/rand"
+
+	"github.com/bkaradzic/go-lz4"
+
+	// lz4's API matches snappy's, so we can easily see how it performs
+	// lz4 "code.google.com/p/snappy-go/snappy"
+)
+
+var input = `
+ADVENTURE I. A SCANDAL IN BOHEMIA
+
+I.
+
+To Sherlock Holmes she is always THE woman. I have seldom heard
+him mention her under any other name. In his eyes she eclipses
+and predominates the whole of her sex. It was not that he felt
+any emotion akin to love for Irene Adler. All emotions, and that
+one particularly, were abhorrent to his cold, precise but
+admirably balanced mind. He was, I take it, the most perfect
+reasoning and observing machine that the world has seen, but as a
+lover he would have placed himself in a false position. He never
+spoke of the softer passions, save with a gibe and a sneer. They
+were admirable things for the observer--excellent for drawing the
+veil from men's motives and actions. But for the trained reasoner
+to admit such intrusions into his own delicate and finely
+adjusted temperament was to introduce a distracting factor which
+might throw a doubt upon all his mental results. Grit in a
+sensitive instrument, or a crack in one of his own high-power
+lenses, would not be more disturbing than a strong emotion in a
+nature such as his. And yet there was but one woman to him, and
+that woman was the late Irene Adler, of dubious and questionable
+memory.
+
+I had seen little of Holmes lately. My marriage had drifted us
+away from each other. My own complete happiness, and the
+home-centred interests which rise up around the man who first
+finds himself master of his own establishment, were sufficient to
+absorb all my attention, while Holmes, who loathed every form of
+society with his whole Bohemian soul, remained in our lodgings in
+Baker Street, buried among his old books, and alternating from
+week to week between cocaine and ambition, the drowsiness of the
+drug, and the fierce energy of his own keen nature. He was still,
+as ever, deeply attracted by the study of crime, and occupied his
+immense faculties and extraordinary powers of observation in
+following out those clues, and clearing up those mysteries which
+had been abandoned as hopeless by the official police. From time
+to time I heard some vague account of his doings: of his summons
+to Odessa in the case of the Trepoff murder, of his clearing up
+of the singular tragedy of the Atkinson brothers at Trincomalee,
+and finally of the mission which he had accomplished so
+delicately and successfully for the reigning family of Holland.
+Beyond these signs of his activity, however, which I merely
+shared with all the readers of the daily press, I knew little of
+my former friend and companion.
+`
+
+func main() {
+
+	compressed, _ := lz4.Encode(nil, []byte(input))
+
+	modified := make([]byte, len(compressed))
+
+	for {
+		copy(modified, compressed)
+		for i := 0; i < 100; i++ {
+			modified[rand.Intn(len(compressed)-4)+4] = byte(rand.Intn(256))
+		}
+		lz4.Decode(nil, modified)
+	}
+
+}

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/lz4-example/main.go

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2011 Branimir Karadzic. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice, this
+ *       list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above copyright notice,
+ *       this list of conditions and the following disclaimer in the documentation
+ *       and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"runtime/pprof"
+
+	lz4 "github.com/bkaradzic/go-lz4"
+)
+
+var (
+	decompress = flag.Bool("d", false, "decompress")
+)
+
+func main() {
+
+	var optCPUProfile = flag.String("cpuprofile", "", "profile")
+	flag.Parse()
+
+	if *optCPUProfile != "" {
+		f, err := os.Create(*optCPUProfile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		pprof.StartCPUProfile(f)
+		defer pprof.StopCPUProfile()
+	}
+
+	args := flag.Args()
+
+	var data []byte
+
+	if len(args) < 2 {
+		fmt.Print("Usage: lz4 [-d] <input> <output>\n")
+		os.Exit(1)
+	}
+
+	input, err := os.OpenFile(args[0], os.O_RDONLY, 0644)
+	if err != nil {
+		fmt.Printf("Failed to open input file %s\n", args[0])
+		os.Exit(1)
+	}
+	defer input.Close()
+
+	if *decompress {
+		data, _ = ioutil.ReadAll(input)
+		data, err = lz4.Decode(nil, data)
+		if err != nil {
+			fmt.Println("Failed to decode:", err)
+			return
+		}
+	} else {
+		data, _ = ioutil.ReadAll(input)
+		data, err = lz4.Encode(nil, data)
+		if err != nil {
+			fmt.Println("Failed to encode:", err)
+			return
+		}
+	}
+
+	err = ioutil.WriteFile(args[1], data, 0644)
+	if err != nil {
+		fmt.Printf("Failed to open output file %s\n", args[1])
+		os.Exit(1)
+	}
+}

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/lz4_test.go

@@ -0,0 +1,63 @@
+package lz4
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+)
+
+var testfile, _ = ioutil.ReadFile("testdata/pg1661.txt")
+
+func roundtrip(t *testing.T, input []byte) {
+
+	dst, err := Encode(nil, input)
+	if err != nil {
+		t.Errorf("got error during compression: %s", err)
+	}
+
+	output, err := Decode(nil, dst)
+
+	if err != nil {
+		t.Errorf("got error during decompress: %s", err)
+	}
+
+	if !bytes.Equal(output, input) {
+		t.Errorf("roundtrip failed")
+	}
+}
+
+func TestEmpty(t *testing.T) {
+	roundtrip(t, nil)
+}
+
+func TestLengths(t *testing.T) {
+
+	for i := 0; i < 1024; i++ {
+		roundtrip(t, testfile[:i])
+	}
+
+	for i := 1024; i < 4096; i += 23 {
+		roundtrip(t, testfile[:i])
+	}
+}
+
+func TestWords(t *testing.T) {
+	roundtrip(t, testfile)
+}
+
+func BenchmarkLZ4Encode(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		Encode(nil, testfile)
+	}
+}
+
+func BenchmarkLZ4Decode(b *testing.B) {
+
+	var compressed, _ = Encode(nil, testfile)
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		Decode(nil, compressed)
+	}
+}

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/reader.go

@@ -0,0 +1,194 @@
+/*
+ * Copyright 2011-2012 Branimir Karadzic. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice, this
+ *       list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above copyright notice,
+ *       this list of conditions and the following disclaimer in the documentation
+ *       and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package lz4
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+)
+
+var (
+	// ErrCorrupt indicates the input was corrupt
+	ErrCorrupt = errors.New("corrupt input")
+)
+
+const (
+	mlBits  = 4
+	mlMask  = (1 << mlBits) - 1
+	runBits = 8 - mlBits
+	runMask = (1 << runBits) - 1
+)
+
+type decoder struct {
+	src  []byte
+	dst  []byte
+	spos uint32
+	dpos uint32
+	ref  uint32
+}
+
+func (d *decoder) readByte() (uint8, error) {
+	if int(d.spos) == len(d.src) {
+		return 0, io.EOF
+	}
+	b := d.src[d.spos]
+	d.spos++
+	return b, nil
+}
+
+func (d *decoder) getLen() (uint32, error) {
+
+	length := uint32(0)
+	ln, err := d.readByte()
+	if err != nil {
+		return 0, ErrCorrupt
+	}
+	for ln == 255 {
+		length += 255
+		ln, err = d.readByte()
+		if err != nil {
+			return 0, ErrCorrupt
+		}
+	}
+	length += uint32(ln)
+
+	return length, nil
+}
+
+func (d *decoder) cp(length, decr uint32) {
+
+	if int(d.ref+length) < int(d.dpos) {
+		copy(d.dst[d.dpos:], d.dst[d.ref:d.ref+length])
+	} else {
+		for ii := uint32(0); ii < length; ii++ {
+			d.dst[d.dpos+ii] = d.dst[d.ref+ii]
+		}
+	}
+	d.dpos += length
+	d.ref += length - decr
+}
+
+func (d *decoder) finish(err error) error {
+	if err == io.EOF {
+		return nil
+	}
+
+	return err
+}
+
+// Decode returns the decoded form of src.  The returned slice may be a
+// subslice of dst if it was large enough to hold the entire decoded block.
+func Decode(dst, src []byte) ([]byte, error) {
+
+	if len(src) < 4 {
+		return nil, ErrCorrupt
+	}
+
+	uncompressedLen := binary.LittleEndian.Uint32(src)
+
+	if uncompressedLen == 0 {
+		return nil, nil
+	}
+
+	if uncompressedLen > MaxInputSize {
+		return nil, ErrTooLarge
+	}
+
+	if dst == nil || len(dst) < int(uncompressedLen) {
+		dst = make([]byte, uncompressedLen)
+	}
+
+	d := decoder{src: src, dst: dst[:uncompressedLen], spos: 4}
+
+	decr := []uint32{0, 3, 2, 3}
+
+	for {
+		code, err := d.readByte()
+		if err != nil {
+			return d.dst, d.finish(err)
+		}
+
+		length := uint32(code >> mlBits)
+		if length == runMask {
+			ln, err := d.getLen()
+			if err != nil {
+				return nil, ErrCorrupt
+			}
+			length += ln
+		}
+
+		if int(d.spos+length) > len(d.src) {
+			return nil, ErrCorrupt
+		}
+
+		for ii := uint32(0); ii < length; ii++ {
+			d.dst[d.dpos+ii] = d.src[d.spos+ii]
+		}
+
+		d.spos += length
+		d.dpos += length
+
+		if int(d.spos) == len(d.src) {
+			return d.dst, nil
+		}
+
+		if int(d.spos+2) >= len(d.src) {
+			return nil, ErrCorrupt
+		}
+
+		back := uint32(d.src[d.spos]) | uint32(d.src[d.spos+1])<<8
+
+		if back > d.dpos {
+			return nil, ErrCorrupt
+		}
+
+		d.spos += 2
+		d.ref = d.dpos - back
+
+		length = uint32(code & mlMask)
+		if length == mlMask {
+			ln, err := d.getLen()
+			if err != nil {
+				return nil, ErrCorrupt
+			}
+			length += ln
+		}
+
+		literal := d.dpos - d.ref
+		if literal < 4 {
+			d.cp(4, decr[literal])
+		} else {
+			length += 4
+		}
+
+		if d.dpos+length > uncompressedLen {
+			return nil, ErrCorrupt
+		}
+
+		d.cp(length, 0)
+	}
+}

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/writer.go

@@ -0,0 +1,188 @@
+/*
+ * Copyright 2011-2012 Branimir Karadzic. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice, this
+ *       list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above copyright notice,
+ *       this list of conditions and the following disclaimer in the documentation
+ *       and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package lz4
+
+import "encoding/binary"
+import "errors"
+
+const (
+	minMatch              = 4
+	hashLog               = 17
+	hashTableSize         = 1 << hashLog
+	hashShift             = (minMatch * 8) - hashLog
+	incompressible uint32 = 128
+	uninitHash            = 0x88888888
+
+	// MaxInputSize is the largest buffer than can be compressed in a single block
+	MaxInputSize = 0x7E000000
+)
+
+var (
+	// ErrTooLarge indicates the input buffer was too large
+	ErrTooLarge = errors.New("input too large")
+)
+
+type encoder struct {
+	src       []byte
+	dst       []byte
+	hashTable []uint32
+	pos       uint32
+	anchor    uint32
+	dpos      uint32
+}
+
+// CompressBound returns the maximum length of a lz4 block, given it's uncompressed length
+func CompressBound(isize int) int {
+	if isize > MaxInputSize {
+		return 0
+	}
+	return isize + ((isize) / 255) + 16 + 4
+}
+
+func (e *encoder) writeLiterals(length, mlLen, pos uint32) {
+
+	ln := length
+
+	var code byte
+	if ln > runMask-1 {
+		code = runMask
+	} else {
+		code = byte(ln)
+	}
+
+	if mlLen > mlMask-1 {
+		e.dst[e.dpos] = (code << mlBits) + byte(mlMask)
+	} else {
+		e.dst[e.dpos] = (code << mlBits) + byte(mlLen)
+	}
+	e.dpos++
+
+	if code == runMask {
+		ln -= runMask
+		for ; ln > 254; ln -= 255 {
+			e.dst[e.dpos] = 255
+			e.dpos++
+		}
+
+		e.dst[e.dpos] = byte(ln)
+		e.dpos++
+	}
+
+	for ii := uint32(0); ii < length; ii++ {
+		e.dst[e.dpos+ii] = e.src[pos+ii]
+	}
+
+	e.dpos += length
+}
+
+// Encode returns the encoded form of src.  The returned array may be a
+// sub-slice of dst if it was large enough to hold the entire output.
+func Encode(dst, src []byte) ([]byte, error) {
+
+	if len(src) >= MaxInputSize {
+		return nil, ErrTooLarge
+	}
+
+	if n := CompressBound(len(src)); len(dst) < n {
+		dst = make([]byte, n)
+	}
+
+	e := encoder{src: src, dst: dst, hashTable: make([]uint32, hashTableSize)}
+
+	binary.LittleEndian.PutUint32(dst, uint32(len(src)))
+	e.dpos = 4
+
+	var (
+		step  uint32 = 1
+		limit        = incompressible
+	)
+
+	for {
+		if int(e.pos)+12 >= len(e.src) {
+			e.writeLiterals(uint32(len(e.src))-e.anchor, 0, e.anchor)
+			return e.dst[:e.dpos], nil
+		}
+
+		sequence := uint32(e.src[e.pos+3])<<24 | uint32(e.src[e.pos+2])<<16 | uint32(e.src[e.pos+1])<<8 | uint32(e.src[e.pos+0])
+
+		hash := (sequence * 2654435761) >> hashShift
+		ref := e.hashTable[hash] + uninitHash
+		e.hashTable[hash] = e.pos - uninitHash
+
+		if ((e.pos-ref)>>16) != 0 || uint32(e.src[ref+3])<<24|uint32(e.src[ref+2])<<16|uint32(e.src[ref+1])<<8|uint32(e.src[ref+0]) != sequence {
+			if e.pos-e.anchor > limit {
+				limit <<= 1
+				step += 1 + (step >> 2)
+			}
+			e.pos += step
+			continue
+		}
+
+		if step > 1 {
+			e.hashTable[hash] = ref - uninitHash
+			e.pos -= step - 1
+			step = 1
+			continue
+		}
+		limit = incompressible
+
+		ln := e.pos - e.anchor
+		back := e.pos - ref
+
+		anchor := e.anchor
+
+		e.pos += minMatch
+		ref += minMatch
+		e.anchor = e.pos
+
+		for int(e.pos) < len(e.src)-5 && e.src[e.pos] == e.src[ref] {
+			e.pos++
+			ref++
+		}
+
+		mlLen := e.pos - e.anchor
+
+		e.writeLiterals(ln, mlLen, anchor)
+		e.dst[e.dpos] = uint8(back)
+		e.dst[e.dpos+1] = uint8(back >> 8)
+		e.dpos += 2
+
+		if mlLen > mlMask-1 {
+			mlLen -= mlMask
+			for mlLen > 254 {
+				mlLen -= 255
+
+				e.dst[e.dpos] = 255
+				e.dpos++
+			}
+
+			e.dst[e.dpos] = byte(mlLen)
+			e.dpos++
+		}
+
+		e.anchor = e.pos
+	}
+}

Godeps/_workspace/src/github.com/calmh/osext/osext_procfs.go

@@ -2,12 +2,13 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build linux netbsd openbsd
+// +build linux netbsd openbsd solaris
 
 package osext
 
 import (
 	"errors"
+	"fmt"
 	"os"
 	"runtime"
 )
@@ -20,6 +21,8 @@ func executable() (string, error) {
 		return os.Readlink("/proc/curproc/exe")
 	case "openbsd":
 		return os.Readlink("/proc/curproc/file")
+	case "solaris":
+		return os.Readlink(fmt.Sprintf("/proc/%d/path/a.out", os.Getpid()))
 	}
 	return "", errors.New("ExecPath not implemented for " + runtime.GOOS)
 }

Godeps/_workspace/src/github.com/calmh/xdr/.gitignore

@@ -0,0 +1 @@
+coverage.out

Godeps/_workspace/src/github.com/calmh/xdr/.travis.yml

@@ -0,0 +1,19 @@
+language: go
+go:
+- tip
+
+install:
+- export PATH=$PATH:$HOME/gopath/bin
+- go get code.google.com/p/go.tools/cmd/cover
+- go get github.com/mattn/goveralls
+
+script:
+- ./generate.sh
+- go test -coverprofile=coverage.out
+
+after_success:
+- goveralls -coverprofile=coverage.out -service=travis-ci -package=calmh/xdr -repotoken="$COVERALLS_TOKEN"
+
+env:
+  global:
+    secure: SmgnrGfp2zLrA44ChRMpjPeujubt9veZ8Fx/OseMWECmacyV5N/TuDhzIbwo6QwV4xB0sBacoPzvxQbJRVjNKsPiSu72UbcQmQ7flN4Tf7nW09tSh1iW8NgrpBCq/3UYLoBu2iPBEBKm93IK0aGNAKs6oEkB0fU27iTVBwiTXOY=

Godeps/_workspace/src/github.com/calmh/xdr/LICENSE

@@ -1,21 +1,19 @@
-The MIT License (MIT)
-
-Copyright (c) 2011-2014 Twitter, Inc
+Copyright (C) 2014 Jakob Borg.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
+of this software and associated documentation files (the "Software"), to
+deal in the Software without restriction, including without limitation the
+rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+sell copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
+- The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+IN THE SOFTWARE.

Godeps/_workspace/src/github.com/calmh/xdr/README.md

@@ -0,0 +1,12 @@
+xdr
+===
+
+[![Build Status](https://img.shields.io/travis/calmh/xdr.svg?style=flat)](https://travis-ci.org/calmh/xdr)
+[![Coverage Status](https://img.shields.io/coveralls/calmh/xdr.svg?style=flat)](https://coveralls.io/r/calmh/xdr?branch=master)
+[![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat)](http://godoc.org/github.com/calmh/xdr)
+[![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://opensource.org/licenses/MIT)
+
+This is an XDR encoding/decoding library. It uses code generation and
+not reflection. It supports the IPDR bastardized XDR format when built
+with `-tags ipdr`.
+

Godeps/_workspace/src/github.com/calmh/xdr/bench_test.go

@@ -1,6 +1,5 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
 
 package xdr_test
 
@@ -8,12 +7,15 @@ import (
 	"io"
 	"io/ioutil"
 	"testing"
+
+	"github.com/calmh/xdr"
 )
 
 type XDRBenchStruct struct {
 	I1  uint64
 	I2  uint32
 	I3  uint16
+	I4  uint8
 	Bs0 []byte // max:128
 	Bs1 []byte
 	S0  string // max:128
@@ -25,6 +27,7 @@ var s = XDRBenchStruct{
 	I1:  42,
 	I2:  43,
 	I3:  44,
+	I4:  45,
 	Bs0: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18},
 	Bs1: []byte{11, 12, 13, 14, 15, 16, 17, 18, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
 	S0:  "Hello World! String one.",
@@ -57,6 +60,16 @@ func BenchmarkThisEncode(b *testing.B) {
 	}
 }
 
+func BenchmarkThisEncoder(b *testing.B) {
+	w := xdr.NewWriter(ioutil.Discard)
+	for i := 0; i < b.N; i++ {
+		_, err := s.encodeXDR(w)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 type repeatReader struct {
 	data []byte
 }
@@ -85,3 +98,16 @@ func BenchmarkThisDecode(b *testing.B) {
 		rr.Reset(e)
 	}
 }
+
+func BenchmarkThisDecoder(b *testing.B) {
+	rr := &repeatReader{e}
+	r := xdr.NewReader(rr)
+	var t XDRBenchStruct
+	for i := 0; i < b.N; i++ {
+		err := t.decodeXDR(r)
+		if err != nil {
+			b.Fatal(err)
+		}
+		rr.Reset(e)
+	}
+}

Godeps/_workspace/src/github.com/calmh/xdr/bench_xdr_test.go

@@ -0,0 +1,183 @@
+// ************************************************************
+// This file is automatically generated by genxdr. Do not edit.
+// ************************************************************
+
+package xdr_test
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/calmh/xdr"
+)
+
+/*
+
+XDRBenchStruct Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                         I1 (64 bits)                          +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                              I2                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|            0x0000             |              I3               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             uint8                             |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of Bs0                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     Bs0 (variable length)                     \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of Bs1                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     Bs1 (variable length)                     \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of S0                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     S0 (variable length)                      \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of S1                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     S1 (variable length)                      \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct XDRBenchStruct {
+	unsigned hyper I1;
+	unsigned int I2;
+	unsigned int I3;
+	uint8 I4;
+	opaque Bs0<128>;
+	opaque Bs1<>;
+	string S0<128>;
+	string S1<>;
+}
+
+*/
+
+func (o XDRBenchStruct) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o XDRBenchStruct) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o XDRBenchStruct) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o XDRBenchStruct) encodeXDR(xw *xdr.Writer) (int, error) {
+	xw.WriteUint64(o.I1)
+	xw.WriteUint32(o.I2)
+	xw.WriteUint16(o.I3)
+	xw.WriteUint8(o.I4)
+	if len(o.Bs0) > 128 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteBytes(o.Bs0)
+	xw.WriteBytes(o.Bs1)
+	if len(o.S0) > 128 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteString(o.S0)
+	xw.WriteString(o.S1)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *XDRBenchStruct) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *XDRBenchStruct) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *XDRBenchStruct) decodeXDR(xr *xdr.Reader) error {
+	o.I1 = xr.ReadUint64()
+	o.I2 = xr.ReadUint32()
+	o.I3 = xr.ReadUint16()
+	o.I4 = xr.ReadUint8()
+	o.Bs0 = xr.ReadBytesMax(128)
+	o.Bs1 = xr.ReadBytes()
+	o.S0 = xr.ReadStringMax(128)
+	o.S1 = xr.ReadString()
+	return xr.Error()
+}
+
+/*
+
+repeatReader Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                        Length of data                         |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                    data (variable length)                     \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct repeatReader {
+	opaque data<>;
+}
+
+*/
+
+func (o repeatReader) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o repeatReader) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o repeatReader) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o repeatReader) encodeXDR(xw *xdr.Writer) (int, error) {
+	xw.WriteBytes(o.data)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *repeatReader) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *repeatReader) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *repeatReader) decodeXDR(xr *xdr.Reader) error {
+	o.data = xr.ReadBytes()
+	return xr.Error()
+}

Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go

@@ -1,6 +1,5 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
 
 package main
 
@@ -34,11 +33,7 @@ type structInfo struct {
 	Fields []fieldInfo
 }
 
-var headerTpl = template.Must(template.New("header").Parse(`// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
-
-// ************************************************************
+var headerTpl = template.Must(template.New("header").Parse(`// ************************************************************
 // This file is automatically generated by genxdr. Do not edit.
 // ************************************************************
 
@@ -48,7 +43,7 @@ import (
 	"bytes"
 	"io"
 
-	"github.com/calmh/syncthing/xdr"
+	"github.com/calmh/xdr"
 )
 `))
 
@@ -82,7 +77,10 @@ func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
 				{{end}}
 				xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}})
 			{{else}}
-				o.{{$fieldInfo.Name}}.encodeXDR(xw)
+				_, err := o.{{$fieldInfo.Name}}.encodeXDR(xw)
+				if err != nil {
+					return xw.Tot(), err
+				}
 			{{end}}
 		{{else}}
 			{{if ge $fieldInfo.Max 1}}
@@ -97,7 +95,10 @@ func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
 			{{else if $fieldInfo.IsBasic}}
 				xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}}[i])
 			{{else}}
-				o.{{$fieldInfo.Name}}[i].encodeXDR(xw)
+				_, err := o.{{$fieldInfo.Name}}[i].encodeXDR(xw)
+				if err != nil {
+					return xw.Tot(), err
+				}
 			{{end}}
 			}
 		{{end}}
@@ -160,6 +161,8 @@ type typeSet struct {
 }
 
 var xdrEncoders = map[string]typeSet{
+	"int8":   typeSet{"uint8", "Uint8"},
+	"uint8":  typeSet{"", "Uint8"},
 	"int16":  typeSet{"uint16", "Uint16"},
 	"uint16": typeSet{"", "Uint16"},
 	"int32":  typeSet{"uint32", "Uint32"},

Godeps/_workspace/src/github.com/calmh/xdr/debug.go

@@ -0,0 +1,16 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+package xdr
+
+import (
+	"log"
+	"os"
+)
+
+var (
+	debug = len(os.Getenv("XDRTRACE")) > 0
+	dl    = log.New(os.Stdout, "xdr: ", log.Lshortfile|log.Ltime|log.Lmicroseconds)
+)
+
+const maxDebugBytes = 32

Godeps/_workspace/src/github.com/calmh/xdr/doc.go

@@ -0,0 +1,5 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+// Package xdr implements an XDR (RFC 4506) encoder/decoder.
+package xdr

Godeps/_workspace/src/github.com/calmh/xdr/encdec_test.go

@@ -1,18 +1,23 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
 
 package xdr_test
 
 import (
 	"bytes"
+	"math/rand"
+	"reflect"
 	"testing"
 	"testing/quick"
+
+	"github.com/calmh/xdr"
 )
 
 // Contains all supported types
 type TestStruct struct {
 	I    int
+	I8   int8
+	UI8  uint8
 	I16  int16
 	UI16 uint16
 	I32  int32
@@ -21,6 +26,25 @@ type TestStruct struct {
 	UI64 uint64
 	BS   []byte
 	S    string
+	C    Opaque
+}
+
+type Opaque [32]byte
+
+func (u *Opaque) encodeXDR(w *xdr.Writer) (int, error) {
+	return w.WriteRaw(u[:])
+}
+
+func (u *Opaque) decodeXDR(r *xdr.Reader) (int, error) {
+	return r.ReadRaw(u[:])
+}
+
+func (Opaque) Generate(rand *rand.Rand, size int) reflect.Value {
+	var u Opaque
+	for i := range u[:] {
+		u[i] = byte(rand.Int())
+	}
+	return reflect.ValueOf(u)
 }
 
 func TestEncDec(t *testing.T) {
@@ -38,7 +62,7 @@ func TestEncDec(t *testing.T) {
 			t0.I32 != t1.I32 || t0.UI32 != t1.UI32 ||
 			t0.I64 != t1.I64 || t0.UI64 != t1.UI64 ||
 			bytes.Compare(t0.BS, t1.BS) != 0 ||
-			t0.S != t1.S {
+			t0.S != t1.S || t0.C != t1.C {
 			t.Logf("%#v", t0)
 			t.Logf("%#v", t1)
 			return false

Godeps/_workspace/src/github.com/calmh/xdr/encdec_xdr_test.go

@@ -0,0 +1,136 @@
+// ************************************************************
+// This file is automatically generated by genxdr. Do not edit.
+// ************************************************************
+
+package xdr_test
+
+import (
+	"bytes"
+	"io"
+
+	"github.com/calmh/xdr"
+)
+
+/*
+
+TestStruct Structure:
+
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                              int                              |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             int8                              |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             uint8                             |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             int16                             |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|            0x0000             |             UI16              |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             int32                             |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             UI32                              |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                         I64 (64 bits)                         +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                        UI64 (64 bits)                         +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of BS                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     BS (variable length)                      \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                          Length of S                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                      S (variable length)                      \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                            Opaque                             |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+struct TestStruct {
+	int I;
+	int8 I8;
+	uint8 UI8;
+	int16 I16;
+	unsigned int UI16;
+	int32 I32;
+	unsigned int UI32;
+	hyper I64;
+	unsigned hyper UI64;
+	opaque BS<>;
+	string S<>;
+	Opaque C;
+}
+
+*/
+
+func (o TestStruct) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o TestStruct) MarshalXDR() []byte {
+	return o.AppendXDR(make([]byte, 0, 128))
+}
+
+func (o TestStruct) AppendXDR(bs []byte) []byte {
+	var aw = xdr.AppendWriter(bs)
+	var xw = xdr.NewWriter(&aw)
+	o.encodeXDR(xw)
+	return []byte(aw)
+}
+
+func (o TestStruct) encodeXDR(xw *xdr.Writer) (int, error) {
+	xw.WriteUint64(uint64(o.I))
+	xw.WriteUint8(uint8(o.I8))
+	xw.WriteUint8(o.UI8)
+	xw.WriteUint16(uint16(o.I16))
+	xw.WriteUint16(o.UI16)
+	xw.WriteUint32(uint32(o.I32))
+	xw.WriteUint32(o.UI32)
+	xw.WriteUint64(uint64(o.I64))
+	xw.WriteUint64(o.UI64)
+	xw.WriteBytes(o.BS)
+	xw.WriteString(o.S)
+	_, err := o.C.encodeXDR(xw)
+	if err != nil {
+		return xw.Tot(), err
+	}
+	return xw.Tot(), xw.Error()
+}
+
+func (o *TestStruct) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *TestStruct) UnmarshalXDR(bs []byte) error {
+	var br = bytes.NewReader(bs)
+	var xr = xdr.NewReader(br)
+	return o.decodeXDR(xr)
+}
+
+func (o *TestStruct) decodeXDR(xr *xdr.Reader) error {
+	o.I = int(xr.ReadUint64())
+	o.I8 = int8(xr.ReadUint8())
+	o.UI8 = xr.ReadUint8()
+	o.I16 = int16(xr.ReadUint16())
+	o.UI16 = xr.ReadUint16()
+	o.I32 = int32(xr.ReadUint32())
+	o.UI32 = xr.ReadUint32()
+	o.I64 = int64(xr.ReadUint64())
+	o.UI64 = xr.ReadUint64()
+	o.BS = xr.ReadBytes()
+	o.S = xr.ReadString()
+	(&o.C).decodeXDR(xr)
+	return xr.Error()
+}

Godeps/_workspace/src/github.com/calmh/xdr/generate.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+go run cmd/genxdr/main.go -- bench_test.go > bench_xdr_test.go
+go run cmd/genxdr/main.go -- encdec_test.go > encdec_xdr_test.go

Godeps/_workspace/src/github.com/calmh/xdr/pad_ipdr.go

@@ -0,0 +1,10 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+// +build ipdr
+
+package xdr
+
+func pad(l int) int {
+	return 0
+}

Godeps/_workspace/src/github.com/calmh/xdr/pad_xdr.go

@@ -0,0 +1,14 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+// +build !ipdr
+
+package xdr
+
+func pad(l int) int {
+	d := l % 4
+	if d == 0 {
+		return 0
+	}
+	return 4 - d
+}

Godeps/_workspace/src/github.com/calmh/xdr/reader.go

@@ -7,18 +7,16 @@ package xdr
 import (
 	"errors"
 	"io"
-	"time"
+	"reflect"
+	"unsafe"
 )
 
 var ErrElementSizeExceeded = errors.New("element size exceeded")
 
 type Reader struct {
-	r    io.Reader
-	tot  int
-	err  error
-	b    [8]byte
-	sb   []byte
-	last time.Time
+	r   io.Reader
+	err error
+	b   [8]byte
 }
 
 func NewReader(r io.Reader) *Reader {
@@ -27,24 +25,28 @@ func NewReader(r io.Reader) *Reader {
 	}
 }
 
-func (r *Reader) ReadString() string {
-	if r.sb == nil {
-		r.sb = make([]byte, 64)
-	} else {
-		r.sb = r.sb[:cap(r.sb)]
+func (r *Reader) ReadRaw(bs []byte) (int, error) {
+	if r.err != nil {
+		return 0, r.err
 	}
-	r.sb = r.ReadBytesInto(r.sb)
-	return string(r.sb)
+
+	var n int
+	n, r.err = io.ReadFull(r.r, bs)
+	return n, r.err
+}
+
+func (r *Reader) ReadString() string {
+	return r.ReadStringMax(0)
 }
 
 func (r *Reader) ReadStringMax(max int) string {
-	if r.sb == nil {
-		r.sb = make([]byte, 64)
-	} else {
-		r.sb = r.sb[:cap(r.sb)]
+	buf := r.ReadBytesMaxInto(max, nil)
+	bh := (*reflect.SliceHeader)(unsafe.Pointer(&buf))
+	sh := reflect.StringHeader{
+		Data: bh.Data,
+		Len:  bh.Len,
 	}
-	r.sb = r.ReadBytesMaxInto(max, r.sb)
-	return string(r.sb)
+	return *((*string)(unsafe.Pointer(&sh)))
 }
 
 func (r *Reader) ReadBytes() []byte {
@@ -63,8 +65,6 @@ func (r *Reader) ReadBytesMaxInto(max int, dst []byte) []byte {
 	if r.err != nil {
 		return nil
 	}
-	r.last = time.Now()
-	s := r.tot
 
 	l := int(r.ReadUint32())
 	if r.err != nil {
@@ -75,53 +75,44 @@ func (r *Reader) ReadBytesMaxInto(max int, dst []byte) []byte {
 		return nil
 	}
 
-	if l+pad(l) > len(dst) {
-		dst = make([]byte, l+pad(l))
+	if fullLen := l + pad(l); fullLen > len(dst) {
+		dst = make([]byte, fullLen)
 	} else {
-		dst = dst[:l+pad(l)]
+		dst = dst[:fullLen]
 	}
 
 	var n int
 	n, r.err = io.ReadFull(r.r, dst)
 	if r.err != nil {
 		if debug {
-			dl.Debugf("@0x%x: rd bytes (%d): %v", s, len(dst), r.err)
+			dl.Printf("rd bytes (%d): %v", len(dst), r.err)
 		}
 		return nil
 	}
-	r.tot += n
 
 	if debug {
 		if n > maxDebugBytes {
-			dl.Debugf("@0x%x: rd bytes (%d): %x...", s, len(dst), dst[:maxDebugBytes])
+			dl.Printf("rd bytes (%d): %x...", len(dst), dst[:maxDebugBytes])
 		} else {
-			dl.Debugf("@0x%x: rd bytes (%d): %x", s, len(dst), dst)
+			dl.Printf("rd bytes (%d): %x", len(dst), dst)
 		}
 	}
 	return dst[:l]
 }
 
 func (r *Reader) ReadBool() bool {
-	return r.ReadUint32() != 0
-}
-
-func (r *Reader) ReadUint16() uint16 {
-	return uint16(r.ReadUint32())
+	return r.ReadUint8() != 0
 }
 
 func (r *Reader) ReadUint32() uint32 {
 	if r.err != nil {
 		return 0
 	}
-	r.last = time.Now()
-	s := r.tot
 
-	var n int
-	n, r.err = io.ReadFull(r.r, r.b[:4])
-	r.tot += n
+	_, r.err = io.ReadFull(r.r, r.b[:4])
 	if r.err != nil {
 		if debug {
-			dl.Debugf("@0x%x: rd uint32: %v", r.tot, r.err)
+			dl.Printf("rd uint32: %v", r.err)
 		}
 		return 0
 	}
@@ -129,7 +120,7 @@ func (r *Reader) ReadUint32() uint32 {
 	v := uint32(r.b[3]) | uint32(r.b[2])<<8 | uint32(r.b[1])<<16 | uint32(r.b[0])<<24
 
 	if debug {
-		dl.Debugf("@0x%x: rd uint32=%d (0x%08x)", s, v, v)
+		dl.Printf("rd uint32=%d (0x%08x)", v, v)
 	}
 	return v
 }
@@ -138,15 +129,11 @@ func (r *Reader) ReadUint64() uint64 {
 	if r.err != nil {
 		return 0
 	}
-	r.last = time.Now()
-	s := r.tot
 
-	var n int
-	n, r.err = io.ReadFull(r.r, r.b[:8])
-	r.tot += n
+	_, r.err = io.ReadFull(r.r, r.b[:8])
 	if r.err != nil {
 		if debug {
-			dl.Debugf("@0x%x: rd uint64: %v", r.tot, r.err)
+			dl.Printf("rd uint64: %v", r.err)
 		}
 		return 0
 	}
@@ -155,19 +142,23 @@ func (r *Reader) ReadUint64() uint64 {
 		uint64(r.b[3])<<32 | uint64(r.b[2])<<40 | uint64(r.b[1])<<48 | uint64(r.b[0])<<56
 
 	if debug {
-		dl.Debugf("@0x%x: rd uint64=%d (0x%016x)", s, v, v)
+		dl.Printf("rd uint64=%d (0x%016x)", v, v)
 	}
 	return v
 }
 
-func (r *Reader) Tot() int {
-	return r.tot
+type XDRError struct {
+	op  string
+	err error
+}
+
+func (e XDRError) Error() string {
+	return "xdr " + e.op + ": " + e.err.Error()
 }
 
 func (r *Reader) Error() error {
-	return r.err
-}
-
-func (r *Reader) LastRead() time.Time {
-	return r.last
+	if r.err == nil {
+		return nil
+	}
+	return XDRError{"read", r.err}
 }

Godeps/_workspace/src/github.com/calmh/xdr/reader_ipdr.go

@@ -0,0 +1,49 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// +build ipdr
+
+package xdr
+
+import "io"
+
+func (r *Reader) ReadUint8() uint8 {
+	if r.err != nil {
+		return 0
+	}
+
+	_, r.err = io.ReadFull(r.r, r.b[:1])
+	if r.err != nil {
+		if debug {
+			dl.Printf("rd uint8: %v", r.err)
+		}
+		return 0
+	}
+
+	if debug {
+		dl.Printf("rd uint8=%d (0x%02x)", r.b[0], r.b[0])
+	}
+	return r.b[0]
+}
+
+func (r *Reader) ReadUint16() uint16 {
+	if r.err != nil {
+		return 0
+	}
+
+	_, r.err = io.ReadFull(r.r, r.b[:2])
+	if r.err != nil {
+		if debug {
+			dl.Printf("rd uint16: %v", r.err)
+		}
+		return 0
+	}
+
+	v := uint16(r.b[1]) | uint16(r.b[0])<<8
+
+	if debug {
+		dl.Printf("rd uint16=%d (0x%04x)", v, v)
+	}
+	return v
+}

Godeps/_workspace/src/github.com/calmh/xdr/reader_xdr.go

@@ -2,5 +2,14 @@
 // All rights reserved. Use of this source code is governed by an MIT-style
 // license that can be found in the LICENSE file.
 
-// Package xdr implements an XDR (RFC 4506) encoder/decoder.
+// +build !ipdr
+
 package xdr
+
+func (r *Reader) ReadUint8() uint8 {
+	return uint8(r.ReadUint32())
+}
+
+func (r *Reader) ReadUint16() uint16 {
+	return uint16(r.ReadUint32())
+}

Godeps/_workspace/src/github.com/calmh/xdr/refl_test.go

@@ -1,6 +1,5 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
 
 // +build refl
 

Godeps/_workspace/src/github.com/calmh/xdr/writer.go

@@ -1,30 +1,21 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
 
 package xdr
 
 import (
 	"io"
-	"time"
+	"reflect"
+	"unsafe"
 )
 
-func pad(l int) int {
-	d := l % 4
-	if d == 0 {
-		return 0
-	}
-	return 4 - d
-}
-
 var padBytes = []byte{0, 0, 0}
 
 type Writer struct {
-	w    io.Writer
-	tot  int
-	err  error
-	b    [8]byte
-	last time.Time
+	w   io.Writer
+	tot int
+	err error
+	b   [8]byte
 }
 
 type AppendWriter []byte
@@ -40,8 +31,24 @@ func NewWriter(w io.Writer) *Writer {
 	}
 }
 
+func (w *Writer) WriteRaw(bs []byte) (int, error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+
+	var n int
+	n, w.err = w.w.Write(bs)
+	return n, w.err
+}
+
 func (w *Writer) WriteString(s string) (int, error) {
-	return w.WriteBytes([]byte(s))
+	sh := *((*reflect.StringHeader)(unsafe.Pointer(&s)))
+	bh := reflect.SliceHeader{
+		Data: sh.Data,
+		Len:  sh.Len,
+		Cap:  sh.Len,
+	}
+	return w.WriteBytes(*(*[]byte)(unsafe.Pointer(&bh)))
 }
 
 func (w *Writer) WriteBytes(bs []byte) (int, error) {
@@ -49,7 +56,6 @@ func (w *Writer) WriteBytes(bs []byte) (int, error) {
 		return 0, w.err
 	}
 
-	w.last = time.Now()
 	w.WriteUint32(uint32(len(bs)))
 	if w.err != nil {
 		return 0, w.err
@@ -57,9 +63,9 @@ func (w *Writer) WriteBytes(bs []byte) (int, error) {
 
 	if debug {
 		if len(bs) > maxDebugBytes {
-			dl.Debugf("wr bytes (%d): %x...", len(bs), bs[:maxDebugBytes])
+			dl.Printf("wr bytes (%d): %x...", len(bs), bs[:maxDebugBytes])
 		} else {
-			dl.Debugf("wr bytes (%d): %x", len(bs), bs)
+			dl.Printf("wr bytes (%d): %x", len(bs), bs)
 		}
 	}
 
@@ -78,24 +84,19 @@ func (w *Writer) WriteBytes(bs []byte) (int, error) {
 
 func (w *Writer) WriteBool(v bool) (int, error) {
 	if v {
-		return w.WriteUint32(1)
+		return w.WriteUint8(1)
 	} else {
-		return w.WriteUint32(0)
+		return w.WriteUint8(0)
 	}
 }
 
-func (w *Writer) WriteUint16(v uint16) (int, error) {
-	return w.WriteUint32(uint32(v))
-}
-
 func (w *Writer) WriteUint32(v uint32) (int, error) {
 	if w.err != nil {
 		return 0, w.err
 	}
 
-	w.last = time.Now()
 	if debug {
-		dl.Debugf("wr uint32=%d", v)
+		dl.Printf("wr uint32=%d", v)
 	}
 
 	w.b[0] = byte(v >> 24)
@@ -114,9 +115,8 @@ func (w *Writer) WriteUint64(v uint64) (int, error) {
 		return 0, w.err
 	}
 
-	w.last = time.Now()
 	if debug {
-		dl.Debugf("wr uint64=%d", v)
+		dl.Printf("wr uint64=%d", v)
 	}
 
 	w.b[0] = byte(v >> 56)
@@ -139,9 +139,8 @@ func (w *Writer) Tot() int {
 }
 
 func (w *Writer) Error() error {
-	return w.err
-}
-
-func (w *Writer) LastWrite() time.Time {
-	return w.last
+	if w.err == nil {
+		return nil
+	}
+	return XDRError{"write", w.err}
 }

Godeps/_workspace/src/github.com/calmh/xdr/writer_ipdr.go

@@ -0,0 +1,41 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+// +build ipdr
+
+package xdr
+
+func (w *Writer) WriteUint8(v uint8) (int, error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+
+	if debug {
+		dl.Printf("wr uint8=%d", v)
+	}
+
+	w.b[0] = byte(v)
+
+	var l int
+	l, w.err = w.w.Write(w.b[:1])
+	w.tot += l
+	return l, w.err
+}
+
+func (w *Writer) WriteUint16(v uint16) (int, error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+
+	if debug {
+		dl.Printf("wr uint8=%d", v)
+	}
+
+	w.b[0] = byte(v >> 8)
+	w.b[1] = byte(v)
+
+	var l int
+	l, w.err = w.w.Write(w.b[:2])
+	w.tot += l
+	return l, w.err
+}

Godeps/_workspace/src/github.com/calmh/xdr/writer_xdr.go

@@ -0,0 +1,14 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+// +build !ipdr
+
+package xdr
+
+func (w *Writer) WriteUint8(v uint8) (int, error) {
+	return w.WriteUint32(uint32(v))
+}
+
+func (w *Writer) WriteUint16(v uint16) (int, error) {
+	return w.WriteUint32(uint32(v))
+}

Godeps/_workspace/src/github.com/calmh/xdr/xdr_test.go

@@ -1,6 +1,5 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
 
 package xdr
 
@@ -10,23 +9,6 @@ import (
 	"testing/quick"
 )
 
-func TestPad(t *testing.T) {
-	tests := [][]int{
-		{0, 0},
-		{1, 3},
-		{2, 2},
-		{3, 1},
-		{4, 0},
-		{32, 0},
-		{33, 3},
-	}
-	for _, tc := range tests {
-		if p := pad(tc[0]); p != tc[1] {
-			t.Errorf("Incorrect padding for %d bytes, %d != %d", tc[0], p, tc[1])
-		}
-	}
-}
-
 func TestBytesNil(t *testing.T) {
 	fn := func(bs []byte) bool {
 		var b = new(bytes.Buffer)
@@ -85,7 +67,7 @@ func TestReadBytesMaxInto(t *testing.T) {
 	}
 }
 
-func TestReadBytesMaxIntoNil(t *testing.T) {
+func TestReadStringMax(t *testing.T) {
 	for tot := 42; tot < 72; tot++ {
 		for max := 0; max < 128; max++ {
 			var b = new(bytes.Buffer)
@@ -95,8 +77,8 @@ func TestReadBytesMaxIntoNil(t *testing.T) {
 			var toWrite = make([]byte, tot)
 			w.WriteBytes(toWrite)
 
-			var bs = r.ReadBytesMaxInto(max, nil)
-			var read = len(bs)
+			var str = r.ReadStringMax(max)
+			var read = len(str)
 
 			if max == 0 || tot <= max {
 				if read != tot {

Godeps/_workspace/src/github.com/golang/groupcache/lru/lru.go

@@ -1,121 +0,0 @@
-/*
-Copyright 2013 Google Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package lru implements an LRU cache.
-package lru
-
-import "container/list"
-
-// Cache is an LRU cache. It is not safe for concurrent access.
-type Cache struct {
-	// MaxEntries is the maximum number of cache entries before
-	// an item is evicted. Zero means no limit.
-	MaxEntries int
-
-	// OnEvicted optionally specificies a callback function to be
-	// executed when an entry is purged from the cache.
-	OnEvicted func(key Key, value interface{})
-
-	ll    *list.List
-	cache map[interface{}]*list.Element
-}
-
-// A Key may be any value that is comparable. See http://golang.org/ref/spec#Comparison_operators
-type Key interface{}
-
-type entry struct {
-	key   Key
-	value interface{}
-}
-
-// New creates a new Cache.
-// If maxEntries is zero, the cache has no limit and it's assumed
-// that eviction is done by the caller.
-func New(maxEntries int) *Cache {
-	return &Cache{
-		MaxEntries: maxEntries,
-		ll:         list.New(),
-		cache:      make(map[interface{}]*list.Element),
-	}
-}
-
-// Add adds a value to the cache.
-func (c *Cache) Add(key Key, value interface{}) {
-	if c.cache == nil {
-		c.cache = make(map[interface{}]*list.Element)
-		c.ll = list.New()
-	}
-	if ee, ok := c.cache[key]; ok {
-		c.ll.MoveToFront(ee)
-		ee.Value.(*entry).value = value
-		return
-	}
-	ele := c.ll.PushFront(&entry{key, value})
-	c.cache[key] = ele
-	if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries {
-		c.RemoveOldest()
-	}
-}
-
-// Get looks up a key's value from the cache.
-func (c *Cache) Get(key Key) (value interface{}, ok bool) {
-	if c.cache == nil {
-		return
-	}
-	if ele, hit := c.cache[key]; hit {
-		c.ll.MoveToFront(ele)
-		return ele.Value.(*entry).value, true
-	}
-	return
-}
-
-// Remove removes the provided key from the cache.
-func (c *Cache) Remove(key Key) {
-	if c.cache == nil {
-		return
-	}
-	if ele, hit := c.cache[key]; hit {
-		c.removeElement(ele)
-	}
-}
-
-// RemoveOldest removes the oldest item from the cache.
-func (c *Cache) RemoveOldest() {
-	if c.cache == nil {
-		return
-	}
-	ele := c.ll.Back()
-	if ele != nil {
-		c.removeElement(ele)
-	}
-}
-
-func (c *Cache) removeElement(e *list.Element) {
-	c.ll.Remove(e)
-	kv := e.Value.(*entry)
-	delete(c.cache, kv.key)
-	if c.OnEvicted != nil {
-		c.OnEvicted(kv.key, kv.value)
-	}
-}
-
-// Len returns the number of items in the cache.
-func (c *Cache) Len() int {
-	if c.cache == nil {
-		return 0
-	}
-	return c.ll.Len()
-}

Godeps/_workspace/src/github.com/golang/groupcache/lru/lru_test.go

@@ -1,73 +0,0 @@
-/*
-Copyright 2013 Google Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package lru
-
-import (
-	"testing"
-)
-
-type simpleStruct struct {
-	int
-	string
-}
-
-type complexStruct struct {
-	int
-	simpleStruct
-}
-
-var getTests = []struct {
-	name       string
-	keyToAdd   interface{}
-	keyToGet   interface{}
-	expectedOk bool
-}{
-	{"string_hit", "myKey", "myKey", true},
-	{"string_miss", "myKey", "nonsense", false},
-	{"simple_struct_hit", simpleStruct{1, "two"}, simpleStruct{1, "two"}, true},
-	{"simeple_struct_miss", simpleStruct{1, "two"}, simpleStruct{0, "noway"}, false},
-	{"complex_struct_hit", complexStruct{1, simpleStruct{2, "three"}},
-		complexStruct{1, simpleStruct{2, "three"}}, true},
-}
-
-func TestGet(t *testing.T) {
-	for _, tt := range getTests {
-		lru := New(0)
-		lru.Add(tt.keyToAdd, 1234)
-		val, ok := lru.Get(tt.keyToGet)
-		if ok != tt.expectedOk {
-			t.Fatalf("%s: cache hit = %v; want %v", tt.name, ok, !ok)
-		} else if ok && val != 1234 {
-			t.Fatalf("%s expected get to return 1234 but got %v", tt.name, val)
-		}
-	}
-}
-
-func TestRemove(t *testing.T) {
-	lru := New(0)
-	lru.Add("myKey", 1234)
-	if val, ok := lru.Get("myKey"); !ok {
-		t.Fatal("TestRemove returned no match")
-	} else if val != 1234 {
-		t.Fatalf("TestRemove failed.  Expected %d, got %v", 1234, val)
-	}
-
-	lru.Remove("myKey")
-	if _, ok := lru.Get("myKey"); ok {
-		t.Fatal("TestRemove returned a removed entry")
-	}
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench_test.go

@@ -170,7 +170,7 @@ func (p *dbBench) writes(perBatch int) {
 	b.SetBytes(116)
 }
 
-func (p *dbBench) drop() {
+func (p *dbBench) gc() {
 	p.keys, p.values = nil, nil
 	runtime.GC()
 }
@@ -249,6 +249,9 @@ func (p *dbBench) newIter() iterator.Iterator {
 }
 
 func (p *dbBench) close() {
+	if bp, err := p.db.GetProperty("leveldb.blockpool"); err == nil {
+		p.b.Log("Block pool stats: ", bp)
+	}
 	p.db.Close()
 	p.stor.Close()
 	os.RemoveAll(benchDB)
@@ -331,7 +334,7 @@ func BenchmarkDBRead(b *testing.B) {
 	p := openDBBench(b, false)
 	p.populate(b.N)
 	p.fill()
-	p.drop()
+	p.gc()
 
 	iter := p.newIter()
 	b.ResetTimer()
@@ -362,7 +365,7 @@ func BenchmarkDBReadUncompressed(b *testing.B) {
 	p := openDBBench(b, true)
 	p.populate(b.N)
 	p.fill()
-	p.drop()
+	p.gc()
 
 	iter := p.newIter()
 	b.ResetTimer()
@@ -379,7 +382,7 @@ func BenchmarkDBReadTable(b *testing.B) {
 	p.populate(b.N)
 	p.fill()
 	p.reopen()
-	p.drop()
+	p.gc()
 
 	iter := p.newIter()
 	b.ResetTimer()
@@ -395,7 +398,7 @@ func BenchmarkDBReadReverse(b *testing.B) {
 	p := openDBBench(b, false)
 	p.populate(b.N)
 	p.fill()
-	p.drop()
+	p.gc()
 
 	iter := p.newIter()
 	b.ResetTimer()
@@ -413,7 +416,7 @@ func BenchmarkDBReadReverseTable(b *testing.B) {
 	p.populate(b.N)
 	p.fill()
 	p.reopen()
-	p.drop()
+	p.gc()
 
 	iter := p.newIter()
 	b.ResetTimer()

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache.go

@@ -11,115 +11,149 @@ import (
 	"sync/atomic"
 )
 
-// SetFunc used by Namespace.Get method to create a cache object. SetFunc
-// may return ok false, in that case the cache object will not be created.
-type SetFunc func() (ok bool, value interface{}, charge int, fin SetFin)
+// SetFunc is the function that will be called by Namespace.Get to create
+// a cache object, if charge is less than one than the cache object will
+// not be registered to cache tree, if value is nil then the cache object
+// will not be created.
+type SetFunc func() (charge int, value interface{})
 
-// SetFin will be called when corresponding cache object are released.
-type SetFin func()
+// DelFin is the function that will be called as the result of a delete operation.
+// Exist == true is indication that the object is exist, and pending == true is
+// indication of deletion already happen but haven't done yet (wait for all handles
+// to be released). And exist == false means the object doesn't exist.
+type DelFin func(exist, pending bool)
 
-// DelFin will be called when corresponding cache object are released.
-// DelFin will be called after SetFin. The exist is true if the corresponding
-// cache object is actually exist in the cache tree.
-type DelFin func(exist bool)
+// PurgeFin is the function that will be called as the result of a purge operation.
+type PurgeFin func(ns, key uint64)
 
-// PurgeFin will be called when corresponding cache object are released.
-// PurgeFin will be called after SetFin. If PurgeFin present DelFin will
-// not be executed but passed to the PurgeFin, it is up to the caller
-// to call it or not.
-type PurgeFin func(ns, key uint64, delfin DelFin)
-
-// Cache is a cache tree.
+// Cache is a cache tree. A cache instance must be goroutine-safe.
 type Cache interface {
-	// SetCapacity sets cache capacity.
+	// SetCapacity sets cache tree capacity.
 	SetCapacity(capacity int)
 
-	// GetNamespace gets or creates a cache namespace for the given id.
+	// Capacity returns cache tree capacity.
+	Capacity() int
+
+	// Used returns used cache tree capacity.
+	Used() int
+
+	// Size returns entire alive cache objects size.
+	Size() int
+
+	// NumObjects returns number of alive objects.
+	NumObjects() int
+
+	// GetNamespace gets cache namespace with the given id.
+	// GetNamespace is never return nil.
 	GetNamespace(id uint64) Namespace
 
-	// Purge purges all cache namespaces, read Namespace.Purge method documentation.
+	// PurgeNamespace purges cache namespace with the given id from this cache tree.
+	// Also read Namespace.Purge.
+	PurgeNamespace(id uint64, fin PurgeFin)
+
+	// ZapNamespace detaches cache namespace with the given id from this cache tree.
+	// Also read Namespace.Zap.
+	ZapNamespace(id uint64)
+
+	// Purge purges all cache namespace from this cache tree.
+	// This is behave the same as calling Namespace.Purge method on all cache namespace.
 	Purge(fin PurgeFin)
 
-	// Zap zaps all cache namespaces, read Namespace.Zap method documentation.
-	Zap(closed bool)
+	// Zap detaches all cache namespace from this cache tree.
+	// This is behave the same as calling Namespace.Zap method on all cache namespace.
+	Zap()
 }
 
-// Namespace is a cache namespace.
+// Namespace is a cache namespace. A namespace instance must be goroutine-safe.
 type Namespace interface {
-	// Get gets cache object for the given key. The given SetFunc (if not nil) will
-	// be called if the given key does not exist.
-	// If the given key does not exist, SetFunc is nil or SetFunc return ok false, Get
-	// will return ok false.
-	Get(key uint64, setf SetFunc) (obj Object, ok bool)
-
-	// Get deletes cache object for the given key. If exist the cache object will
-	// be deleted later when all of its handles have been released (i.e. no one use
-	// it anymore) and the given DelFin (if not nil) will finally be  executed. If
-	// such cache object does not exist the given DelFin will be executed anyway.
+	// Get gets cache object with the given key.
+	// If cache object is not found and setf is not nil, Get will atomically creates
+	// the cache object by calling setf. Otherwise Get will returns nil.
 	//
-	// Delete returns true if such cache object exist.
+	// The returned cache handle should be released after use by calling Release
+	// method.
+	Get(key uint64, setf SetFunc) Handle
+
+	// Delete removes cache object with the given key from cache tree.
+	// A deleted cache object will be released as soon as all of its handles have
+	// been released.
+	// Delete only happen once, subsequent delete will consider cache object doesn't
+	// exist, even if the cache object ins't released yet.
+	//
+	// If not nil, fin will be called if the cache object doesn't exist or when
+	// finally be released.
+	//
+	// Delete returns true if such cache object exist and never been deleted.
 	Delete(key uint64, fin DelFin) bool
 
-	// Purge deletes all cache objects, read Delete method documentation.
+	// Purge removes all cache objects within this namespace from cache tree.
+	// This is the same as doing delete on all cache objects.
+	//
+	// If not nil, fin will be called on all cache objects when its finally be
+	// released.
 	Purge(fin PurgeFin)
 
-	// Zap detaches the namespace from the cache tree and delete all its cache
-	// objects. The cache objects deletion and finalizers execution are happen
-	// immediately, even if its existing handles haven't yet been released.
-	// A zapped namespace can't never be filled again.
-	// If closed is false then the Get function will always call the given SetFunc
-	// if it is not nil, but resultant of the SetFunc will not be cached.
-	Zap(closed bool)
+	// Zap detaches namespace from cache tree and release all its cache objects.
+	// A zapped namespace can never be filled again.
+	// Calling Get on zapped namespace will always return nil.
+	Zap()
 }
 
-// Object is a cache object.
-type Object interface {
-	// Release releases the cache object. Other methods should not be called
-	// after the cache object has been released.
+// Handle is a cache handle.
+type Handle interface {
+	// Release releases this cache handle. This method can be safely called mutiple
+	// times.
 	Release()
 
-	// Value returns value of the cache object.
+	// Value returns value of this cache handle.
+	// Value will returns nil after this cache handle have be released.
 	Value() interface{}
 }
 
+const (
+	DelNotExist = iota
+	DelExist
+	DelPendig
+)
+
 // Namespace state.
 type nsState int
 
 const (
 	nsEffective nsState = iota
 	nsZapped
-	nsClosed
 )
 
 // Node state.
 type nodeState int
 
 const (
-	nodeEffective nodeState = iota
+	nodeZero nodeState = iota
+	nodeEffective
 	nodeEvicted
-	nodeRemoved
+	nodeDeleted
 )
 
-// Fake object.
-type fakeObject struct {
+// Fake handle.
+type fakeHandle struct {
 	value interface{}
 	fin   func()
 	once  uint32
 }
 
-func (o *fakeObject) Value() interface{} {
-	if atomic.LoadUint32(&o.once) == 0 {
-		return o.value
+func (h *fakeHandle) Value() interface{} {
+	if atomic.LoadUint32(&h.once) == 0 {
+		return h.value
 	}
 	return nil
 }
 
-func (o *fakeObject) Release() {
-	if !atomic.CompareAndSwapUint32(&o.once, 0, 1) {
+func (h *fakeHandle) Release() {
+	if !atomic.CompareAndSwapUint32(&h.once, 0, 1) {
 		return
 	}
-	if o.fin != nil {
-		o.fin()
-		o.fin = nil
+	if h.fin != nil {
+		h.fin()
+		h.fin = nil
 	}
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go

@@ -8,14 +8,32 @@ package cache
 
 import (
 	"math/rand"
+	"runtime"
+	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
 )
 
-func set(ns Namespace, key uint64, value interface{}, charge int, fin func()) Object {
-	obj, _ := ns.Get(key, func() (bool, interface{}, int, SetFin) {
-		return true, value, charge, fin
+type releaserFunc struct {
+	fn    func()
+	value interface{}
+}
+
+func (r releaserFunc) Release() {
+	if r.fn != nil {
+		r.fn()
+	}
+}
+
+func set(ns Namespace, key uint64, value interface{}, charge int, relf func()) Handle {
+	return ns.Get(key, func() (int, interface{}) {
+		if relf != nil {
+			return charge, releaserFunc{relf, value}
+		} else {
+			return charge, value
+		}
 	})
-	return obj
 }
 
 func TestCache_HitMiss(t *testing.T) {
@@ -43,29 +61,31 @@ func TestCache_HitMiss(t *testing.T) {
 			setfin++
 		}).Release()
 		for j, y := range cases {
-			r, ok := ns.Get(y.key, nil)
+			h := ns.Get(y.key, nil)
 			if j <= i {
 				// should hit
-				if !ok {
+				if h == nil {
 					t.Errorf("case '%d' iteration '%d' is miss", i, j)
-				} else if r.Value().(string) != y.value {
-					t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, r.Value().(string), y.value)
+				} else {
+					if x := h.Value().(releaserFunc).value.(string); x != y.value {
+						t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, x, y.value)
+					}
 				}
 			} else {
 				// should miss
-				if ok {
-					t.Errorf("case '%d' iteration '%d' is hit , value '%s'", i, j, r.Value().(string))
+				if h != nil {
+					t.Errorf("case '%d' iteration '%d' is hit , value '%s'", i, j, h.Value().(releaserFunc).value.(string))
 				}
 			}
-			if ok {
-				r.Release()
+			if h != nil {
+				h.Release()
 			}
 		}
 	}
 
 	for i, x := range cases {
 		finalizerOk := false
-		ns.Delete(x.key, func(exist bool) {
+		ns.Delete(x.key, func(exist, pending bool) {
 			finalizerOk = true
 		})
 
@@ -74,22 +94,24 @@ func TestCache_HitMiss(t *testing.T) {
 		}
 
 		for j, y := range cases {
-			r, ok := ns.Get(y.key, nil)
+			h := ns.Get(y.key, nil)
 			if j > i {
 				// should hit
-				if !ok {
+				if h == nil {
 					t.Errorf("case '%d' iteration '%d' is miss", i, j)
-				} else if r.Value().(string) != y.value {
-					t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, r.Value().(string), y.value)
+				} else {
+					if x := h.Value().(releaserFunc).value.(string); x != y.value {
+						t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, x, y.value)
+					}
 				}
 			} else {
 				// should miss
-				if ok {
-					t.Errorf("case '%d' iteration '%d' is hit, value '%s'", i, j, r.Value().(string))
+				if h != nil {
+					t.Errorf("case '%d' iteration '%d' is hit, value '%s'", i, j, h.Value().(releaserFunc).value.(string))
 				}
 			}
-			if ok {
-				r.Release()
+			if h != nil {
+				h.Release()
 			}
 		}
 	}
@@ -107,42 +129,42 @@ func TestLRUCache_Eviction(t *testing.T) {
 	set(ns, 3, 3, 1, nil).Release()
 	set(ns, 4, 4, 1, nil).Release()
 	set(ns, 5, 5, 1, nil).Release()
-	if r, ok := ns.Get(2, nil); ok { // 1,3,4,5,2
-		r.Release()
+	if h := ns.Get(2, nil); h != nil { // 1,3,4,5,2
+		h.Release()
 	}
 	set(ns, 9, 9, 10, nil).Release() // 5,2,9
 
-	for _, x := range []uint64{9, 2, 5, 1} {
-		r, ok := ns.Get(x, nil)
-		if !ok {
-			t.Errorf("miss for key '%d'", x)
+	for _, key := range []uint64{9, 2, 5, 1} {
+		h := ns.Get(key, nil)
+		if h == nil {
+			t.Errorf("miss for key '%d'", key)
 		} else {
-			if r.Value().(int) != int(x) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
+			if x := h.Value().(int); x != int(key) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
 			}
-			r.Release()
+			h.Release()
 		}
 	}
 	o1.Release()
-	for _, x := range []uint64{1, 2, 5} {
-		r, ok := ns.Get(x, nil)
-		if !ok {
-			t.Errorf("miss for key '%d'", x)
+	for _, key := range []uint64{1, 2, 5} {
+		h := ns.Get(key, nil)
+		if h == nil {
+			t.Errorf("miss for key '%d'", key)
 		} else {
-			if r.Value().(int) != int(x) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
+			if x := h.Value().(int); x != int(key) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
 			}
-			r.Release()
+			h.Release()
 		}
 	}
-	for _, x := range []uint64{3, 4, 9} {
-		r, ok := ns.Get(x, nil)
-		if ok {
-			t.Errorf("hit for key '%d'", x)
-			if r.Value().(int) != int(x) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
+	for _, key := range []uint64{3, 4, 9} {
+		h := ns.Get(key, nil)
+		if h != nil {
+			t.Errorf("hit for key '%d'", key)
+			if x := h.Value().(int); x != int(key) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
 			}
-			r.Release()
+			h.Release()
 		}
 	}
 }
@@ -153,16 +175,15 @@ func TestLRUCache_SetGet(t *testing.T) {
 	for i := 0; i < 200; i++ {
 		n := uint64(rand.Intn(99999) % 20)
 		set(ns, n, n, 1, nil).Release()
-		if p, ok := ns.Get(n, nil); ok {
-			if p.Value() == nil {
+		if h := ns.Get(n, nil); h != nil {
+			if h.Value() == nil {
 				t.Errorf("key '%d' contains nil value", n)
 			} else {
-				got := p.Value().(uint64)
-				if got != n {
-					t.Errorf("invalid value for key '%d' want '%d', got '%d'", n, n, got)
+				if x := h.Value().(uint64); x != n {
+					t.Errorf("invalid value for key '%d' want '%d', got '%d'", n, n, x)
 				}
 			}
-			p.Release()
+			h.Release()
 		} else {
 			t.Errorf("key '%d' doesn't exist", n)
 		}
@@ -176,31 +197,429 @@ func TestLRUCache_Purge(t *testing.T) {
 	o2 := set(ns1, 2, 2, 1, nil)
 	ns1.Purge(nil)
 	set(ns1, 3, 3, 1, nil).Release()
-	for _, x := range []uint64{1, 2, 3} {
-		r, ok := ns1.Get(x, nil)
-		if !ok {
-			t.Errorf("miss for key '%d'", x)
+	for _, key := range []uint64{1, 2, 3} {
+		h := ns1.Get(key, nil)
+		if h == nil {
+			t.Errorf("miss for key '%d'", key)
 		} else {
-			if r.Value().(int) != int(x) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
+			if x := h.Value().(int); x != int(key) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
 			}
-			r.Release()
+			h.Release()
 		}
 	}
 	o1.Release()
 	o2.Release()
-	for _, x := range []uint64{1, 2} {
-		r, ok := ns1.Get(x, nil)
-		if ok {
-			t.Errorf("hit for key '%d'", x)
-			if r.Value().(int) != int(x) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
+	for _, key := range []uint64{1, 2} {
+		h := ns1.Get(key, nil)
+		if h != nil {
+			t.Errorf("hit for key '%d'", key)
+			if x := h.Value().(int); x != int(key) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
 			}
-			r.Release()
+			h.Release()
 		}
 	}
 }
 
+type testingCacheObjectCounter struct {
+	created  uint
+	released uint
+}
+
+func (c *testingCacheObjectCounter) createOne() {
+	c.created++
+}
+
+func (c *testingCacheObjectCounter) releaseOne() {
+	c.released++
+}
+
+type testingCacheObject struct {
+	t   *testing.T
+	cnt *testingCacheObjectCounter
+
+	ns, key uint64
+
+	releaseCalled bool
+}
+
+func (x *testingCacheObject) Release() {
+	if !x.releaseCalled {
+		x.releaseCalled = true
+		x.cnt.releaseOne()
+	} else {
+		x.t.Errorf("duplicate setfin NS#%d KEY#%s", x.ns, x.key)
+	}
+}
+
+func TestLRUCache_ConcurrentSetGet(t *testing.T) {
+	runtime.GOMAXPROCS(runtime.NumCPU())
+
+	seed := time.Now().UnixNano()
+	t.Logf("seed=%d", seed)
+
+	const (
+		N = 2000000
+		M = 4000
+		C = 3
+	)
+
+	var set, get uint32
+
+	wg := &sync.WaitGroup{}
+	c := NewLRUCache(M / 4)
+	for ni := uint64(0); ni < C; ni++ {
+		r0 := rand.New(rand.NewSource(seed + int64(ni)))
+		r1 := rand.New(rand.NewSource(seed + int64(ni) + 1))
+		ns := c.GetNamespace(ni)
+
+		wg.Add(2)
+		go func(ns Namespace, r *rand.Rand) {
+			for i := 0; i < N; i++ {
+				x := uint64(r.Int63n(M))
+				o := ns.Get(x, func() (int, interface{}) {
+					atomic.AddUint32(&set, 1)
+					return 1, x
+				})
+				if v := o.Value().(uint64); v != x {
+					t.Errorf("#%d invalid value, got=%d", x, v)
+				}
+				o.Release()
+			}
+			wg.Done()
+		}(ns, r0)
+		go func(ns Namespace, r *rand.Rand) {
+			for i := 0; i < N; i++ {
+				x := uint64(r.Int63n(M))
+				o := ns.Get(x, nil)
+				if o != nil {
+					atomic.AddUint32(&get, 1)
+					if v := o.Value().(uint64); v != x {
+						t.Errorf("#%d invalid value, got=%d", x, v)
+					}
+					o.Release()
+				}
+			}
+			wg.Done()
+		}(ns, r1)
+	}
+
+	wg.Wait()
+
+	t.Logf("set=%d get=%d", set, get)
+}
+
+func TestLRUCache_Finalizer(t *testing.T) {
+	const (
+		capacity   = 100
+		goroutines = 100
+		iterations = 10000
+		keymax     = 8000
+	)
+
+	cnt := &testingCacheObjectCounter{}
+
+	c := NewLRUCache(capacity)
+
+	type instance struct {
+		seed       int64
+		rnd        *rand.Rand
+		nsid       uint64
+		ns         Namespace
+		effective  int
+		handles    []Handle
+		handlesMap map[uint64]int
+
+		delete          bool
+		purge           bool
+		zap             bool
+		wantDel         int
+		delfinCalled    int
+		delfinCalledAll int
+		delfinCalledEff int
+		purgefinCalled  int
+	}
+
+	instanceGet := func(p *instance, key uint64) {
+		h := p.ns.Get(key, func() (charge int, value interface{}) {
+			to := &testingCacheObject{
+				t: t, cnt: cnt,
+				ns:  p.nsid,
+				key: key,
+			}
+			p.effective++
+			cnt.createOne()
+			return 1, releaserFunc{func() {
+				to.Release()
+				p.effective--
+			}, to}
+		})
+		p.handles = append(p.handles, h)
+		p.handlesMap[key] = p.handlesMap[key] + 1
+	}
+	instanceRelease := func(p *instance, i int) {
+		h := p.handles[i]
+		key := h.Value().(releaserFunc).value.(*testingCacheObject).key
+		if n := p.handlesMap[key]; n == 0 {
+			t.Fatal("key ref == 0")
+		} else if n > 1 {
+			p.handlesMap[key] = n - 1
+		} else {
+			delete(p.handlesMap, key)
+		}
+		h.Release()
+		p.handles = append(p.handles[:i], p.handles[i+1:]...)
+		p.handles[len(p.handles) : len(p.handles)+1][0] = nil
+	}
+
+	seed := time.Now().UnixNano()
+	t.Logf("seed=%d", seed)
+
+	instances := make([]*instance, goroutines)
+	for i := range instances {
+		p := &instance{}
+		p.handlesMap = make(map[uint64]int)
+		p.seed = seed + int64(i)
+		p.rnd = rand.New(rand.NewSource(p.seed))
+		p.nsid = uint64(i)
+		p.ns = c.GetNamespace(p.nsid)
+		p.delete = i%6 == 0
+		p.purge = i%8 == 0
+		p.zap = i%12 == 0 || i%3 == 0
+		instances[i] = p
+	}
+
+	runr := rand.New(rand.NewSource(seed - 1))
+	run := func(rnd *rand.Rand, x []*instance, init func(p *instance) bool, fn func(p *instance, i int) bool) {
+		var (
+			rx []*instance
+			rn []int
+		)
+		if init == nil {
+			rx = append([]*instance{}, x...)
+			rn = make([]int, len(x))
+		} else {
+			for _, p := range x {
+				if init(p) {
+					rx = append(rx, p)
+					rn = append(rn, 0)
+				}
+			}
+		}
+		for len(rx) > 0 {
+			i := rand.Intn(len(rx))
+			if fn(rx[i], rn[i]) {
+				rn[i]++
+			} else {
+				rx = append(rx[:i], rx[i+1:]...)
+				rn = append(rn[:i], rn[i+1:]...)
+			}
+		}
+	}
+
+	// Get and release.
+	run(runr, instances, nil, func(p *instance, i int) bool {
+		if i < iterations {
+			if len(p.handles) == 0 || p.rnd.Int()%2 == 0 {
+				instanceGet(p, uint64(p.rnd.Intn(keymax)))
+			} else {
+				instanceRelease(p, p.rnd.Intn(len(p.handles)))
+			}
+			return true
+		} else {
+			return false
+		}
+	})
+
+	if used, cap := c.Used(), c.Capacity(); used > cap {
+		t.Errorf("Used > capacity, used=%d cap=%d", used, cap)
+	}
+
+	// Check effective objects.
+	for i, p := range instances {
+		if int(p.effective) < len(p.handlesMap) {
+			t.Errorf("#%d effective objects < acquired handle, eo=%d ah=%d", i, p.effective, len(p.handlesMap))
+		}
+	}
+
+	if want := int(cnt.created - cnt.released); c.Size() != want {
+		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
+	}
+
+	// First delete.
+	run(runr, instances, func(p *instance) bool {
+		p.wantDel = p.effective
+		return p.delete
+	}, func(p *instance, i int) bool {
+		key := uint64(i)
+		if key < keymax {
+			_, wantExist := p.handlesMap[key]
+			gotExist := p.ns.Delete(key, func(exist, pending bool) {
+				p.delfinCalledAll++
+				if exist {
+					p.delfinCalledEff++
+				}
+			})
+			if !gotExist && wantExist {
+				t.Errorf("delete on NS#%d KEY#%d not found", p.nsid, key)
+			}
+			return true
+		} else {
+			return false
+		}
+	})
+
+	// Second delete.
+	run(runr, instances, func(p *instance) bool {
+		p.delfinCalled = 0
+		return p.delete
+	}, func(p *instance, i int) bool {
+		key := uint64(i)
+		if key < keymax {
+			gotExist := p.ns.Delete(key, func(exist, pending bool) {
+				if exist && !pending {
+					t.Errorf("delete fin on NS#%d KEY#%d exist and not pending for deletion", p.nsid, key)
+				}
+				p.delfinCalled++
+			})
+			if gotExist {
+				t.Errorf("delete on NS#%d KEY#%d found", p.nsid, key)
+			}
+			return true
+		} else {
+			if p.delfinCalled != keymax {
+				t.Errorf("(2) #%d not all delete fin called, diff=%d", p.ns, keymax-p.delfinCalled)
+			}
+			return false
+		}
+	})
+
+	// Purge.
+	run(runr, instances, func(p *instance) bool {
+		return p.purge
+	}, func(p *instance, i int) bool {
+		p.ns.Purge(func(ns, key uint64) {
+			p.purgefinCalled++
+		})
+		return false
+	})
+
+	if want := int(cnt.created - cnt.released); c.Size() != want {
+		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
+	}
+
+	// Release.
+	run(runr, instances, func(p *instance) bool {
+		return !p.zap
+	}, func(p *instance, i int) bool {
+		if len(p.handles) > 0 {
+			instanceRelease(p, len(p.handles)-1)
+			return true
+		} else {
+			return false
+		}
+	})
+
+	if want := int(cnt.created - cnt.released); c.Size() != want {
+		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
+	}
+
+	// Zap.
+	run(runr, instances, func(p *instance) bool {
+		return p.zap
+	}, func(p *instance, i int) bool {
+		p.ns.Zap()
+		p.handles = nil
+		p.handlesMap = nil
+		return false
+	})
+
+	if want := int(cnt.created - cnt.released); c.Size() != want {
+		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
+	}
+
+	if notrel, used := int(cnt.created-cnt.released), c.Used(); notrel != used {
+		t.Errorf("Invalid used value, want=%d got=%d", notrel, used)
+	}
+
+	c.Purge(nil)
+
+	for _, p := range instances {
+		if p.delete {
+			if p.delfinCalledAll != keymax {
+				t.Errorf("#%d not all delete fin called, purge=%v zap=%v diff=%d", p.nsid, p.purge, p.zap, keymax-p.delfinCalledAll)
+			}
+			if p.delfinCalledEff != p.wantDel {
+				t.Errorf("#%d not all effective delete fin called, diff=%d", p.nsid, p.wantDel-p.delfinCalledEff)
+			}
+			if p.purge && p.purgefinCalled > 0 {
+				t.Errorf("#%d some purge fin called, delete=%v zap=%v n=%d", p.nsid, p.delete, p.zap, p.purgefinCalled)
+			}
+		} else {
+			if p.purge {
+				if p.purgefinCalled != p.wantDel {
+					t.Errorf("#%d not all purge fin called, delete=%v zap=%v diff=%d", p.nsid, p.delete, p.zap, p.wantDel-p.purgefinCalled)
+				}
+			}
+		}
+	}
+
+	if cnt.created != cnt.released {
+		t.Errorf("Some cache object weren't released, created=%d released=%d", cnt.created, cnt.released)
+	}
+}
+
+func BenchmarkLRUCache_Set(b *testing.B) {
+	c := NewLRUCache(0)
+	ns := c.GetNamespace(0)
+	b.ResetTimer()
+	for i := uint64(0); i < uint64(b.N); i++ {
+		set(ns, i, "", 1, nil)
+	}
+}
+
+func BenchmarkLRUCache_Get(b *testing.B) {
+	c := NewLRUCache(0)
+	ns := c.GetNamespace(0)
+	b.ResetTimer()
+	for i := uint64(0); i < uint64(b.N); i++ {
+		set(ns, i, "", 1, nil)
+	}
+	b.ResetTimer()
+	for i := uint64(0); i < uint64(b.N); i++ {
+		ns.Get(i, nil)
+	}
+}
+
+func BenchmarkLRUCache_Get2(b *testing.B) {
+	c := NewLRUCache(0)
+	ns := c.GetNamespace(0)
+	b.ResetTimer()
+	for i := uint64(0); i < uint64(b.N); i++ {
+		set(ns, i, "", 1, nil)
+	}
+	b.ResetTimer()
+	for i := uint64(0); i < uint64(b.N); i++ {
+		ns.Get(i, func() (charge int, value interface{}) {
+			return 0, nil
+		})
+	}
+}
+
+func BenchmarkLRUCache_Release(b *testing.B) {
+	c := NewLRUCache(0)
+	ns := c.GetNamespace(0)
+	handles := make([]Handle, b.N)
+	for i := uint64(0); i < uint64(b.N); i++ {
+		handles[i] = set(ns, i, "", 1, nil)
+	}
+	b.ResetTimer()
+	for _, h := range handles {
+		h.Release()
+	}
+}
+
 func BenchmarkLRUCache_SetRelease(b *testing.B) {
 	capacity := b.N / 100
 	if capacity <= 0 {
@@ -210,7 +629,7 @@ func BenchmarkLRUCache_SetRelease(b *testing.B) {
 	ns := c.GetNamespace(0)
 	b.ResetTimer()
 	for i := uint64(0); i < uint64(b.N); i++ {
-		set(ns, i, nil, 1, nil).Release()
+		set(ns, i, "", 1, nil).Release()
 	}
 }
 
@@ -227,10 +646,10 @@ func BenchmarkLRUCache_SetReleaseTwice(b *testing.B) {
 	nb := b.N - na
 
 	for i := uint64(0); i < uint64(na); i++ {
-		set(ns, i, nil, 1, nil).Release()
+		set(ns, i, "", 1, nil).Release()
 	}
 
 	for i := uint64(0); i < uint64(nb); i++ {
-		set(ns, i, nil, 1, nil).Release()
+		set(ns, i, "", 1, nil).Release()
 	}
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/empty_cache.go

@@ -1,246 +0,0 @@
-// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package cache
-
-import (
-	"sync"
-	"sync/atomic"
-)
-
-type emptyCache struct {
-	sync.Mutex
-	table map[uint64]*emptyNS
-}
-
-// NewEmptyCache creates a new initialized empty cache.
-func NewEmptyCache() Cache {
-	return &emptyCache{
-		table: make(map[uint64]*emptyNS),
-	}
-}
-
-func (c *emptyCache) GetNamespace(id uint64) Namespace {
-	c.Lock()
-	defer c.Unlock()
-
-	if ns, ok := c.table[id]; ok {
-		return ns
-	}
-
-	ns := &emptyNS{
-		cache: c,
-		id:    id,
-		table: make(map[uint64]*emptyNode),
-	}
-	c.table[id] = ns
-	return ns
-}
-
-func (c *emptyCache) Purge(fin PurgeFin) {
-	c.Lock()
-	for _, ns := range c.table {
-		ns.purgeNB(fin)
-	}
-	c.Unlock()
-}
-
-func (c *emptyCache) Zap(closed bool) {
-	c.Lock()
-	for _, ns := range c.table {
-		ns.zapNB(closed)
-	}
-	c.table = make(map[uint64]*emptyNS)
-	c.Unlock()
-}
-
-func (*emptyCache) SetCapacity(capacity int) {}
-
-type emptyNS struct {
-	cache *emptyCache
-	id    uint64
-	table map[uint64]*emptyNode
-	state nsState
-}
-
-func (ns *emptyNS) Get(key uint64, setf SetFunc) (o Object, ok bool) {
-	ns.cache.Lock()
-
-	switch ns.state {
-	case nsZapped:
-		ns.cache.Unlock()
-		if setf == nil {
-			return
-		}
-
-		var value interface{}
-		var fin func()
-		ok, value, _, fin = setf()
-		if ok {
-			o = &fakeObject{
-				value: value,
-				fin:   fin,
-			}
-		}
-		return
-	case nsClosed:
-		ns.cache.Unlock()
-		return
-	}
-
-	n, ok := ns.table[key]
-	if ok {
-		n.ref++
-	} else {
-		if setf == nil {
-			ns.cache.Unlock()
-			return
-		}
-
-		var value interface{}
-		var fin func()
-		ok, value, _, fin = setf()
-		if !ok {
-			ns.cache.Unlock()
-			return
-		}
-
-		n = &emptyNode{
-			ns:     ns,
-			key:    key,
-			value:  value,
-			setfin: fin,
-			ref:    1,
-		}
-		ns.table[key] = n
-	}
-
-	ns.cache.Unlock()
-	o = &emptyObject{node: n}
-	return
-}
-
-func (ns *emptyNS) Delete(key uint64, fin DelFin) bool {
-	ns.cache.Lock()
-
-	if ns.state != nsEffective {
-		ns.cache.Unlock()
-		if fin != nil {
-			fin(false)
-		}
-		return false
-	}
-
-	n, ok := ns.table[key]
-	if !ok {
-		ns.cache.Unlock()
-		if fin != nil {
-			fin(false)
-		}
-		return false
-	}
-	n.delfin = fin
-	ns.cache.Unlock()
-	return true
-}
-
-func (ns *emptyNS) purgeNB(fin PurgeFin) {
-	if ns.state != nsEffective {
-		return
-	}
-	for _, n := range ns.table {
-		n.purgefin = fin
-	}
-}
-
-func (ns *emptyNS) Purge(fin PurgeFin) {
-	ns.cache.Lock()
-	ns.purgeNB(fin)
-	ns.cache.Unlock()
-}
-
-func (ns *emptyNS) zapNB(closed bool) {
-	if ns.state != nsEffective {
-		return
-	}
-	for _, n := range ns.table {
-		n.execFin()
-	}
-	if closed {
-		ns.state = nsClosed
-	} else {
-		ns.state = nsZapped
-	}
-	ns.table = nil
-}
-
-func (ns *emptyNS) Zap(closed bool) {
-	ns.cache.Lock()
-	ns.zapNB(closed)
-	delete(ns.cache.table, ns.id)
-	ns.cache.Unlock()
-}
-
-type emptyNode struct {
-	ns       *emptyNS
-	key      uint64
-	value    interface{}
-	ref      int
-	setfin   SetFin
-	delfin   DelFin
-	purgefin PurgeFin
-}
-
-func (n *emptyNode) execFin() {
-	if n.setfin != nil {
-		n.setfin()
-		n.setfin = nil
-	}
-	if n.purgefin != nil {
-		n.purgefin(n.ns.id, n.key, n.delfin)
-		n.delfin = nil
-		n.purgefin = nil
-	} else if n.delfin != nil {
-		n.delfin(true)
-		n.delfin = nil
-	}
-}
-
-func (n *emptyNode) evict() {
-	n.ns.cache.Lock()
-	n.ref--
-	if n.ref == 0 {
-		if n.ns.state == nsEffective {
-			// Remove elem.
-			delete(n.ns.table, n.key)
-			// Execute finalizer.
-			n.execFin()
-		}
-	} else if n.ref < 0 {
-		panic("leveldb/cache: emptyNode: negative node reference")
-	}
-	n.ns.cache.Unlock()
-}
-
-type emptyObject struct {
-	node *emptyNode
-	once uint32
-}
-
-func (o *emptyObject) Value() interface{} {
-	if atomic.LoadUint32(&o.once) == 0 {
-		return o.node.value
-	}
-	return nil
-}
-
-func (o *emptyObject) Release() {
-	if !atomic.CompareAndSwapUint32(&o.once, 0, 1) {
-		return
-	}
-	o.node.evict()
-	o.node = nil
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru_cache.go

@@ -9,16 +9,24 @@ package cache
 import (
 	"sync"
 	"sync/atomic"
+
+	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
+// The LLRB implementation were taken from https://github.com/petar/GoLLRB.
+// Which contains the following header:
+//
+// Copyright 2010 Petar Maymounkov. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 // lruCache represent a LRU cache state.
 type lruCache struct {
-	sync.Mutex
-
-	recent   lruNode
-	table    map[uint64]*lruNs
-	capacity int
-	size     int
+	mu                sync.Mutex
+	recent            lruNode
+	table             map[uint64]*lruNs
+	capacity          int
+	used, size, alive int
 }
 
 // NewLRUCache creates a new initialized LRU cache with the given capacity.
@@ -32,245 +40,415 @@ func NewLRUCache(capacity int) Cache {
 	return c
 }
 
+func (c *lruCache) Capacity() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.capacity
+}
+
+func (c *lruCache) Used() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.used
+}
+
+func (c *lruCache) Size() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.size
+}
+
+func (c *lruCache) NumObjects() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.alive
+}
+
 // SetCapacity set cache capacity.
 func (c *lruCache) SetCapacity(capacity int) {
-	c.Lock()
+	c.mu.Lock()
 	c.capacity = capacity
 	c.evict()
-	c.Unlock()
+	c.mu.Unlock()
 }
 
 // GetNamespace return namespace object for given id.
 func (c *lruCache) GetNamespace(id uint64) Namespace {
-	c.Lock()
-	defer c.Unlock()
+	c.mu.Lock()
+	defer c.mu.Unlock()
 
-	if p, ok := c.table[id]; ok {
-		return p
+	if ns, ok := c.table[id]; ok {
+		return ns
 	}
 
-	p := &lruNs{
-		lru:   c,
-		id:    id,
-		table: make(map[uint64]*lruNode),
+	ns := &lruNs{lru: c, id: id}
+	c.table[id] = ns
+	return ns
+}
+
+func (c *lruCache) ZapNamespace(id uint64) {
+	c.mu.Lock()
+	if ns, exist := c.table[id]; exist {
+		ns.zapNB()
+		delete(c.table, id)
 	}
-	c.table[id] = p
-	return p
+	c.mu.Unlock()
+}
+
+func (c *lruCache) PurgeNamespace(id uint64, fin PurgeFin) {
+	c.mu.Lock()
+	if ns, exist := c.table[id]; exist {
+		ns.purgeNB(fin)
+	}
+	c.mu.Unlock()
 }
 
 // Purge purge entire cache.
 func (c *lruCache) Purge(fin PurgeFin) {
-	c.Lock()
+	c.mu.Lock()
 	for _, ns := range c.table {
 		ns.purgeNB(fin)
 	}
-	c.Unlock()
+	c.mu.Unlock()
 }
 
-func (c *lruCache) Zap(closed bool) {
-	c.Lock()
+func (c *lruCache) Zap() {
+	c.mu.Lock()
 	for _, ns := range c.table {
-		ns.zapNB(closed)
+		ns.zapNB()
 	}
 	c.table = make(map[uint64]*lruNs)
-	c.Unlock()
+	c.mu.Unlock()
 }
 
 func (c *lruCache) evict() {
 	top := &c.recent
-	for n := c.recent.rPrev; c.size > c.capacity && n != top; {
+	for n := c.recent.rPrev; c.used > c.capacity && n != top; {
+		if n.state != nodeEffective {
+			panic("evicting non effective node")
+		}
 		n.state = nodeEvicted
 		n.rRemove()
-		n.evictNB()
-		c.size -= n.charge
+		n.derefNB()
+		c.used -= n.charge
 		n = c.recent.rPrev
 	}
 }
 
 type lruNs struct {
-	lru   *lruCache
-	id    uint64
-	table map[uint64]*lruNode
-	state nsState
+	lru    *lruCache
+	id     uint64
+	rbRoot *lruNode
+	state  nsState
 }
 
-func (ns *lruNs) Get(key uint64, setf SetFunc) (o Object, ok bool) {
-	lru := ns.lru
-	lru.Lock()
-
-	switch ns.state {
-	case nsZapped:
-		lru.Unlock()
-		if setf == nil {
-			return
-		}
-
-		var value interface{}
-		var fin func()
-		ok, value, _, fin = setf()
-		if ok {
-			o = &fakeObject{
-				value: value,
-				fin:   fin,
-			}
-		}
-		return
-	case nsClosed:
-		lru.Unlock()
-		return
+func (ns *lruNs) rbGetOrCreateNode(h *lruNode, key uint64) (hn, n *lruNode) {
+	if h == nil {
+		n = &lruNode{ns: ns, key: key}
+		return n, n
 	}
 
-	n, ok := ns.table[key]
-	if ok {
-		switch n.state {
-		case nodeEvicted:
-			// Insert to recent list.
-			n.state = nodeEffective
-			n.ref++
-			lru.size += n.charge
-			lru.evict()
-			fallthrough
-		case nodeEffective:
-			// Bump to front
-			n.rRemove()
-			n.rInsert(&lru.recent)
+	if key < h.key {
+		hn, n = ns.rbGetOrCreateNode(h.rbLeft, key)
+		if hn != nil {
+			h.rbLeft = hn
+		} else {
+			return nil, n
+		}
+	} else if key > h.key {
+		hn, n = ns.rbGetOrCreateNode(h.rbRight, key)
+		if hn != nil {
+			h.rbRight = hn
+		} else {
+			return nil, n
 		}
-		n.ref++
 	} else {
-		if setf == nil {
-			lru.Unlock()
-			return
-		}
-
-		var value interface{}
-		var charge int
-		var fin func()
-		ok, value, charge, fin = setf()
-		if !ok {
-			lru.Unlock()
-			return
-		}
-
-		n = &lruNode{
-			ns:     ns,
-			key:    key,
-			value:  value,
-			charge: charge,
-			setfin: fin,
-			ref:    2,
-		}
-		ns.table[key] = n
-		n.rInsert(&lru.recent)
-
-		lru.size += charge
-		lru.evict()
+		return nil, h
 	}
 
-	lru.Unlock()
-	o = &lruObject{node: n}
-	return
+	if rbIsRed(h.rbRight) && !rbIsRed(h.rbLeft) {
+		h = rbRotLeft(h)
+	}
+	if rbIsRed(h.rbLeft) && rbIsRed(h.rbLeft.rbLeft) {
+		h = rbRotRight(h)
+	}
+	if rbIsRed(h.rbLeft) && rbIsRed(h.rbRight) {
+		rbFlip(h)
+	}
+	return h, n
+}
+
+func (ns *lruNs) getOrCreateNode(key uint64) *lruNode {
+	hn, n := ns.rbGetOrCreateNode(ns.rbRoot, key)
+	if hn != nil {
+		ns.rbRoot = hn
+		ns.rbRoot.rbBlack = true
+	}
+	return n
+}
+
+func (ns *lruNs) rbGetNode(key uint64) *lruNode {
+	h := ns.rbRoot
+	for h != nil {
+		switch {
+		case key < h.key:
+			h = h.rbLeft
+		case key > h.key:
+			h = h.rbRight
+		default:
+			return h
+		}
+	}
+	return nil
+}
+
+func (ns *lruNs) getNode(key uint64) *lruNode {
+	return ns.rbGetNode(key)
+}
+
+func (ns *lruNs) rbDeleteNode(h *lruNode, key uint64) *lruNode {
+	if h == nil {
+		return nil
+	}
+
+	if key < h.key {
+		if h.rbLeft == nil { // key not present. Nothing to delete
+			return h
+		}
+		if !rbIsRed(h.rbLeft) && !rbIsRed(h.rbLeft.rbLeft) {
+			h = rbMoveLeft(h)
+		}
+		h.rbLeft = ns.rbDeleteNode(h.rbLeft, key)
+	} else {
+		if rbIsRed(h.rbLeft) {
+			h = rbRotRight(h)
+		}
+		// If @key equals @h.key and no right children at @h
+		if h.key == key && h.rbRight == nil {
+			return nil
+		}
+		if h.rbRight != nil && !rbIsRed(h.rbRight) && !rbIsRed(h.rbRight.rbLeft) {
+			h = rbMoveRight(h)
+		}
+		// If @key equals @h.key, and (from above) 'h.Right != nil'
+		if h.key == key {
+			var x *lruNode
+			h.rbRight, x = rbDeleteMin(h.rbRight)
+			if x == nil {
+				panic("logic")
+			}
+			x.rbLeft, h.rbLeft = h.rbLeft, nil
+			x.rbRight, h.rbRight = h.rbRight, nil
+			x.rbBlack = h.rbBlack
+			h = x
+		} else { // Else, @key is bigger than @h.key
+			h.rbRight = ns.rbDeleteNode(h.rbRight, key)
+		}
+	}
+
+	return rbFixup(h)
+}
+
+func (ns *lruNs) deleteNode(key uint64) {
+	ns.rbRoot = ns.rbDeleteNode(ns.rbRoot, key)
+	if ns.rbRoot != nil {
+		ns.rbRoot.rbBlack = true
+	}
+}
+
+func (ns *lruNs) rbIterateNodes(h *lruNode, pivot uint64, iter func(n *lruNode) bool) bool {
+	if h == nil {
+		return true
+	}
+	if h.key >= pivot {
+		if !ns.rbIterateNodes(h.rbLeft, pivot, iter) {
+			return false
+		}
+		if !iter(h) {
+			return false
+		}
+	}
+	return ns.rbIterateNodes(h.rbRight, pivot, iter)
+}
+
+func (ns *lruNs) iterateNodes(iter func(n *lruNode) bool) {
+	ns.rbIterateNodes(ns.rbRoot, 0, iter)
+}
+
+func (ns *lruNs) Get(key uint64, setf SetFunc) Handle {
+	ns.lru.mu.Lock()
+	defer ns.lru.mu.Unlock()
+
+	if ns.state != nsEffective {
+		return nil
+	}
+
+	var n *lruNode
+	if setf == nil {
+		n = ns.getNode(key)
+		if n == nil {
+			return nil
+		}
+	} else {
+		n = ns.getOrCreateNode(key)
+	}
+	switch n.state {
+	case nodeZero:
+		charge, value := setf()
+		if value == nil {
+			ns.deleteNode(key)
+			return nil
+		}
+		if charge < 0 {
+			charge = 0
+		}
+
+		n.value = value
+		n.charge = charge
+		n.state = nodeEvicted
+
+		ns.lru.size += charge
+		ns.lru.alive++
+
+		fallthrough
+	case nodeEvicted:
+		if n.charge == 0 {
+			break
+		}
+
+		// Insert to recent list.
+		n.state = nodeEffective
+		n.ref++
+		ns.lru.used += n.charge
+		ns.lru.evict()
+
+		fallthrough
+	case nodeEffective:
+		// Bump to front.
+		n.rRemove()
+		n.rInsert(&ns.lru.recent)
+	case nodeDeleted:
+		// Do nothing.
+	default:
+		panic("invalid state")
+	}
+	n.ref++
+
+	return &lruHandle{node: n}
 }
 
 func (ns *lruNs) Delete(key uint64, fin DelFin) bool {
-	lru := ns.lru
-	lru.Lock()
+	ns.lru.mu.Lock()
+	defer ns.lru.mu.Unlock()
 
 	if ns.state != nsEffective {
-		lru.Unlock()
 		if fin != nil {
-			fin(false)
+			fin(false, false)
 		}
 		return false
 	}
 
-	n, ok := ns.table[key]
-	if !ok {
-		lru.Unlock()
+	n := ns.getNode(key)
+	if n == nil {
 		if fin != nil {
-			fin(false)
+			fin(false, false)
 		}
 		return false
+
 	}
 
-	n.delfin = fin
 	switch n.state {
-	case nodeRemoved:
-		lru.Unlock()
-		return false
 	case nodeEffective:
-		lru.size -= n.charge
+		ns.lru.used -= n.charge
+		n.state = nodeDeleted
+		n.delfin = fin
 		n.rRemove()
-		n.evictNB()
+		n.derefNB()
+	case nodeEvicted:
+		n.state = nodeDeleted
+		n.delfin = fin
+	case nodeDeleted:
+		if fin != nil {
+			fin(true, true)
+		}
+		return false
+	default:
+		panic("invalid state")
 	}
-	n.state = nodeRemoved
 
-	lru.Unlock()
 	return true
 }
 
 func (ns *lruNs) purgeNB(fin PurgeFin) {
-	lru := ns.lru
-	if ns.state != nsEffective {
-		return
-	}
-
-	for _, n := range ns.table {
-		n.purgefin = fin
-		if n.state == nodeEffective {
-			lru.size -= n.charge
-			n.rRemove()
-			n.evictNB()
+	if ns.state == nsEffective {
+		var nodes []*lruNode
+		ns.iterateNodes(func(n *lruNode) bool {
+			nodes = append(nodes, n)
+			return true
+		})
+		for _, n := range nodes {
+			switch n.state {
+			case nodeEffective:
+				ns.lru.used -= n.charge
+				n.state = nodeDeleted
+				n.purgefin = fin
+				n.rRemove()
+				n.derefNB()
+			case nodeEvicted:
+				n.state = nodeDeleted
+				n.purgefin = fin
+			case nodeDeleted:
+			default:
+				panic("invalid state")
+			}
 		}
-		n.state = nodeRemoved
 	}
 }
 
 func (ns *lruNs) Purge(fin PurgeFin) {
-	ns.lru.Lock()
+	ns.lru.mu.Lock()
 	ns.purgeNB(fin)
-	ns.lru.Unlock()
+	ns.lru.mu.Unlock()
 }
 
-func (ns *lruNs) zapNB(closed bool) {
-	lru := ns.lru
-	if ns.state != nsEffective {
-		return
-	}
-
-	if closed {
-		ns.state = nsClosed
-	} else {
+func (ns *lruNs) zapNB() {
+	if ns.state == nsEffective {
 		ns.state = nsZapped
+
+		ns.iterateNodes(func(n *lruNode) bool {
+			if n.state == nodeEffective {
+				ns.lru.used -= n.charge
+				n.rRemove()
+			}
+			ns.lru.size -= n.charge
+			n.state = nodeDeleted
+			n.fin()
+
+			return true
+		})
+		ns.rbRoot = nil
 	}
-	for _, n := range ns.table {
-		if n.state == nodeEffective {
-			lru.size -= n.charge
-			n.rRemove()
-		}
-		n.state = nodeRemoved
-		n.execFin()
-	}
-	ns.table = nil
 }
 
-func (ns *lruNs) Zap(closed bool) {
-	ns.lru.Lock()
-	ns.zapNB(closed)
+func (ns *lruNs) Zap() {
+	ns.lru.mu.Lock()
+	ns.zapNB()
 	delete(ns.lru.table, ns.id)
-	ns.lru.Unlock()
+	ns.lru.mu.Unlock()
 }
 
 type lruNode struct {
 	ns *lruNs
 
-	rNext, rPrev *lruNode
+	rNext, rPrev    *lruNode
+	rbLeft, rbRight *lruNode
+	rbBlack         bool
 
 	key      uint64
 	value    interface{}
 	charge   int
 	ref      int
 	state    nodeState
-	setfin   SetFin
 	delfin   DelFin
 	purgefin PurgeFin
 }
@@ -284,7 +462,6 @@ func (n *lruNode) rInsert(at *lruNode) {
 }
 
 func (n *lruNode) rRemove() bool {
-	// only remove if not already removed
 	if n.rPrev == nil {
 		return false
 	}
@@ -297,58 +474,149 @@ func (n *lruNode) rRemove() bool {
 	return true
 }
 
-func (n *lruNode) execFin() {
-	if n.setfin != nil {
-		n.setfin()
-		n.setfin = nil
+func (n *lruNode) fin() {
+	if r, ok := n.value.(util.Releaser); ok {
+		r.Release()
 	}
 	if n.purgefin != nil {
-		n.purgefin(n.ns.id, n.key, n.delfin)
-		n.delfin = nil
+		if n.delfin != nil {
+			panic("conflicting delete and purge fin")
+		}
+		n.purgefin(n.ns.id, n.key)
 		n.purgefin = nil
 	} else if n.delfin != nil {
-		n.delfin(true)
+		n.delfin(true, false)
 		n.delfin = nil
 	}
 }
 
-func (n *lruNode) evictNB() {
+func (n *lruNode) derefNB() {
 	n.ref--
 	if n.ref == 0 {
 		if n.ns.state == nsEffective {
-			// remove elem
-			delete(n.ns.table, n.key)
-			// execute finalizer
-			n.execFin()
+			// Remove elemement.
+			n.ns.deleteNode(n.key)
+			n.ns.lru.size -= n.charge
+			n.ns.lru.alive--
+			n.fin()
 		}
+		n.value = nil
 	} else if n.ref < 0 {
 		panic("leveldb/cache: lruCache: negative node reference")
 	}
 }
 
-func (n *lruNode) evict() {
-	n.ns.lru.Lock()
-	n.evictNB()
-	n.ns.lru.Unlock()
+func (n *lruNode) deref() {
+	n.ns.lru.mu.Lock()
+	n.derefNB()
+	n.ns.lru.mu.Unlock()
 }
 
-type lruObject struct {
+type lruHandle struct {
 	node *lruNode
 	once uint32
 }
 
-func (o *lruObject) Value() interface{} {
-	if atomic.LoadUint32(&o.once) == 0 {
-		return o.node.value
+func (h *lruHandle) Value() interface{} {
+	if atomic.LoadUint32(&h.once) == 0 {
+		return h.node.value
 	}
 	return nil
 }
 
-func (o *lruObject) Release() {
-	if !atomic.CompareAndSwapUint32(&o.once, 0, 1) {
+func (h *lruHandle) Release() {
+	if !atomic.CompareAndSwapUint32(&h.once, 0, 1) {
 		return
 	}
-
-	o.node.evict()
-	o.node = nil
+	h.node.deref()
+	h.node = nil
+}
+
+func rbIsRed(h *lruNode) bool {
+	if h == nil {
+		return false
+	}
+	return !h.rbBlack
+}
+
+func rbRotLeft(h *lruNode) *lruNode {
+	x := h.rbRight
+	if x.rbBlack {
+		panic("rotating a black link")
+	}
+	h.rbRight = x.rbLeft
+	x.rbLeft = h
+	x.rbBlack = h.rbBlack
+	h.rbBlack = false
+	return x
+}
+
+func rbRotRight(h *lruNode) *lruNode {
+	x := h.rbLeft
+	if x.rbBlack {
+		panic("rotating a black link")
+	}
+	h.rbLeft = x.rbRight
+	x.rbRight = h
+	x.rbBlack = h.rbBlack
+	h.rbBlack = false
+	return x
+}
+
+func rbFlip(h *lruNode) {
+	h.rbBlack = !h.rbBlack
+	h.rbLeft.rbBlack = !h.rbLeft.rbBlack
+	h.rbRight.rbBlack = !h.rbRight.rbBlack
+}
+
+func rbMoveLeft(h *lruNode) *lruNode {
+	rbFlip(h)
+	if rbIsRed(h.rbRight.rbLeft) {
+		h.rbRight = rbRotRight(h.rbRight)
+		h = rbRotLeft(h)
+		rbFlip(h)
+	}
+	return h
+}
+
+func rbMoveRight(h *lruNode) *lruNode {
+	rbFlip(h)
+	if rbIsRed(h.rbLeft.rbLeft) {
+		h = rbRotRight(h)
+		rbFlip(h)
+	}
+	return h
+}
+
+func rbFixup(h *lruNode) *lruNode {
+	if rbIsRed(h.rbRight) {
+		h = rbRotLeft(h)
+	}
+
+	if rbIsRed(h.rbLeft) && rbIsRed(h.rbLeft.rbLeft) {
+		h = rbRotRight(h)
+	}
+
+	if rbIsRed(h.rbLeft) && rbIsRed(h.rbRight) {
+		rbFlip(h)
+	}
+
+	return h
+}
+
+func rbDeleteMin(h *lruNode) (hn, n *lruNode) {
+	if h == nil {
+		return nil, nil
+	}
+	if h.rbLeft == nil {
+		return nil, h
+	}
+
+	if !rbIsRed(h.rbLeft) && !rbIsRed(h.rbLeft.rbLeft) {
+		h = rbMoveLeft(h)
+	}
+
+	h.rbLeft, n = rbDeleteMin(h.rbLeft)
+
+	return rbFixup(h), n
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go

@@ -14,6 +14,7 @@ import (
 	"runtime"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/syndtr/goleveldb/leveldb/iterator"
@@ -30,23 +31,27 @@ type DB struct {
 	// Need 64-bit alignment.
 	seq uint64
 
+	// Session.
 	s *session
 
-	// MemDB
+	// MemDB.
 	memMu             sync.RWMutex
-	mem               *memdb.DB
-	frozenMem         *memdb.DB
+	memPool           chan *memdb.DB
+	mem, frozenMem    *memDB
 	journal           *journal.Writer
 	journalWriter     storage.Writer
 	journalFile       storage.File
 	frozenJournalFile storage.File
 	frozenSeq         uint64
 
-	// Snapshot
+	// Snapshot.
 	snapsMu   sync.Mutex
 	snapsRoot snapshotElement
 
-	// Write
+	// Stats.
+	aliveSnaps, aliveIters int32
+
+	// Write.
 	writeC       chan *Batch
 	writeMergedC chan bool
 	writeLockC   chan struct{}
@@ -54,7 +59,7 @@ type DB struct {
 	journalC     chan *Batch
 	journalAckC  chan error
 
-	// Compaction
+	// Compaction.
 	tcompCmdC     chan cCmd
 	tcompPauseC   chan chan<- struct{}
 	tcompTriggerC chan struct{}
@@ -64,7 +69,7 @@ type DB struct {
 	compErrSetC   chan error
 	compStats     [kNumLevels]cStats
 
-	// Close
+	// Close.
 	closeW sync.WaitGroup
 	closeC chan struct{}
 	closed uint32
@@ -78,6 +83,8 @@ func openDB(s *session) (*DB, error) {
 		s: s,
 		// Initial sequence
 		seq: s.stSeq,
+		// MemDB
+		memPool: make(chan *memdb.DB, 1),
 		// Write
 		writeC:       make(chan *Batch),
 		writeMergedC: make(chan bool),
@@ -119,6 +126,7 @@ func openDB(s *session) (*DB, error) {
 	go db.tCompaction()
 	go db.mCompaction()
 	go db.jWriter()
+	go db.mpoolDrain()
 
 	s.logf("db@open done T·%v", time.Since(start))
 
@@ -135,9 +143,10 @@ func openDB(s *session) (*DB, error) {
 // detected in the DB. Corrupted DB can be recovered with Recover
 // function.
 //
+// The returned DB instance is goroutine-safe.
 // The DB must be closed after use, by calling Close method.
-func Open(p storage.Storage, o *opt.Options) (db *DB, err error) {
-	s, err := newSession(p, o)
+func Open(stor storage.Storage, o *opt.Options) (db *DB, err error) {
+	s, err := newSession(stor, o)
 	if err != nil {
 		return
 	}
@@ -177,6 +186,7 @@ func Open(p storage.Storage, o *opt.Options) (db *DB, err error) {
 // detected in the DB. Corrupted DB can be recovered with Recover
 // function.
 //
+// The returned DB instance is goroutine-safe.
 // The DB must be closed after use, by calling Close method.
 func OpenFile(path string, o *opt.Options) (db *DB, err error) {
 	stor, err := storage.OpenFile(path)
@@ -197,9 +207,10 @@ func OpenFile(path string, o *opt.Options) (db *DB, err error) {
 // The DB must already exist or it will returns an error.
 // Also, Recover will ignore ErrorIfMissing and ErrorIfExist options.
 //
+// The returned DB instance is goroutine-safe.
 // The DB must be closed after use, by calling Close method.
-func Recover(p storage.Storage, o *opt.Options) (db *DB, err error) {
-	s, err := newSession(p, o)
+func Recover(stor storage.Storage, o *opt.Options) (db *DB, err error) {
+	s, err := newSession(stor, o)
 	if err != nil {
 		return
 	}
@@ -225,6 +236,7 @@ func Recover(p storage.Storage, o *opt.Options) (db *DB, err error) {
 // RecoverFile uses standard file-system backed storage implementation as desribed
 // in the leveldb/storage package.
 //
+// The returned DB instance is goroutine-safe.
 // The DB must be closed after use, by calling Close method.
 func RecoverFile(path string, o *opt.Options) (db *DB, err error) {
 	stor, err := storage.OpenFile(path)
@@ -241,16 +253,18 @@ func RecoverFile(path string, o *opt.Options) (db *DB, err error) {
 }
 
 func recoverTable(s *session, o *opt.Options) error {
-	ff0, err := s.getFiles(storage.TypeTable)
+	// Get all tables and sort it by file number.
+	tableFiles_, err := s.getFiles(storage.TypeTable)
 	if err != nil {
 		return err
 	}
-	ff1 := files(ff0)
-	ff1.sort()
+	tableFiles := files(tableFiles_)
+	tableFiles.sort()
 
 	var mSeq uint64
 	var good, corrupted int
 	rec := new(sessionRecord)
+	bpool := util.NewBufferPool(o.GetBlockSize() + 5)
 	buildTable := func(iter iterator.Iterator) (tmp storage.File, size int64, err error) {
 		tmp = s.newTemp()
 		writer, err := tmp.Create()
@@ -264,8 +278,9 @@ func recoverTable(s *session, o *opt.Options) error {
 				tmp = nil
 			}
 		}()
+
+		// Copy entries.
 		tw := table.NewWriter(writer, o)
-		// Copy records.
 		for iter.Next() {
 			key := iter.Key()
 			if validIkey(key) {
@@ -297,20 +312,23 @@ func recoverTable(s *session, o *opt.Options) error {
 			return err
 		}
 		defer reader.Close()
+
 		// Get file size.
 		size, err := reader.Seek(0, 2)
 		if err != nil {
 			return err
 		}
+
 		var tSeq uint64
 		var tgood, tcorrupted, blockerr int
-		var min, max []byte
-		tr := table.NewReader(reader, size, nil, o)
+		var imin, imax []byte
+		tr := table.NewReader(reader, size, nil, bpool, o)
 		iter := tr.NewIterator(nil, nil)
 		iter.(iterator.ErrorCallbackSetter).SetErrorCallback(func(err error) {
 			s.logf("table@recovery found error @%d %q", file.Num(), err)
 			blockerr++
 		})
+
 		// Scan the table.
 		for iter.Next() {
 			key := iter.Key()
@@ -323,16 +341,17 @@ func recoverTable(s *session, o *opt.Options) error {
 			if seq > tSeq {
 				tSeq = seq
 			}
-			if min == nil {
-				min = append([]byte{}, key...)
+			if imin == nil {
+				imin = append([]byte{}, key...)
 			}
-			max = append(max[:0], key...)
+			imax = append(imax[:0], key...)
 		}
 		if err := iter.Error(); err != nil {
 			iter.Release()
 			return err
 		}
 		iter.Release()
+
 		if tgood > 0 {
 			if tcorrupted > 0 || blockerr > 0 {
 				// Rebuild the table.
@@ -353,7 +372,7 @@ func recoverTable(s *session, o *opt.Options) error {
 				mSeq = tSeq
 			}
 			// Add table to level 0.
-			rec.addTable(0, file.Num(), uint64(size), min, max)
+			rec.addTable(0, file.Num(), uint64(size), imin, imax)
 			s.logf("table@recovery recovered @%d N·%d C·%d B·%d S·%d Q·%d", file.Num(), tgood, tcorrupted, blockerr, size, tSeq)
 		} else {
 			s.logf("table@recovery unrecoverable @%d C·%d B·%d S·%d", file.Num(), tcorrupted, blockerr, size)
@@ -364,41 +383,56 @@ func recoverTable(s *session, o *opt.Options) error {
 
 		return nil
 	}
+
 	// Recover all tables.
-	if len(ff1) > 0 {
-		s.logf("table@recovery F·%d", len(ff1))
-		s.markFileNum(ff1[len(ff1)-1].Num())
-		for _, file := range ff1 {
+	if len(tableFiles) > 0 {
+		s.logf("table@recovery F·%d", len(tableFiles))
+
+		// Mark file number as used.
+		s.markFileNum(tableFiles[len(tableFiles)-1].Num())
+
+		for _, file := range tableFiles {
 			if err := recoverTable(file); err != nil {
 				return err
 			}
 		}
-		s.logf("table@recovery recovered F·%d N·%d C·%d Q·%d", len(ff1), good, corrupted, mSeq)
+
+		s.logf("table@recovery recovered F·%d N·%d C·%d Q·%d", len(tableFiles), good, corrupted, mSeq)
 	}
+
 	// Set sequence number.
 	rec.setSeq(mSeq + 1)
+
 	// Create new manifest.
 	if err := s.create(); err != nil {
 		return err
 	}
+
 	// Commit.
 	return s.commit(rec)
 }
 
-func (d *DB) recoverJournal() error {
-	s := d.s
-
-	ff0, err := s.getFiles(storage.TypeJournal)
+func (db *DB) recoverJournal() error {
+	// Get all tables and sort it by file number.
+	journalFiles_, err := db.s.getFiles(storage.TypeJournal)
 	if err != nil {
 		return err
 	}
-	ff1 := files(ff0)
-	ff1.sort()
-	ff2 := make([]storage.File, 0, len(ff1))
-	for _, file := range ff1 {
-		if file.Num() >= s.stJournalNum || file.Num() == s.stPrevJournalNum {
-			s.markFileNum(file.Num())
-			ff2 = append(ff2, file)
+	journalFiles := files(journalFiles_)
+	journalFiles.sort()
+
+	// Discard older journal.
+	prev := -1
+	for i, file := range journalFiles {
+		if file.Num() >= db.s.stJournalNum {
+			if prev >= 0 {
+				i--
+				journalFiles[i] = journalFiles[prev]
+			}
+			journalFiles = journalFiles[i:]
+			break
+		} else if file.Num() == db.s.stPrevJournalNum {
+			prev = i
 		}
 	}
 
@@ -406,38 +440,43 @@ func (d *DB) recoverJournal() error {
 	var of storage.File
 	var mem *memdb.DB
 	batch := new(Batch)
-	cm := newCMem(s)
+	cm := newCMem(db.s)
 	buf := new(util.Buffer)
 	// Options.
-	strict := s.o.GetStrict(opt.StrictJournal)
-	checksum := s.o.GetStrict(opt.StrictJournalChecksum)
-	writeBuffer := s.o.GetWriteBuffer()
+	strict := db.s.o.GetStrict(opt.StrictJournal)
+	checksum := db.s.o.GetStrict(opt.StrictJournalChecksum)
+	writeBuffer := db.s.o.GetWriteBuffer()
 	recoverJournal := func(file storage.File) error {
-		s.logf("journal@recovery recovering @%d", file.Num())
+		db.logf("journal@recovery recovering @%d", file.Num())
 		reader, err := file.Open()
 		if err != nil {
 			return err
 		}
 		defer reader.Close()
+
+		// Create/reset journal reader instance.
 		if jr == nil {
-			jr = journal.NewReader(reader, dropper{s, file}, strict, checksum)
+			jr = journal.NewReader(reader, dropper{db.s, file}, strict, checksum)
 		} else {
-			jr.Reset(reader, dropper{s, file}, strict, checksum)
+			jr.Reset(reader, dropper{db.s, file}, strict, checksum)
 		}
+
+		// Flush memdb and remove obsolete journal file.
 		if of != nil {
 			if mem.Len() > 0 {
 				if err := cm.flush(mem, 0); err != nil {
 					return err
 				}
 			}
-			if err := cm.commit(file.Num(), d.seq); err != nil {
+			if err := cm.commit(file.Num(), db.seq); err != nil {
 				return err
 			}
 			cm.reset()
 			of.Remove()
 			of = nil
 		}
-		// Reset memdb.
+
+		// Replay journal to memdb.
 		mem.Reset()
 		for {
 			r, err := jr.Next()
@@ -447,12 +486,14 @@ func (d *DB) recoverJournal() error {
 				}
 				return err
 			}
+
 			buf.Reset()
 			if _, err := buf.ReadFrom(r); err != nil {
-				if strict {
+				if err == io.ErrUnexpectedEOF {
+					continue
+				} else {
 					return err
 				}
-				continue
 			}
 			if err := batch.decode(buf.Bytes()); err != nil {
 				return err
@@ -460,28 +501,37 @@ func (d *DB) recoverJournal() error {
 			if err := batch.memReplay(mem); err != nil {
 				return err
 			}
-			d.seq = batch.seq + uint64(batch.len())
+
+			// Save sequence number.
+			db.seq = batch.seq + uint64(batch.len())
+
+			// Flush it if large enough.
 			if mem.Size() >= writeBuffer {
-				// Large enough, flush it.
 				if err := cm.flush(mem, 0); err != nil {
 					return err
 				}
-				// Reset memdb.
 				mem.Reset()
 			}
 		}
+
 		of = file
 		return nil
 	}
+
 	// Recover all journals.
-	if len(ff2) > 0 {
-		s.logf("journal@recovery F·%d", len(ff2))
-		mem = memdb.New(s.icmp, writeBuffer)
-		for _, file := range ff2 {
+	if len(journalFiles) > 0 {
+		db.logf("journal@recovery F·%d", len(journalFiles))
+
+		// Mark file number as used.
+		db.s.markFileNum(journalFiles[len(journalFiles)-1].Num())
+
+		mem = memdb.New(db.s.icmp, writeBuffer)
+		for _, file := range journalFiles {
 			if err := recoverJournal(file); err != nil {
 				return err
 			}
 		}
+
 		// Flush the last journal.
 		if mem.Len() > 0 {
 			if err := cm.flush(mem, 0); err != nil {
@@ -489,51 +539,60 @@ func (d *DB) recoverJournal() error {
 			}
 		}
 	}
+
 	// Create a new journal.
-	if _, err := d.newMem(0); err != nil {
+	if _, err := db.newMem(0); err != nil {
 		return err
 	}
+
 	// Commit.
-	if err := cm.commit(d.journalFile.Num(), d.seq); err != nil {
+	if err := cm.commit(db.journalFile.Num(), db.seq); err != nil {
+		// Close journal.
+		if db.journal != nil {
+			db.journal.Close()
+			db.journalWriter.Close()
+		}
 		return err
 	}
-	// Remove the last journal.
+
+	// Remove the last obsolete journal file.
 	if of != nil {
 		of.Remove()
 	}
+
 	return nil
 }
 
-func (d *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, err error) {
-	s := d.s
-
+func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, err error) {
 	ikey := newIKey(key, seq, tSeek)
 
-	em, fm := d.getMems()
-	for _, m := range [...]*memdb.DB{em, fm} {
+	em, fm := db.getMems()
+	for _, m := range [...]*memDB{em, fm} {
 		if m == nil {
 			continue
 		}
-		mk, mv, me := m.Find(ikey)
+		defer m.decref()
+
+		mk, mv, me := m.mdb.Find(ikey)
 		if me == nil {
 			ukey, _, t, ok := parseIkey(mk)
-			if ok && s.icmp.uCompare(ukey, key) == 0 {
+			if ok && db.s.icmp.uCompare(ukey, key) == 0 {
 				if t == tDel {
 					return nil, ErrNotFound
 				}
-				return mv, nil
+				return append([]byte{}, mv...), nil
 			}
 		} else if me != ErrNotFound {
 			return nil, me
 		}
 	}
 
-	v := s.version()
+	v := db.s.version()
 	value, cSched, err := v.get(ikey, ro)
 	v.release()
 	if cSched {
 		// Trigger table compaction.
-		d.compTrigger(d.tcompTriggerC)
+		db.compTrigger(db.tcompTriggerC)
 	}
 	return
 }
@@ -541,15 +600,16 @@ func (d *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, err
 // Get gets the value for the given key. It returns ErrNotFound if the
 // DB does not contain the key.
 //
-// The caller should not modify the contents of the returned slice, but
-// it is safe to modify the contents of the argument after Get returns.
-func (d *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
-	err = d.ok()
+// The returned slice is its own copy, it is safe to modify the contents
+// of the returned slice.
+// It is safe to modify the contents of the argument after Get returns.
+func (db *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
+	err = db.ok()
 	if err != nil {
 		return
 	}
 
-	return d.get(key, d.getSeq(), ro)
+	return db.get(key, db.getSeq(), ro)
 }
 
 // NewIterator returns an iterator for the latest snapshot of the
@@ -568,14 +628,14 @@ func (d *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
 // The iterator must be released after use, by calling Release method.
 //
 // Also read Iterator documentation of the leveldb/iterator package.
-func (d *DB) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
-	if err := d.ok(); err != nil {
+func (db *DB) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
+	if err := db.ok(); err != nil {
 		return iterator.NewEmptyIterator(err)
 	}
 
-	p := d.newSnapshot()
-	defer p.Release()
-	return p.NewIterator(slice, ro)
+	snap := db.newSnapshot()
+	defer snap.Release()
+	return snap.NewIterator(slice, ro)
 }
 
 // GetSnapshot returns a latest snapshot of the underlying DB. A snapshot
@@ -583,12 +643,12 @@ func (d *DB) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterat
 // content of snapshot are guaranteed to be consistent.
 //
 // The snapshot must be released after use, by calling Release method.
-func (d *DB) GetSnapshot() (*Snapshot, error) {
-	if err := d.ok(); err != nil {
+func (db *DB) GetSnapshot() (*Snapshot, error) {
+	if err := db.ok(); err != nil {
 		return nil, err
 	}
 
-	return d.newSnapshot(), nil
+	return db.newSnapshot(), nil
 }
 
 // GetProperty returns value of the given property name.
@@ -600,8 +660,18 @@ func (d *DB) GetSnapshot() (*Snapshot, error) {
 //		Returns statistics of the underlying DB.
 //	leveldb.sstables
 //		Returns sstables list for each level.
-func (d *DB) GetProperty(name string) (value string, err error) {
-	err = d.ok()
+//	leveldb.blockpool
+//		Returns block pool stats.
+//	leveldb.cachedblock
+//		Returns size of cached block.
+//	leveldb.openedtables
+//		Returns number of opened tables.
+//	leveldb.alivesnaps
+//		Returns number of alive snapshots.
+//	leveldb.aliveiters
+//		Returns number of alive iterators.
+func (db *DB) GetProperty(name string) (value string, err error) {
+	err = db.ok()
 	if err != nil {
 		return
 	}
@@ -610,11 +680,9 @@ func (d *DB) GetProperty(name string) (value string, err error) {
 	if !strings.HasPrefix(name, prefix) {
 		return "", errors.New("leveldb: GetProperty: unknown property: " + name)
 	}
-
 	p := name[len(prefix):]
 
-	s := d.s
-	v := s.version()
+	v := db.s.version()
 	defer v.release()
 
 	switch {
@@ -631,22 +699,36 @@ func (d *DB) GetProperty(name string) (value string, err error) {
 		value = "Compactions\n" +
 			" Level |   Tables   |    Size(MB)   |    Time(sec)  |    Read(MB)   |   Write(MB)\n" +
 			"-------+------------+---------------+---------------+---------------+---------------\n"
-		for level, tt := range v.tables {
-			duration, read, write := d.compStats[level].get()
-			if len(tt) == 0 && duration == 0 {
+		for level, tables := range v.tables {
+			duration, read, write := db.compStats[level].get()
+			if len(tables) == 0 && duration == 0 {
 				continue
 			}
 			value += fmt.Sprintf(" %3d   | %10d | %13.5f | %13.5f | %13.5f | %13.5f\n",
-				level, len(tt), float64(tt.size())/1048576.0, duration.Seconds(),
+				level, len(tables), float64(tables.size())/1048576.0, duration.Seconds(),
 				float64(read)/1048576.0, float64(write)/1048576.0)
 		}
 	case p == "sstables":
-		for level, tt := range v.tables {
+		for level, tables := range v.tables {
 			value += fmt.Sprintf("--- level %d ---\n", level)
-			for _, t := range tt {
-				value += fmt.Sprintf("%d:%d[%q .. %q]\n", t.file.Num(), t.size, t.min, t.max)
+			for _, t := range tables {
+				value += fmt.Sprintf("%d:%d[%q .. %q]\n", t.file.Num(), t.size, t.imin, t.imax)
 			}
 		}
+	case p == "blockpool":
+		value = fmt.Sprintf("%v", db.s.tops.bpool)
+	case p == "cachedblock":
+		if bc := db.s.o.GetBlockCache(); bc != nil {
+			value = fmt.Sprintf("%d", bc.Size())
+		} else {
+			value = "<nil>"
+		}
+	case p == "openedtables":
+		value = fmt.Sprintf("%d", db.s.tops.cache.Size())
+	case p == "alivesnaps":
+		value = fmt.Sprintf("%d", atomic.LoadInt32(&db.aliveSnaps))
+	case p == "aliveiters":
+		value = fmt.Sprintf("%d", atomic.LoadInt32(&db.aliveIters))
 	default:
 		err = errors.New("leveldb: GetProperty: unknown property: " + name)
 	}
@@ -660,23 +742,23 @@ func (d *DB) GetProperty(name string) (value string, err error) {
 // data compresses by a factor of ten, the returned sizes will be one-tenth
 // the size of the corresponding user data size.
 // The results may not include the sizes of recently written data.
-func (d *DB) SizeOf(ranges []util.Range) (Sizes, error) {
-	if err := d.ok(); err != nil {
+func (db *DB) SizeOf(ranges []util.Range) (Sizes, error) {
+	if err := db.ok(); err != nil {
 		return nil, err
 	}
 
-	v := d.s.version()
+	v := db.s.version()
 	defer v.release()
 
 	sizes := make(Sizes, 0, len(ranges))
 	for _, r := range ranges {
-		min := newIKey(r.Start, kMaxSeq, tSeek)
-		max := newIKey(r.Limit, kMaxSeq, tSeek)
-		start, err := v.offsetOf(min)
+		imin := newIKey(r.Start, kMaxSeq, tSeek)
+		imax := newIKey(r.Limit, kMaxSeq, tSeek)
+		start, err := v.offsetOf(imin)
 		if err != nil {
 			return nil, err
 		}
-		limit, err := v.offsetOf(max)
+		limit, err := v.offsetOf(imax)
 		if err != nil {
 			return nil, err
 		}
@@ -690,61 +772,63 @@ func (d *DB) SizeOf(ranges []util.Range) (Sizes, error) {
 	return sizes, nil
 }
 
-// Close closes the DB. This will also releases any outstanding snapshot.
+// Close closes the DB. This will also releases any outstanding snapshot and
+// abort any in-flight compaction.
 //
 // It is not safe to close a DB until all outstanding iterators are released.
 // It is valid to call Close multiple times. Other methods should not be
 // called after the DB has been closed.
-func (d *DB) Close() error {
-	if !d.setClosed() {
+func (db *DB) Close() error {
+	if !db.setClosed() {
 		return ErrClosed
 	}
 
-	s := d.s
 	start := time.Now()
-	s.log("db@close closing")
+	db.log("db@close closing")
 
 	// Clear the finalizer.
-	runtime.SetFinalizer(d, nil)
+	runtime.SetFinalizer(db, nil)
 
 	// Get compaction error.
 	var err error
 	select {
-	case err = <-d.compErrC:
+	case err = <-db.compErrC:
 	default:
 	}
 
-	close(d.closeC)
+	close(db.closeC)
 
 	// Wait for the close WaitGroup.
-	d.closeW.Wait()
+	db.closeW.Wait()
 
 	// Close journal.
-	if d.journal != nil {
-		d.journal.Close()
-		d.journalWriter.Close()
+	db.writeLockC <- struct{}{}
+	if db.journal != nil {
+		db.journal.Close()
+		db.journalWriter.Close()
 	}
 
 	// Close session.
-	s.close()
-	s.logf("db@close done T·%v", time.Since(start))
-	s.release()
+	db.s.close()
+	db.logf("db@close done T·%v", time.Since(start))
+	db.s.release()
 
-	if d.closer != nil {
-		if err1 := d.closer.Close(); err == nil {
+	if db.closer != nil {
+		if err1 := db.closer.Close(); err == nil {
 			err = err1
 		}
 	}
 
-	d.s = nil
-	d.mem = nil
-	d.frozenMem = nil
-	d.journal = nil
-	d.journalWriter = nil
-	d.journalFile = nil
-	d.frozenJournalFile = nil
-	d.snapsRoot = snapshotElement{}
-	d.closer = nil
+	// NIL'ing pointers.
+	db.s = nil
+	db.mem = nil
+	db.frozenMem = nil
+	db.journal = nil
+	db.journalWriter = nil
+	db.journalFile = nil
+	db.frozenJournalFile = nil
+	db.snapsRoot = snapshotElement{}
+	db.closer = nil
 
 	return err
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go

@@ -74,7 +74,7 @@ func newCMem(s *session) *cMem {
 func (c *cMem) flush(mem *memdb.DB, level int) error {
 	s := c.s
 
-	// Write memdb to table
+	// Write memdb to table.
 	iter := mem.NewIterator(nil)
 	defer iter.Release()
 	t, n, err := s.tops.createFrom(iter)
@@ -82,12 +82,13 @@ func (c *cMem) flush(mem *memdb.DB, level int) error {
 		return err
 	}
 
+	// Pick level.
 	if level < 0 {
-		level = s.version_NB().pickLevel(t.min.ukey(), t.max.ukey())
+		level = s.version_NB().pickLevel(t.imin.ukey(), t.imax.ukey())
 	}
 	c.rec.addTableFile(level, t)
 
-	s.logf("mem@flush created L%d@%d N·%d S·%s %q:%q", level, t.file.Num(), n, shortenb(int(t.size)), t.min, t.max)
+	s.logf("mem@flush created L%d@%d N·%d S·%s %q:%q", level, t.file.Num(), n, shortenb(int(t.size)), t.imin, t.imax)
 
 	c.level = level
 	return nil
@@ -100,33 +101,34 @@ func (c *cMem) reset() {
 func (c *cMem) commit(journal, seq uint64) error {
 	c.rec.setJournalNum(journal)
 	c.rec.setSeq(seq)
-	// Commit changes
+
+	// Commit changes.
 	return c.s.commit(c.rec)
 }
 
-func (d *DB) compactionError() {
+func (db *DB) compactionError() {
 	var err error
 noerr:
 	for {
 		select {
-		case _, _ = <-d.closeC:
-			return
-		case err = <-d.compErrSetC:
+		case err = <-db.compErrSetC:
 			if err != nil {
 				goto haserr
 			}
+		case _, _ = <-db.closeC:
+			return
 		}
 	}
 haserr:
 	for {
 		select {
-		case _, _ = <-d.closeC:
-			return
-		case err = <-d.compErrSetC:
+		case db.compErrC <- err:
+		case err = <-db.compErrSetC:
 			if err == nil {
 				goto noerr
 			}
-		case d.compErrC <- err:
+		case _, _ = <-db.closeC:
+			return
 		}
 	}
 }
@@ -137,18 +139,18 @@ func (cnt *compactionTransactCounter) incr() {
 	*cnt++
 }
 
-func (d *DB) compactionTransact(name string, exec func(cnt *compactionTransactCounter) error, rollback func() error) {
-	s := d.s
+func (db *DB) compactionTransact(name string, exec func(cnt *compactionTransactCounter) error, rollback func() error) {
 	defer func() {
 		if x := recover(); x != nil {
 			if x == errCompactionTransactExiting && rollback != nil {
 				if err := rollback(); err != nil {
-					s.logf("%s rollback error %q", name, err)
+					db.logf("%s rollback error %q", name, err)
 				}
 			}
 			panic(x)
 		}
 	}()
+
 	const (
 		backoffMin = 1 * time.Second
 		backoffMax = 8 * time.Second
@@ -159,11 +161,11 @@ func (d *DB) compactionTransact(name string, exec func(cnt *compactionTransactCo
 	lastCnt := compactionTransactCounter(0)
 	for n := 0; ; n++ {
 		// Check wether the DB is closed.
-		if d.isClosed() {
-			s.logf("%s exiting", name)
-			d.compactionExitTransact()
+		if db.isClosed() {
+			db.logf("%s exiting", name)
+			db.compactionExitTransact()
 		} else if n > 0 {
-			s.logf("%s retrying N·%d", name, n)
+			db.logf("%s retrying N·%d", name, n)
 		}
 
 		// Execute.
@@ -172,15 +174,15 @@ func (d *DB) compactionTransact(name string, exec func(cnt *compactionTransactCo
 
 		// Set compaction error status.
 		select {
-		case d.compErrSetC <- err:
-		case _, _ = <-d.closeC:
-			s.logf("%s exiting", name)
-			d.compactionExitTransact()
+		case db.compErrSetC <- err:
+		case _, _ = <-db.closeC:
+			db.logf("%s exiting", name)
+			db.compactionExitTransact()
 		}
 		if err == nil {
 			return
 		}
-		s.logf("%s error I·%d %q", name, cnt, err)
+		db.logf("%s error I·%d %q", name, cnt, err)
 
 		// Reset backoff duration if counter is advancing.
 		if cnt > lastCnt {
@@ -198,53 +200,53 @@ func (d *DB) compactionTransact(name string, exec func(cnt *compactionTransactCo
 		}
 		select {
 		case <-backoffT.C:
-		case _, _ = <-d.closeC:
-			s.logf("%s exiting", name)
-			d.compactionExitTransact()
+		case _, _ = <-db.closeC:
+			db.logf("%s exiting", name)
+			db.compactionExitTransact()
 		}
 	}
 }
 
-func (d *DB) compactionExitTransact() {
+func (db *DB) compactionExitTransact() {
 	panic(errCompactionTransactExiting)
 }
 
-func (d *DB) memCompaction() {
-	mem := d.getFrozenMem()
+func (db *DB) memCompaction() {
+	mem := db.getFrozenMem()
 	if mem == nil {
 		return
 	}
+	defer mem.decref()
 
-	s := d.s
-	c := newCMem(s)
+	c := newCMem(db.s)
 	stats := new(cStatsStaging)
 
-	s.logf("mem@flush N·%d S·%s", mem.Len(), shortenb(mem.Size()))
+	db.logf("mem@flush N·%d S·%s", mem.mdb.Len(), shortenb(mem.mdb.Size()))
 
 	// Don't compact empty memdb.
-	if mem.Len() == 0 {
-		s.logf("mem@flush skipping")
+	if mem.mdb.Len() == 0 {
+		db.logf("mem@flush skipping")
 		// drop frozen mem
-		d.dropFrozenMem()
+		db.dropFrozenMem()
 		return
 	}
 
 	// Pause table compaction.
 	ch := make(chan struct{})
 	select {
-	case d.tcompPauseC <- (chan<- struct{})(ch):
-	case _, _ = <-d.closeC:
+	case db.tcompPauseC <- (chan<- struct{})(ch):
+	case _, _ = <-db.closeC:
 		return
 	}
 
-	d.compactionTransact("mem@flush", func(cnt *compactionTransactCounter) (err error) {
+	db.compactionTransact("mem@flush", func(cnt *compactionTransactCounter) (err error) {
 		stats.startTimer()
 		defer stats.stopTimer()
-		return c.flush(mem, -1)
+		return c.flush(mem.mdb, -1)
 	}, func() error {
 		for _, r := range c.rec.addedTables {
-			s.logf("mem@flush rollback @%d", r.num)
-			f := s.getTableFile(r.num)
+			db.logf("mem@flush rollback @%d", r.num)
+			f := db.s.getTableFile(r.num)
 			if err := f.Remove(); err != nil {
 				return err
 			}
@@ -252,61 +254,59 @@ func (d *DB) memCompaction() {
 		return nil
 	})
 
-	d.compactionTransact("mem@commit", func(cnt *compactionTransactCounter) (err error) {
+	db.compactionTransact("mem@commit", func(cnt *compactionTransactCounter) (err error) {
 		stats.startTimer()
 		defer stats.stopTimer()
-		return c.commit(d.journalFile.Num(), d.frozenSeq)
+		return c.commit(db.journalFile.Num(), db.frozenSeq)
 	}, nil)
 
-	s.logf("mem@flush commited F·%d T·%v", len(c.rec.addedTables), stats.duration)
+	db.logf("mem@flush committed F·%d T·%v", len(c.rec.addedTables), stats.duration)
 
 	for _, r := range c.rec.addedTables {
 		stats.write += r.size
 	}
-	d.compStats[c.level].add(stats)
+	db.compStats[c.level].add(stats)
 
 	// Drop frozen mem.
-	d.dropFrozenMem()
+	db.dropFrozenMem()
 
 	// Resume table compaction.
 	select {
 	case <-ch:
-	case _, _ = <-d.closeC:
+	case _, _ = <-db.closeC:
 		return
 	}
 
 	// Trigger table compaction.
-	d.compTrigger(d.mcompTriggerC)
+	db.compTrigger(db.mcompTriggerC)
 }
 
-func (d *DB) tableCompaction(c *compaction, noTrivial bool) {
-	s := d.s
-
+func (db *DB) tableCompaction(c *compaction, noTrivial bool) {
 	rec := new(sessionRecord)
-	rec.addCompactionPointer(c.level, c.max)
+	rec.addCompactionPointer(c.level, c.imax)
 
 	if !noTrivial && c.trivial() {
 		t := c.tables[0][0]
-		s.logf("table@move L%d@%d -> L%d", c.level, t.file.Num(), c.level+1)
+		db.logf("table@move L%d@%d -> L%d", c.level, t.file.Num(), c.level+1)
 		rec.deleteTable(c.level, t.file.Num())
 		rec.addTableFile(c.level+1, t)
-		d.compactionTransact("table@move", func(cnt *compactionTransactCounter) (err error) {
-			return s.commit(rec)
+		db.compactionTransact("table@move", func(cnt *compactionTransactCounter) (err error) {
+			return db.s.commit(rec)
 		}, nil)
 		return
 	}
 
 	var stats [2]cStatsStaging
-	for i, tt := range c.tables {
-		for _, t := range tt {
+	for i, tables := range c.tables {
+		for _, t := range tables {
 			stats[i].read += t.size
 			// Insert deleted tables into record
 			rec.deleteTable(c.level+i, t.file.Num())
 		}
 	}
 	sourceSize := int(stats[0].read + stats[1].read)
-	minSeq := d.minSeq()
-	s.logf("table@compaction L%d·%d -> L%d·%d S·%s Q·%d", c.level, len(c.tables[0]), c.level+1, len(c.tables[1]), shortenb(sourceSize), minSeq)
+	minSeq := db.minSeq()
+	db.logf("table@compaction L%d·%d -> L%d·%d S·%s Q·%d", c.level, len(c.tables[0]), c.level+1, len(c.tables[1]), shortenb(sourceSize), minSeq)
 
 	var snapUkey []byte
 	var snapHasUkey bool
@@ -314,7 +314,7 @@ func (d *DB) tableCompaction(c *compaction, noTrivial bool) {
 	var snapIter int
 	var snapDropCnt int
 	var dropCnt int
-	d.compactionTransact("table@build", func(cnt *compactionTransactCounter) (err error) {
+	db.compactionTransact("table@build", func(cnt *compactionTransactCounter) (err error) {
 		ukey := append([]byte{}, snapUkey...)
 		hasUkey := snapHasUkey
 		lseq := snapSeq
@@ -329,7 +329,7 @@ func (d *DB) tableCompaction(c *compaction, noTrivial bool) {
 			}
 			rec.addTableFile(c.level+1, t)
 			stats[1].write += t.size
-			s.logf("table@build created L%d@%d N·%d S·%s %q:%q", c.level+1, t.file.Num(), tw.tw.EntriesLen(), shortenb(int(t.size)), t.min, t.max)
+			db.logf("table@build created L%d@%d N·%d S·%s %q:%q", c.level+1, t.file.Num(), tw.tw.EntriesLen(), shortenb(int(t.size)), t.imin, t.imax)
 			return nil
 		}
 
@@ -353,9 +353,9 @@ func (d *DB) tableCompaction(c *compaction, noTrivial bool) {
 				continue
 			}
 
-			key := iKey(iter.Key())
+			ikey := iKey(iter.Key())
 
-			if c.shouldStopBefore(key) && tw != nil {
+			if c.shouldStopBefore(ikey) && tw != nil {
 				err = finish()
 				if err != nil {
 					return
@@ -375,15 +375,15 @@ func (d *DB) tableCompaction(c *compaction, noTrivial bool) {
 				snapSched = false
 			}
 
-			if seq, t, ok := key.parseNum(); !ok {
+			if seq, vt, ok := ikey.parseNum(); !ok {
 				// Don't drop error keys
 				ukey = ukey[:0]
 				hasUkey = false
 				lseq = kMaxSeq
 			} else {
-				if !hasUkey || s.icmp.uCompare(key.ukey(), ukey) != 0 {
+				if !hasUkey || db.s.icmp.uCompare(ikey.ukey(), ukey) != 0 {
 					// First occurrence of this user key
-					ukey = append(ukey[:0], key.ukey()...)
+					ukey = append(ukey[:0], ikey.ukey()...)
 					hasUkey = true
 					lseq = kMaxSeq
 				}
@@ -392,7 +392,7 @@ func (d *DB) tableCompaction(c *compaction, noTrivial bool) {
 				if lseq <= minSeq {
 					// Dropped because newer entry for same user key exist
 					drop = true // (A)
-				} else if t == tDel && seq <= minSeq && c.isBaseLevelForKey(ukey) {
+				} else if vt == tDel && seq <= minSeq && c.baseLevelForKey(ukey) {
 					// For this user key:
 					// (1) there is no data in higher levels
 					// (2) data in lower levels will have larger seq numbers
@@ -414,22 +414,22 @@ func (d *DB) tableCompaction(c *compaction, noTrivial bool) {
 			if tw == nil {
 				// Check for pause event.
 				select {
-				case ch := <-d.tcompPauseC:
-					d.pauseCompaction(ch)
-				case _, _ = <-d.closeC:
-					d.compactionExitTransact()
+				case ch := <-db.tcompPauseC:
+					db.pauseCompaction(ch)
+				case _, _ = <-db.closeC:
+					db.compactionExitTransact()
 				default:
 				}
 
 				// Create new table.
-				tw, err = s.tops.create()
+				tw, err = db.s.tops.create()
 				if err != nil {
 					return
 				}
 			}
 
 			// Write key/value into table
-			err = tw.add(key, iter.Value())
+			err = tw.append(ikey, iter.Value())
 			if err != nil {
 				return
 			}
@@ -461,8 +461,8 @@ func (d *DB) tableCompaction(c *compaction, noTrivial bool) {
 		return
 	}, func() error {
 		for _, r := range rec.addedTables {
-			s.logf("table@build rollback @%d", r.num)
-			f := s.getTableFile(r.num)
+			db.logf("table@build rollback @%d", r.num)
+			f := db.s.getTableFile(r.num)
 			if err := f.Remove(); err != nil {
 				return err
 			}
@@ -471,60 +471,61 @@ func (d *DB) tableCompaction(c *compaction, noTrivial bool) {
 	})
 
 	// Commit changes
-	d.compactionTransact("table@commit", func(cnt *compactionTransactCounter) (err error) {
+	db.compactionTransact("table@commit", func(cnt *compactionTransactCounter) (err error) {
 		stats[1].startTimer()
 		defer stats[1].stopTimer()
-		return s.commit(rec)
+		return db.s.commit(rec)
 	}, nil)
 
-	resultSize := int(int(stats[1].write))
-	s.logf("table@compaction commited F%s S%s D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), dropCnt, stats[1].duration)
+	resultSize := int(stats[1].write)
+	db.logf("table@compaction committed F%s S%s D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), dropCnt, stats[1].duration)
 
 	// Save compaction stats
 	for i := range stats {
-		d.compStats[c.level+1].add(&stats[i])
+		db.compStats[c.level+1].add(&stats[i])
 	}
 }
 
-func (d *DB) tableRangeCompaction(level int, min, max []byte) {
-	s := d.s
-	s.logf("table@compaction range L%d %q:%q", level, min, max)
+func (db *DB) tableRangeCompaction(level int, umin, umax []byte) {
+	db.logf("table@compaction range L%d %q:%q", level, umin, umax)
 
 	if level >= 0 {
-		if c := s.getCompactionRange(level, min, max); c != nil {
-			d.tableCompaction(c, true)
+		if c := db.s.getCompactionRange(level, umin, umax); c != nil {
+			db.tableCompaction(c, true)
 		}
 	} else {
-		v := s.version_NB()
+		v := db.s.version_NB()
+
 		m := 1
 		for i, t := range v.tables[1:] {
-			if t.isOverlaps(min, max, true, s.icmp) {
+			if t.overlaps(db.s.icmp, umin, umax, false) {
 				m = i + 1
 			}
 		}
+
 		for level := 0; level < m; level++ {
-			if c := s.getCompactionRange(level, min, max); c != nil {
-				d.tableCompaction(c, true)
+			if c := db.s.getCompactionRange(level, umin, umax); c != nil {
+				db.tableCompaction(c, true)
 			}
 		}
 	}
 }
 
-func (d *DB) tableAutoCompaction() {
-	if c := d.s.pickCompaction(); c != nil {
-		d.tableCompaction(c, false)
+func (db *DB) tableAutoCompaction() {
+	if c := db.s.pickCompaction(); c != nil {
+		db.tableCompaction(c, false)
 	}
 }
 
-func (d *DB) tableNeedCompaction() bool {
-	return d.s.version_NB().needCompaction()
+func (db *DB) tableNeedCompaction() bool {
+	return db.s.version_NB().needCompaction()
 }
 
-func (d *DB) pauseCompaction(ch chan<- struct{}) {
+func (db *DB) pauseCompaction(ch chan<- struct{}) {
 	select {
 	case ch <- struct{}{}:
-	case _, _ = <-d.closeC:
-		d.compactionExitTransact()
+	case _, _ = <-db.closeC:
+		db.compactionExitTransact()
 	}
 }
 
@@ -555,48 +556,48 @@ func (r cRange) ack(err error) {
 	}
 }
 
-func (d *DB) compSendIdle(compC chan<- cCmd) error {
+func (db *DB) compSendIdle(compC chan<- cCmd) error {
 	ch := make(chan error)
 	defer close(ch)
 	// Send cmd.
 	select {
 	case compC <- cIdle{ch}:
-	case err := <-d.compErrC:
+	case err := <-db.compErrC:
 		return err
-	case _, _ = <-d.closeC:
+	case _, _ = <-db.closeC:
 		return ErrClosed
 	}
 	// Wait cmd.
 	return <-ch
 }
 
-func (d *DB) compSendRange(compC chan<- cCmd, level int, min, max []byte) (err error) {
+func (db *DB) compSendRange(compC chan<- cCmd, level int, min, max []byte) (err error) {
 	ch := make(chan error)
 	defer close(ch)
 	// Send cmd.
 	select {
 	case compC <- cRange{level, min, max, ch}:
-	case err := <-d.compErrC:
+	case err := <-db.compErrC:
 		return err
-	case _, _ = <-d.closeC:
+	case _, _ = <-db.closeC:
 		return ErrClosed
 	}
 	// Wait cmd.
 	select {
-	case err = <-d.compErrC:
+	case err = <-db.compErrC:
 	case err = <-ch:
 	}
 	return err
 }
 
-func (d *DB) compTrigger(compTriggerC chan struct{}) {
+func (db *DB) compTrigger(compTriggerC chan struct{}) {
 	select {
 	case compTriggerC <- struct{}{}:
 	default:
 	}
 }
 
-func (d *DB) mCompaction() {
+func (db *DB) mCompaction() {
 	var x cCmd
 
 	defer func() {
@@ -608,24 +609,24 @@ func (d *DB) mCompaction() {
 		if x != nil {
 			x.ack(ErrClosed)
 		}
-		d.closeW.Done()
+		db.closeW.Done()
 	}()
 
 	for {
 		select {
-		case _, _ = <-d.closeC:
-			return
-		case x = <-d.mcompCmdC:
-			d.memCompaction()
+		case x = <-db.mcompCmdC:
+			db.memCompaction()
 			x.ack(nil)
 			x = nil
-		case <-d.mcompTriggerC:
-			d.memCompaction()
+		case <-db.mcompTriggerC:
+			db.memCompaction()
+		case _, _ = <-db.closeC:
+			return
 		}
 	}
 }
 
-func (d *DB) tCompaction() {
+func (db *DB) tCompaction() {
 	var x cCmd
 	var ackQ []cCmd
 
@@ -642,19 +643,19 @@ func (d *DB) tCompaction() {
 		if x != nil {
 			x.ack(ErrClosed)
 		}
-		d.closeW.Done()
+		db.closeW.Done()
 	}()
 
 	for {
-		if d.tableNeedCompaction() {
+		if db.tableNeedCompaction() {
 			select {
-			case x = <-d.tcompCmdC:
-			case <-d.tcompTriggerC:
-			case _, _ = <-d.closeC:
-				return
-			case ch := <-d.tcompPauseC:
-				d.pauseCompaction(ch)
+			case x = <-db.tcompCmdC:
+			case <-db.tcompTriggerC:
+			case ch := <-db.tcompPauseC:
+				db.pauseCompaction(ch)
 				continue
+			case _, _ = <-db.closeC:
+				return
 			default:
 			}
 		} else {
@@ -664,12 +665,12 @@ func (d *DB) tCompaction() {
 			}
 			ackQ = ackQ[:0]
 			select {
-			case x = <-d.tcompCmdC:
-			case <-d.tcompTriggerC:
-			case ch := <-d.tcompPauseC:
-				d.pauseCompaction(ch)
+			case x = <-db.tcompCmdC:
+			case <-db.tcompTriggerC:
+			case ch := <-db.tcompPauseC:
+				db.pauseCompaction(ch)
 				continue
-			case _, _ = <-d.closeC:
+			case _, _ = <-db.closeC:
 				return
 			}
 		}
@@ -678,11 +679,11 @@ func (d *DB) tCompaction() {
 			case cIdle:
 				ackQ = append(ackQ, x)
 			case cRange:
-				d.tableRangeCompaction(cmd.level, cmd.min, cmd.max)
+				db.tableRangeCompaction(cmd.level, cmd.min, cmd.max)
 				x.ack(nil)
 			}
 			x = nil
 		}
-		d.tableAutoCompaction()
+		db.tableAutoCompaction()
 	}
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_iter.go

@@ -9,6 +9,8 @@ package leveldb
 import (
 	"errors"
 	"runtime"
+	"sync"
+	"sync/atomic"
 
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -19,39 +21,53 @@ var (
 	errInvalidIkey = errors.New("leveldb: Iterator: invalid internal key")
 )
 
-func (db *DB) newRawIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
-	s := db.s
+type memdbReleaser struct {
+	once sync.Once
+	m    *memDB
+}
 
+func (mr *memdbReleaser) Release() {
+	mr.once.Do(func() {
+		mr.m.decref()
+	})
+}
+
+func (db *DB) newRawIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
 	em, fm := db.getMems()
-	v := s.version()
+	v := db.s.version()
 
 	ti := v.getIterators(slice, ro)
 	n := len(ti) + 2
 	i := make([]iterator.Iterator, 0, n)
-	i = append(i, em.NewIterator(slice))
+	emi := em.mdb.NewIterator(slice)
+	emi.SetReleaser(&memdbReleaser{m: em})
+	i = append(i, emi)
 	if fm != nil {
-		i = append(i, fm.NewIterator(slice))
+		fmi := fm.mdb.NewIterator(slice)
+		fmi.SetReleaser(&memdbReleaser{m: fm})
+		i = append(i, fmi)
 	}
 	i = append(i, ti...)
-	strict := s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator)
-	mi := iterator.NewMergedIterator(i, s.icmp, strict)
+	strict := db.s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator)
+	mi := iterator.NewMergedIterator(i, db.s.icmp, strict)
 	mi.SetReleaser(&versionReleaser{v: v})
 	return mi
 }
 
 func (db *DB) newIterator(seq uint64, slice *util.Range, ro *opt.ReadOptions) *dbIter {
-	var slice_ *util.Range
+	var islice *util.Range
 	if slice != nil {
-		slice_ = &util.Range{}
+		islice = &util.Range{}
 		if slice.Start != nil {
-			slice_.Start = newIKey(slice.Start, kMaxSeq, tSeek)
+			islice.Start = newIKey(slice.Start, kMaxSeq, tSeek)
 		}
 		if slice.Limit != nil {
-			slice_.Limit = newIKey(slice.Limit, kMaxSeq, tSeek)
+			islice.Limit = newIKey(slice.Limit, kMaxSeq, tSeek)
 		}
 	}
-	rawIter := db.newRawIterator(slice_, ro)
+	rawIter := db.newRawIterator(islice, ro)
 	iter := &dbIter{
+		db:     db,
 		icmp:   db.s.icmp,
 		iter:   rawIter,
 		seq:    seq,
@@ -59,6 +75,7 @@ func (db *DB) newIterator(seq uint64, slice *util.Range, ro *opt.ReadOptions) *d
 		key:    make([]byte, 0),
 		value:  make([]byte, 0),
 	}
+	atomic.AddInt32(&db.aliveIters, 1)
 	runtime.SetFinalizer(iter, (*dbIter).Release)
 	return iter
 }
@@ -75,6 +92,7 @@ const (
 
 // dbIter represent an interator states over a database session.
 type dbIter struct {
+	db     *DB
 	icmp   *iComparer
 	iter   iterator.Iterator
 	seq    uint64
@@ -289,6 +307,7 @@ func (i *dbIter) Release() {
 
 		if i.releaser != nil {
 			i.releaser.Release()
+			i.releaser = nil
 		}
 
 		i.dir = dirReleased
@@ -296,13 +315,19 @@ func (i *dbIter) Release() {
 		i.value = nil
 		i.iter.Release()
 		i.iter = nil
+		atomic.AddInt32(&i.db.aliveIters, -1)
+		i.db = nil
 	}
 }
 
 func (i *dbIter) SetReleaser(releaser util.Releaser) {
-	if i.dir != dirReleased {
-		i.releaser = releaser
+	if i.dir == dirReleased {
+		panic(util.ErrReleased)
 	}
+	if i.releaser != nil && releaser != nil {
+		panic(util.ErrHasReleaser)
+	}
+	i.releaser = releaser
 }
 
 func (i *dbIter) Error() error {

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_snapshot.go

@@ -9,6 +9,7 @@ package leveldb
 import (
 	"runtime"
 	"sync"
+	"sync/atomic"
 
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -70,7 +71,7 @@ func (db *DB) releaseSnapshot(elem *snapshotElement) {
 func (db *DB) minSeq() uint64 {
 	db.snapsMu.Lock()
 	defer db.snapsMu.Unlock()
-	elem := db.snapsRoot.prev
+	elem := db.snapsRoot.next
 	if elem != &db.snapsRoot {
 		return elem.seq
 	}
@@ -81,18 +82,19 @@ func (db *DB) minSeq() uint64 {
 type Snapshot struct {
 	db       *DB
 	elem     *snapshotElement
-	mu       sync.Mutex
+	mu       sync.RWMutex
 	released bool
 }
 
 // Creates new snapshot object.
 func (db *DB) newSnapshot() *Snapshot {
-	p := &Snapshot{
+	snap := &Snapshot{
 		db:   db,
 		elem: db.acquireSnapshot(),
 	}
-	runtime.SetFinalizer(p, (*Snapshot).Release)
-	return p
+	atomic.AddInt32(&db.aliveSnaps, 1)
+	runtime.SetFinalizer(snap, (*Snapshot).Release)
+	return snap
 }
 
 // Get gets the value for the given key. It returns ErrNotFound if
@@ -100,19 +102,18 @@ func (db *DB) newSnapshot() *Snapshot {
 //
 // The caller should not modify the contents of the returned slice, but
 // it is safe to modify the contents of the argument after Get returns.
-func (p *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
-	db := p.db
-	err = db.ok()
+func (snap *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
+	err = snap.db.ok()
 	if err != nil {
 		return
 	}
-	p.mu.Lock()
-	defer p.mu.Unlock()
-	if p.released {
+	snap.mu.RLock()
+	defer snap.mu.RUnlock()
+	if snap.released {
 		err = ErrSnapshotReleased
 		return
 	}
-	return db.get(key, p.elem.seq, ro)
+	return snap.db.get(key, snap.elem.seq, ro)
 }
 
 // NewIterator returns an iterator for the snapshot of the uderlying DB.
@@ -132,17 +133,18 @@ func (p *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error
 // iterator would be still valid until released.
 //
 // Also read Iterator documentation of the leveldb/iterator package.
-func (p *Snapshot) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
-	db := p.db
-	if err := db.ok(); err != nil {
+func (snap *Snapshot) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
+	if err := snap.db.ok(); err != nil {
 		return iterator.NewEmptyIterator(err)
 	}
-	p.mu.Lock()
-	defer p.mu.Unlock()
-	if p.released {
+	snap.mu.Lock()
+	defer snap.mu.Unlock()
+	if snap.released {
 		return iterator.NewEmptyIterator(ErrSnapshotReleased)
 	}
-	return db.newIterator(p.elem.seq, slice, ro)
+	// Since iterator already hold version ref, it doesn't need to
+	// hold snapshot ref.
+	return snap.db.newIterator(snap.elem.seq, slice, ro)
 }
 
 // Release releases the snapshot. This will not release any returned
@@ -150,16 +152,18 @@ func (p *Snapshot) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.
 // underlying DB is closed.
 //
 // Other methods should not be called after the snapshot has been released.
-func (p *Snapshot) Release() {
-	p.mu.Lock()
-	if !p.released {
-		// Clear the finalizer.
-		runtime.SetFinalizer(p, nil)
+func (snap *Snapshot) Release() {
+	snap.mu.Lock()
+	defer snap.mu.Unlock()
 
-		p.released = true
-		p.db.releaseSnapshot(p.elem)
-		p.db = nil
-		p.elem = nil
+	if !snap.released {
+		// Clear the finalizer.
+		runtime.SetFinalizer(snap, nil)
+
+		snap.released = true
+		snap.db.releaseSnapshot(snap.elem)
+		atomic.AddInt32(&snap.db.aliveSnaps, -1)
+		snap.db = nil
+		snap.elem = nil
 	}
-	p.mu.Unlock()
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_state.go

@@ -8,106 +8,194 @@ package leveldb
 
 import (
 	"sync/atomic"
+	"time"
 
 	"github.com/syndtr/goleveldb/leveldb/journal"
 	"github.com/syndtr/goleveldb/leveldb/memdb"
 )
 
+type memDB struct {
+	db  *DB
+	mdb *memdb.DB
+	ref int32
+}
+
+func (m *memDB) incref() {
+	atomic.AddInt32(&m.ref, 1)
+}
+
+func (m *memDB) decref() {
+	if ref := atomic.AddInt32(&m.ref, -1); ref == 0 {
+		// Only put back memdb with std capacity.
+		if m.mdb.Capacity() == m.db.s.o.GetWriteBuffer() {
+			m.mdb.Reset()
+			m.db.mpoolPut(m.mdb)
+		}
+		m.db = nil
+		m.mdb = nil
+	} else if ref < 0 {
+		panic("negative memdb ref")
+	}
+}
+
 // Get latest sequence number.
-func (d *DB) getSeq() uint64 {
-	return atomic.LoadUint64(&d.seq)
+func (db *DB) getSeq() uint64 {
+	return atomic.LoadUint64(&db.seq)
 }
 
 // Atomically adds delta to seq.
-func (d *DB) addSeq(delta uint64) {
-	atomic.AddUint64(&d.seq, delta)
+func (db *DB) addSeq(delta uint64) {
+	atomic.AddUint64(&db.seq, delta)
+}
+
+func (db *DB) mpoolPut(mem *memdb.DB) {
+	defer func() {
+		recover()
+	}()
+	select {
+	case db.memPool <- mem:
+	default:
+	}
+}
+
+func (db *DB) mpoolGet() *memdb.DB {
+	select {
+	case mem := <-db.memPool:
+		return mem
+	default:
+		return nil
+	}
+}
+
+func (db *DB) mpoolDrain() {
+	ticker := time.NewTicker(30 * time.Second)
+	for {
+		select {
+		case <-ticker.C:
+			select {
+			case <-db.memPool:
+			default:
+			}
+		case _, _ = <-db.closeC:
+			close(db.memPool)
+			return
+		}
+	}
 }
 
 // Create new memdb and froze the old one; need external synchronization.
 // newMem only called synchronously by the writer.
-func (d *DB) newMem(n int) (mem *memdb.DB, err error) {
-	s := d.s
-
-	num := s.allocFileNum()
-	file := s.getJournalFile(num)
+func (db *DB) newMem(n int) (mem *memDB, err error) {
+	num := db.s.allocFileNum()
+	file := db.s.getJournalFile(num)
 	w, err := file.Create()
 	if err != nil {
-		s.reuseFileNum(num)
+		db.s.reuseFileNum(num)
 		return
 	}
-	d.memMu.Lock()
-	if d.journal == nil {
-		d.journal = journal.NewWriter(w)
-	} else {
-		d.journal.Reset(w)
-		d.journalWriter.Close()
-		d.frozenJournalFile = d.journalFile
+
+	db.memMu.Lock()
+	defer db.memMu.Unlock()
+
+	if db.frozenMem != nil {
+		panic("still has frozen mem")
 	}
-	d.journalWriter = w
-	d.journalFile = file
-	d.frozenMem = d.mem
-	d.mem = memdb.New(s.icmp, maxInt(d.s.o.GetWriteBuffer(), n))
-	mem = d.mem
-	// The seq only incremented by the writer.
-	d.frozenSeq = d.seq
-	d.memMu.Unlock()
+
+	if db.journal == nil {
+		db.journal = journal.NewWriter(w)
+	} else {
+		db.journal.Reset(w)
+		db.journalWriter.Close()
+		db.frozenJournalFile = db.journalFile
+	}
+	db.journalWriter = w
+	db.journalFile = file
+	db.frozenMem = db.mem
+	mdb := db.mpoolGet()
+	if mdb == nil || mdb.Capacity() < n {
+		mdb = memdb.New(db.s.icmp, maxInt(db.s.o.GetWriteBuffer(), n))
+	}
+	mem = &memDB{
+		db:  db,
+		mdb: mdb,
+		ref: 2,
+	}
+	db.mem = mem
+	// The seq only incremented by the writer. And whoever called newMem
+	// should hold write lock, so no need additional synchronization here.
+	db.frozenSeq = db.seq
 	return
 }
 
 // Get all memdbs.
-func (d *DB) getMems() (e *memdb.DB, f *memdb.DB) {
-	d.memMu.RLock()
-	defer d.memMu.RUnlock()
-	return d.mem, d.frozenMem
+func (db *DB) getMems() (e, f *memDB) {
+	db.memMu.RLock()
+	defer db.memMu.RUnlock()
+	if db.mem == nil {
+		panic("nil effective mem")
+	}
+	db.mem.incref()
+	if db.frozenMem != nil {
+		db.frozenMem.incref()
+	}
+	return db.mem, db.frozenMem
 }
 
 // Get frozen memdb.
-func (d *DB) getEffectiveMem() *memdb.DB {
-	d.memMu.RLock()
-	defer d.memMu.RUnlock()
-	return d.mem
+func (db *DB) getEffectiveMem() *memDB {
+	db.memMu.RLock()
+	defer db.memMu.RUnlock()
+	if db.mem == nil {
+		panic("nil effective mem")
+	}
+	db.mem.incref()
+	return db.mem
 }
 
 // Check whether we has frozen memdb.
-func (d *DB) hasFrozenMem() bool {
-	d.memMu.RLock()
-	defer d.memMu.RUnlock()
-	return d.frozenMem != nil
+func (db *DB) hasFrozenMem() bool {
+	db.memMu.RLock()
+	defer db.memMu.RUnlock()
+	return db.frozenMem != nil
 }
 
 // Get frozen memdb.
-func (d *DB) getFrozenMem() *memdb.DB {
-	d.memMu.RLock()
-	defer d.memMu.RUnlock()
-	return d.frozenMem
+func (db *DB) getFrozenMem() *memDB {
+	db.memMu.RLock()
+	defer db.memMu.RUnlock()
+	if db.frozenMem != nil {
+		db.frozenMem.incref()
+	}
+	return db.frozenMem
 }
 
 // Drop frozen memdb; assume that frozen memdb isn't nil.
-func (d *DB) dropFrozenMem() {
-	d.memMu.Lock()
-	if err := d.frozenJournalFile.Remove(); err != nil {
-		d.s.logf("journal@remove removing @%d %q", d.frozenJournalFile.Num(), err)
+func (db *DB) dropFrozenMem() {
+	db.memMu.Lock()
+	if err := db.frozenJournalFile.Remove(); err != nil {
+		db.logf("journal@remove removing @%d %q", db.frozenJournalFile.Num(), err)
 	} else {
-		d.s.logf("journal@remove removed @%d", d.frozenJournalFile.Num())
+		db.logf("journal@remove removed @%d", db.frozenJournalFile.Num())
 	}
-	d.frozenJournalFile = nil
-	d.frozenMem = nil
-	d.memMu.Unlock()
+	db.frozenJournalFile = nil
+	db.frozenMem.decref()
+	db.frozenMem = nil
+	db.memMu.Unlock()
 }
 
 // Set closed flag; return true if not already closed.
-func (d *DB) setClosed() bool {
-	return atomic.CompareAndSwapUint32(&d.closed, 0, 1)
+func (db *DB) setClosed() bool {
+	return atomic.CompareAndSwapUint32(&db.closed, 0, 1)
 }
 
 // Check whether DB was closed.
-func (d *DB) isClosed() bool {
-	return atomic.LoadUint32(&d.closed) != 0
+func (db *DB) isClosed() bool {
+	return atomic.LoadUint32(&db.closed) != 0
 }
 
 // Check read ok status.
-func (d *DB) ok() error {
-	if d.isClosed() {
+func (db *DB) ok() error {
+	if db.isClosed() {
 		return ErrClosed
 	}
 	return nil

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_test.go

@@ -154,9 +154,7 @@ func (h *dbHarness) maxNextLevelOverlappingBytes(want uint64) {
 		level := i + 1
 		next := v.tables[level+1]
 		for _, t := range tt {
-			var r tFiles
-			min, max := t.min.ukey(), t.max.ukey()
-			next.getOverlaps(min, max, &r, true, db.s.icmp.ucmp)
+			r := next.getOverlaps(nil, db.s.icmp, t.imin.ukey(), t.imax.ukey(), false)
 			sum := r.size()
 			if sum > res {
 				res = sum
@@ -1127,6 +1125,45 @@ func TestDb_Snapshot(t *testing.T) {
 	})
 }
 
+func TestDb_SnapshotList(t *testing.T) {
+	db := &DB{}
+	db.initSnapshot()
+	e0a := db.acquireSnapshot()
+	e0b := db.acquireSnapshot()
+	db.seq = 1
+	e1 := db.acquireSnapshot()
+	db.seq = 2
+	e2 := db.acquireSnapshot()
+
+	if db.minSeq() != 0 {
+		t.Fatalf("invalid sequence number, got=%d", db.minSeq())
+	}
+	db.releaseSnapshot(e0a)
+	if db.minSeq() != 0 {
+		t.Fatalf("invalid sequence number, got=%d", db.minSeq())
+	}
+	db.releaseSnapshot(e2)
+	if db.minSeq() != 0 {
+		t.Fatalf("invalid sequence number, got=%d", db.minSeq())
+	}
+	db.releaseSnapshot(e0b)
+	if db.minSeq() != 1 {
+		t.Fatalf("invalid sequence number, got=%d", db.minSeq())
+	}
+	e2 = db.acquireSnapshot()
+	if db.minSeq() != 1 {
+		t.Fatalf("invalid sequence number, got=%d", db.minSeq())
+	}
+	db.releaseSnapshot(e1)
+	if db.minSeq() != 2 {
+		t.Fatalf("invalid sequence number, got=%d", db.minSeq())
+	}
+	db.releaseSnapshot(e2)
+	if db.minSeq() != 2 {
+		t.Fatalf("invalid sequence number, got=%d", db.minSeq())
+	}
+}
+
 func TestDb_HiddenValuesAreRemoved(t *testing.T) {
 	trun(t, func(h *dbHarness) {
 		s := h.db.s
@@ -1212,7 +1249,7 @@ func TestDb_DeletionMarkers2(t *testing.T) {
 }
 
 func TestDb_CompactionTableOpenError(t *testing.T) {
-	h := newDbHarnessWopt(t, &opt.Options{MaxOpenFiles: 0})
+	h := newDbHarnessWopt(t, &opt.Options{CachedOpenFiles: -1})
 	defer h.close()
 
 	im := 10
@@ -1579,7 +1616,7 @@ func TestDb_BloomFilter(t *testing.T) {
 		return fmt.Sprintf("key%06d", i)
 	}
 
-	n := 10000
+	const n = 10000
 
 	// Populate multiple layers
 	for i := 0; i < n; i++ {
@@ -1886,3 +1923,102 @@ func TestDb_LeveldbIssue200(t *testing.T) {
 	iter.Next()
 	assertBytes(t, []byte("5"), iter.Key())
 }
+
+func TestDb_GoleveldbIssue74(t *testing.T) {
+	h := newDbHarnessWopt(t, &opt.Options{
+		WriteBuffer: 1 * opt.MiB,
+	})
+	defer h.close()
+
+	const n, dur = 10000, 5 * time.Second
+
+	runtime.GOMAXPROCS(runtime.NumCPU())
+
+	until := time.Now().Add(dur)
+	wg := new(sync.WaitGroup)
+	wg.Add(2)
+	var done uint32
+	go func() {
+		var i int
+		defer func() {
+			t.Logf("WRITER DONE #%d", i)
+			atomic.StoreUint32(&done, 1)
+			wg.Done()
+		}()
+
+		b := new(Batch)
+		for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ {
+			iv := fmt.Sprintf("VAL%010d", i)
+			for k := 0; k < n; k++ {
+				key := fmt.Sprintf("KEY%06d", k)
+				b.Put([]byte(key), []byte(key+iv))
+				b.Put([]byte(fmt.Sprintf("PTR%06d", k)), []byte(key))
+			}
+			h.write(b)
+
+			b.Reset()
+			snap := h.getSnapshot()
+			iter := snap.NewIterator(util.BytesPrefix([]byte("PTR")), nil)
+			var k int
+			for ; iter.Next(); k++ {
+				ptrKey := iter.Key()
+				key := iter.Value()
+
+				if _, err := snap.Get(ptrKey, nil); err != nil {
+					t.Fatalf("WRITER #%d snapshot.Get %q: %v", i, ptrKey, err)
+				}
+				if value, err := snap.Get(key, nil); err != nil {
+					t.Fatalf("WRITER #%d snapshot.Get %q: %v", i, key, err)
+				} else if string(value) != string(key)+iv {
+					t.Fatalf("WRITER #%d snapshot.Get %q got invalid value, want %q got %q", i, key, string(key)+iv, value)
+				}
+
+				b.Delete(key)
+				b.Delete(ptrKey)
+			}
+			h.write(b)
+			iter.Release()
+			snap.Release()
+			if k != n {
+				t.Fatalf("#%d %d != %d", i, k, n)
+			}
+		}
+		t.Logf("writer done after %d iterations", i)
+	}()
+	go func() {
+		var i int
+		defer func() {
+			t.Logf("READER DONE #%d", i)
+			atomic.StoreUint32(&done, 1)
+			wg.Done()
+		}()
+		for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ {
+			snap := h.getSnapshot()
+			iter := snap.NewIterator(util.BytesPrefix([]byte("PTR")), nil)
+			var prevValue string
+			var k int
+			for ; iter.Next(); k++ {
+				ptrKey := iter.Key()
+				key := iter.Value()
+
+				if _, err := snap.Get(ptrKey, nil); err != nil {
+					t.Fatalf("READER #%d snapshot.Get %q: %v", i, ptrKey, err)
+				}
+
+				if value, err := snap.Get(key, nil); err != nil {
+					t.Fatalf("READER #%d snapshot.Get %q: %v", i, key, err)
+				} else if prevValue != "" && string(value) != string(key)+prevValue {
+					t.Fatalf("READER #%d snapshot.Get %q got invalid value, want %q got %q", i, key, string(key)+prevValue, value)
+				} else {
+					prevValue = string(value[len(key):])
+				}
+			}
+			iter.Release()
+			snap.Release()
+			if k > 0 && k != n {
+				t.Fatalf("#%d %d != %d", i, k, n)
+			}
+		}
+	}()
+	wg.Wait()
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_util.go

@@ -32,40 +32,42 @@ func (p Sizes) Sum() (n uint64) {
 	return n
 }
 
-// Check and clean files.
-func (d *DB) checkAndCleanFiles() error {
-	s := d.s
+// Logging.
+func (db *DB) log(v ...interface{})                 { db.s.log(v...) }
+func (db *DB) logf(format string, v ...interface{}) { db.s.logf(format, v...) }
 
-	v := s.version_NB()
-	tables := make(map[uint64]bool)
-	for _, tt := range v.tables {
-		for _, t := range tt {
-			tables[t.file.Num()] = false
+// Check and clean files.
+func (db *DB) checkAndCleanFiles() error {
+	v := db.s.version_NB()
+	tablesMap := make(map[uint64]bool)
+	for _, tables := range v.tables {
+		for _, t := range tables {
+			tablesMap[t.file.Num()] = false
 		}
 	}
 
-	ff, err := s.getFiles(storage.TypeAll)
+	files, err := db.s.getFiles(storage.TypeAll)
 	if err != nil {
 		return err
 	}
 
 	var nTables int
 	var rem []storage.File
-	for _, f := range ff {
+	for _, f := range files {
 		keep := true
 		switch f.Type() {
 		case storage.TypeManifest:
-			keep = f.Num() >= s.manifestFile.Num()
+			keep = f.Num() >= db.s.manifestFile.Num()
 		case storage.TypeJournal:
-			if d.frozenJournalFile != nil {
-				keep = f.Num() >= d.frozenJournalFile.Num()
+			if db.frozenJournalFile != nil {
+				keep = f.Num() >= db.frozenJournalFile.Num()
 			} else {
-				keep = f.Num() >= d.journalFile.Num()
+				keep = f.Num() >= db.journalFile.Num()
 			}
 		case storage.TypeTable:
-			_, keep = tables[f.Num()]
+			_, keep = tablesMap[f.Num()]
 			if keep {
-				tables[f.Num()] = true
+				tablesMap[f.Num()] = true
 				nTables++
 			}
 		}
@@ -75,18 +77,18 @@ func (d *DB) checkAndCleanFiles() error {
 		}
 	}
 
-	if nTables != len(tables) {
-		for num, present := range tables {
+	if nTables != len(tablesMap) {
+		for num, present := range tablesMap {
 			if !present {
-				s.logf("db@janitor table missing @%d", num)
+				db.logf("db@janitor table missing @%d", num)
 			}
 		}
 		return ErrCorrupted{Type: MissingFiles, Err: errors.New("leveldb: table files missing")}
 	}
 
-	s.logf("db@janitor F·%d G·%d", len(ff), len(rem))
+	db.logf("db@janitor F·%d G·%d", len(files), len(rem))
 	for _, f := range rem {
-		s.logf("db@janitor removing %s-%d", f.Type(), f.Num())
+		db.logf("db@janitor removing %s-%d", f.Type(), f.Num())
 		if err := f.Remove(); err != nil {
 			return err
 		}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_write.go

@@ -14,64 +14,68 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
-func (d *DB) writeJournal(b *Batch) error {
-	w, err := d.journal.Next()
+func (db *DB) writeJournal(b *Batch) error {
+	w, err := db.journal.Next()
 	if err != nil {
 		return err
 	}
 	if _, err := w.Write(b.encode()); err != nil {
 		return err
 	}
-	if err := d.journal.Flush(); err != nil {
+	if err := db.journal.Flush(); err != nil {
 		return err
 	}
 	if b.sync {
-		return d.journalWriter.Sync()
+		return db.journalWriter.Sync()
 	}
 	return nil
 }
 
-func (d *DB) jWriter() {
-	defer d.closeW.Done()
+func (db *DB) jWriter() {
+	defer db.closeW.Done()
 	for {
 		select {
-		case b := <-d.journalC:
+		case b := <-db.journalC:
 			if b != nil {
-				d.journalAckC <- d.writeJournal(b)
+				db.journalAckC <- db.writeJournal(b)
 			}
-		case _, _ = <-d.closeC:
+		case _, _ = <-db.closeC:
 			return
 		}
 	}
 }
 
-func (d *DB) rotateMem(n int) (mem *memdb.DB, err error) {
+func (db *DB) rotateMem(n int) (mem *memDB, err error) {
 	// Wait for pending memdb compaction.
-	err = d.compSendIdle(d.mcompCmdC)
+	err = db.compSendIdle(db.mcompCmdC)
 	if err != nil {
 		return
 	}
 
 	// Create new memdb and journal.
-	mem, err = d.newMem(n)
+	mem, err = db.newMem(n)
 	if err != nil {
 		return
 	}
 
 	// Schedule memdb compaction.
-	d.compTrigger(d.mcompTriggerC)
+	db.compTrigger(db.mcompTriggerC)
 	return
 }
 
-func (d *DB) flush(n int) (mem *memdb.DB, nn int, err error) {
-	s := d.s
-
+func (db *DB) flush(n int) (mem *memDB, nn int, err error) {
 	delayed := false
-	flush := func() bool {
-		v := s.version()
+	flush := func() (retry bool) {
+		v := db.s.version()
 		defer v.release()
-		mem = d.getEffectiveMem()
-		nn = mem.Free()
+		mem = db.getEffectiveMem()
+		defer func() {
+			if retry {
+				mem.decref()
+				mem = nil
+			}
+		}()
+		nn = mem.mdb.Free()
 		switch {
 		case v.tLen(0) >= kL0_SlowdownWritesTrigger && !delayed:
 			delayed = true
@@ -80,18 +84,23 @@ func (d *DB) flush(n int) (mem *memdb.DB, nn int, err error) {
 			return false
 		case v.tLen(0) >= kL0_StopWritesTrigger:
 			delayed = true
-			err = d.compSendIdle(d.tcompCmdC)
+			err = db.compSendIdle(db.tcompCmdC)
 			if err != nil {
 				return false
 			}
 		default:
 			// Allow memdb to grow if it has no entry.
-			if mem.Len() == 0 {
+			if mem.mdb.Len() == 0 {
 				nn = n
-				return false
+			} else {
+				mem.decref()
+				mem, err = db.rotateMem(n)
+				if err == nil {
+					nn = mem.mdb.Free()
+				} else {
+					nn = 0
+				}
 			}
-			mem, err = d.rotateMem(n)
-			nn = mem.Free()
 			return false
 		}
 		return true
@@ -100,7 +109,7 @@ func (d *DB) flush(n int) (mem *memdb.DB, nn int, err error) {
 	for flush() {
 	}
 	if delayed {
-		s.logf("db@write delayed T·%v", time.Since(start))
+		db.logf("db@write delayed T·%v", time.Since(start))
 	}
 	return
 }
@@ -109,8 +118,8 @@ func (d *DB) flush(n int) (mem *memdb.DB, nn int, err error) {
 // sequentially.
 //
 // It is safe to modify the contents of the arguments after Write returns.
-func (d *DB) Write(b *Batch, wo *opt.WriteOptions) (err error) {
-	err = d.ok()
+func (db *DB) Write(b *Batch, wo *opt.WriteOptions) (err error) {
+	err = db.ok()
 	if err != nil || b == nil || b.len() == 0 {
 		return
 	}
@@ -120,28 +129,29 @@ func (d *DB) Write(b *Batch, wo *opt.WriteOptions) (err error) {
 	// The write happen synchronously.
 retry:
 	select {
-	case d.writeC <- b:
-		if <-d.writeMergedC {
-			return <-d.writeAckC
+	case db.writeC <- b:
+		if <-db.writeMergedC {
+			return <-db.writeAckC
 		}
 		goto retry
-	case d.writeLockC <- struct{}{}:
-	case _, _ = <-d.closeC:
+	case db.writeLockC <- struct{}{}:
+	case _, _ = <-db.closeC:
 		return ErrClosed
 	}
 
 	merged := 0
 	defer func() {
-		<-d.writeLockC
+		<-db.writeLockC
 		for i := 0; i < merged; i++ {
-			d.writeAckC <- err
+			db.writeAckC <- err
 		}
 	}()
 
-	mem, memFree, err := d.flush(b.size())
+	mem, memFree, err := db.flush(b.size())
 	if err != nil {
 		return
 	}
+	defer mem.decref()
 
 	// Calculate maximum size of the batch.
 	m := 1 << 20
@@ -154,13 +164,13 @@ retry:
 drain:
 	for b.size() < m && !b.sync {
 		select {
-		case nb := <-d.writeC:
+		case nb := <-db.writeC:
 			if b.size()+nb.size() <= m {
 				b.append(nb)
-				d.writeMergedC <- true
+				db.writeMergedC <- true
 				merged++
 			} else {
-				d.writeMergedC <- false
+				db.writeMergedC <- false
 				break drain
 			}
 		default:
@@ -169,44 +179,44 @@ drain:
 	}
 
 	// Set batch first seq number relative from last seq.
-	b.seq = d.seq + 1
+	b.seq = db.seq + 1
 
 	// Write journal concurrently if it is large enough.
 	if b.size() >= (128 << 10) {
 		// Push the write batch to the journal writer
 		select {
-		case _, _ = <-d.closeC:
+		case _, _ = <-db.closeC:
 			err = ErrClosed
 			return
-		case d.journalC <- b:
+		case db.journalC <- b:
 			// Write into memdb
-			b.memReplay(mem)
+			b.memReplay(mem.mdb)
 		}
 		// Wait for journal writer
 		select {
-		case _, _ = <-d.closeC:
+		case _, _ = <-db.closeC:
 			err = ErrClosed
 			return
-		case err = <-d.journalAckC:
+		case err = <-db.journalAckC:
 			if err != nil {
 				// Revert memdb if error detected
-				b.revertMemReplay(mem)
+				b.revertMemReplay(mem.mdb)
 				return
 			}
 		}
 	} else {
-		err = d.writeJournal(b)
+		err = db.writeJournal(b)
 		if err != nil {
 			return
 		}
-		b.memReplay(mem)
+		b.memReplay(mem.mdb)
 	}
 
 	// Set last seq number.
-	d.addSeq(uint64(b.len()))
+	db.addSeq(uint64(b.len()))
 
 	if b.size() >= memFree {
-		d.rotateMem(0)
+		db.rotateMem(0)
 	}
 	return
 }
@@ -215,20 +225,20 @@ drain:
 // for that key; a DB is not a multi-map.
 //
 // It is safe to modify the contents of the arguments after Put returns.
-func (d *DB) Put(key, value []byte, wo *opt.WriteOptions) error {
+func (db *DB) Put(key, value []byte, wo *opt.WriteOptions) error {
 	b := new(Batch)
 	b.Put(key, value)
-	return d.Write(b, wo)
+	return db.Write(b, wo)
 }
 
 // Delete deletes the value for the given key. It returns ErrNotFound if
 // the DB does not contain the key.
 //
 // It is safe to modify the contents of the arguments after Delete returns.
-func (d *DB) Delete(key []byte, wo *opt.WriteOptions) error {
+func (db *DB) Delete(key []byte, wo *opt.WriteOptions) error {
 	b := new(Batch)
 	b.Delete(key)
-	return d.Write(b, wo)
+	return db.Write(b, wo)
 }
 
 func isMemOverlaps(icmp *iComparer, mem *memdb.DB, min, max []byte) bool {
@@ -247,33 +257,34 @@ func isMemOverlaps(icmp *iComparer, mem *memdb.DB, min, max []byte) bool {
 // A nil Range.Start is treated as a key before all keys in the DB.
 // And a nil Range.Limit is treated as a key after all keys in the DB.
 // Therefore if both is nil then it will compact entire DB.
-func (d *DB) CompactRange(r util.Range) error {
-	if err := d.ok(); err != nil {
+func (db *DB) CompactRange(r util.Range) error {
+	if err := db.ok(); err != nil {
 		return err
 	}
 
 	select {
-	case d.writeLockC <- struct{}{}:
-	case _, _ = <-d.closeC:
+	case db.writeLockC <- struct{}{}:
+	case _, _ = <-db.closeC:
 		return ErrClosed
 	}
 
 	// Check for overlaps in memdb.
-	mem := d.getEffectiveMem()
-	if isMemOverlaps(d.s.icmp, mem, r.Start, r.Limit) {
+	mem := db.getEffectiveMem()
+	defer mem.decref()
+	if isMemOverlaps(db.s.icmp, mem.mdb, r.Start, r.Limit) {
 		// Memdb compaction.
-		if _, err := d.rotateMem(0); err != nil {
-			<-d.writeLockC
+		if _, err := db.rotateMem(0); err != nil {
+			<-db.writeLockC
 			return err
 		}
-		<-d.writeLockC
-		if err := d.compSendIdle(d.mcompCmdC); err != nil {
+		<-db.writeLockC
+		if err := db.compSendIdle(db.mcompCmdC); err != nil {
 			return err
 		}
 	} else {
-		<-d.writeLockC
+		<-db.writeLockC
 	}
 
 	// Table compaction.
-	return d.compSendRange(d.tcompCmdC, -1, r.Start, r.Limit)
+	return db.compSendRange(db.tcompCmdC, -1, r.Start, r.Limit)
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/doc.go

@@ -37,6 +37,16 @@
 //	err = iter.Error()
 //	...
 //
+// Iterate over subset of database content with a particular prefix:
+//	iter := db.NewIterator(util.BytesPrefix([]byte("foo-")), nil)
+//	for iter.Next() {
+//		// Use key/value.
+//		...
+//	}
+//	iter.Release()
+//	err = iter.Error()
+//	...
+//
 // Seek-then-Iterate:
 //
 // 	iter := db.NewIterator(nil, nil)

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/external_test.go

@@ -21,7 +21,7 @@ var _ = testutil.Defer(func() {
 			BlockRestartInterval: 5,
 			BlockSize:            50,
 			Compression:          opt.NoCompression,
-			MaxOpenFiles:         0,
+			CachedOpenFiles:      -1,
 			Strict:               opt.StrictAll,
 			WriteBuffer:          1000,
 		}
@@ -36,22 +36,21 @@ var _ = testutil.Defer(func() {
 				testutil.DoDBTesting(&t)
 				db.TestClose()
 				done <- true
-			}, 9.0)
+			}, 20.0)
 		})
 
 		Describe("read test", func() {
-			testutil.AllKeyValueTesting(nil, func(kv testutil.KeyValue) testutil.DB {
+			testutil.AllKeyValueTesting(nil, nil, func(kv testutil.KeyValue) testutil.DB {
 				// Building the DB.
 				db := newTestingDB(o, nil, nil)
 				kv.IterateShuffled(nil, func(i int, key, value []byte) {
 					err := db.TestPut(key, value)
 					Expect(err).NotTo(HaveOccurred())
 				})
-				testutil.Defer("teardown", func() {
-					db.TestClose()
-				})
 
 				return db
+			}, func(db testutil.DB) {
+				db.(*testingDB).TestClose()
 			})
 		})
 	})

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/go13_bench_test.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// +build go1.3
+
+package leveldb
+
+import (
+	"sync/atomic"
+	"testing"
+)
+
+func BenchmarkDBReadConcurrent(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.fill()
+	p.gc()
+	defer p.close()
+
+	b.ResetTimer()
+	b.SetBytes(116)
+
+	b.RunParallel(func(pb *testing.PB) {
+		iter := p.newIter()
+		defer iter.Release()
+		for pb.Next() && iter.Next() {
+		}
+	})
+}
+
+func BenchmarkDBReadConcurrent2(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.fill()
+	p.gc()
+	defer p.close()
+
+	b.ResetTimer()
+	b.SetBytes(116)
+
+	var dir uint32
+	b.RunParallel(func(pb *testing.PB) {
+		iter := p.newIter()
+		defer iter.Release()
+		if atomic.AddUint32(&dir, 1)%2 == 0 {
+			for pb.Next() && iter.Next() {
+			}
+		} else {
+			if pb.Next() && iter.Last() {
+				for pb.Next() && iter.Prev() {
+				}
+			}
+		}
+	})
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/array_iter.go

@@ -40,13 +40,19 @@ type basicArrayIterator struct {
 	util.BasicReleaser
 	array BasicArray
 	pos   int
+	err   error
 }
 
 func (i *basicArrayIterator) Valid() bool {
-	return i.pos >= 0 && i.pos < i.array.Len()
+	return i.pos >= 0 && i.pos < i.array.Len() && !i.Released()
 }
 
 func (i *basicArrayIterator) First() bool {
+	if i.Released() {
+		i.err = ErrIterReleased
+		return false
+	}
+
 	if i.array.Len() == 0 {
 		i.pos = -1
 		return false
@@ -56,6 +62,11 @@ func (i *basicArrayIterator) First() bool {
 }
 
 func (i *basicArrayIterator) Last() bool {
+	if i.Released() {
+		i.err = ErrIterReleased
+		return false
+	}
+
 	n := i.array.Len()
 	if n == 0 {
 		i.pos = 0
@@ -66,6 +77,11 @@ func (i *basicArrayIterator) Last() bool {
 }
 
 func (i *basicArrayIterator) Seek(key []byte) bool {
+	if i.Released() {
+		i.err = ErrIterReleased
+		return false
+	}
+
 	n := i.array.Len()
 	if n == 0 {
 		i.pos = 0
@@ -79,6 +95,11 @@ func (i *basicArrayIterator) Seek(key []byte) bool {
 }
 
 func (i *basicArrayIterator) Next() bool {
+	if i.Released() {
+		i.err = ErrIterReleased
+		return false
+	}
+
 	i.pos++
 	if n := i.array.Len(); i.pos >= n {
 		i.pos = n
@@ -88,6 +109,11 @@ func (i *basicArrayIterator) Next() bool {
 }
 
 func (i *basicArrayIterator) Prev() bool {
+	if i.Released() {
+		i.err = ErrIterReleased
+		return false
+	}
+
 	i.pos--
 	if i.pos < 0 {
 		i.pos = -1
@@ -96,7 +122,7 @@ func (i *basicArrayIterator) Prev() bool {
 	return true
 }
 
-func (i *basicArrayIterator) Error() error { return nil }
+func (i *basicArrayIterator) Error() error { return i.err }
 
 type arrayIterator struct {
 	basicArrayIterator

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter.go

@@ -26,9 +26,10 @@ type indexedIterator struct {
 	strict    bool
 	strictGet bool
 
-	data Iterator
-	err  error
-	errf func(err error)
+	data   Iterator
+	err    error
+	errf   func(err error)
+	closed bool
 }
 
 func (i *indexedIterator) setData() {
@@ -50,6 +51,15 @@ func (i *indexedIterator) clearData() {
 	i.data = nil
 }
 
+func (i *indexedIterator) indexErr() {
+	if err := i.index.Error(); err != nil {
+		if i.errf != nil {
+			i.errf(err)
+		}
+		i.err = err
+	}
+}
+
 func (i *indexedIterator) dataErr() bool {
 	if i.errf != nil {
 		if err := i.data.Error(); err != nil {
@@ -72,9 +82,13 @@ func (i *indexedIterator) Valid() bool {
 func (i *indexedIterator) First() bool {
 	if i.err != nil {
 		return false
+	} else if i.Released() {
+		i.err = ErrIterReleased
+		return false
 	}
 
 	if !i.index.First() {
+		i.indexErr()
 		i.clearData()
 		return false
 	}
@@ -85,9 +99,13 @@ func (i *indexedIterator) First() bool {
 func (i *indexedIterator) Last() bool {
 	if i.err != nil {
 		return false
+	} else if i.Released() {
+		i.err = ErrIterReleased
+		return false
 	}
 
 	if !i.index.Last() {
+		i.indexErr()
 		i.clearData()
 		return false
 	}
@@ -105,9 +123,13 @@ func (i *indexedIterator) Last() bool {
 func (i *indexedIterator) Seek(key []byte) bool {
 	if i.err != nil {
 		return false
+	} else if i.Released() {
+		i.err = ErrIterReleased
+		return false
 	}
 
 	if !i.index.Seek(key) {
+		i.indexErr()
 		i.clearData()
 		return false
 	}
@@ -125,6 +147,9 @@ func (i *indexedIterator) Seek(key []byte) bool {
 func (i *indexedIterator) Next() bool {
 	if i.err != nil {
 		return false
+	} else if i.Released() {
+		i.err = ErrIterReleased
+		return false
 	}
 
 	switch {
@@ -136,6 +161,7 @@ func (i *indexedIterator) Next() bool {
 		fallthrough
 	case i.data == nil:
 		if !i.index.Next() {
+			i.indexErr()
 			return false
 		}
 		i.setData()
@@ -147,6 +173,9 @@ func (i *indexedIterator) Next() bool {
 func (i *indexedIterator) Prev() bool {
 	if i.err != nil {
 		return false
+	} else if i.Released() {
+		i.err = ErrIterReleased
+		return false
 	}
 
 	switch {
@@ -158,6 +187,7 @@ func (i *indexedIterator) Prev() bool {
 		fallthrough
 	case i.data == nil:
 		if !i.index.Prev() {
+			i.indexErr()
 			return false
 		}
 		i.setData()

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/iter.go

@@ -14,6 +14,10 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
+var (
+	ErrIterReleased = errors.New("leveldb/iterator: iterator released")
+)
+
 // IteratorSeeker is the interface that wraps the 'seeks method'.
 type IteratorSeeker interface {
 	// First moves the iterator to the first key/value pair. If the iterator
@@ -100,28 +104,13 @@ type ErrorCallbackSetter interface {
 }
 
 type emptyIterator struct {
-	releaser util.Releaser
-	released bool
-	err      error
+	util.BasicReleaser
+	err error
 }
 
 func (i *emptyIterator) rErr() {
-	if i.err == nil && i.released {
-		i.err = errors.New("leveldb/iterator: iterator released")
-	}
-}
-
-func (i *emptyIterator) Release() {
-	if i.releaser != nil {
-		i.releaser.Release()
-		i.releaser = nil
-	}
-	i.released = true
-}
-
-func (i *emptyIterator) SetReleaser(releaser util.Releaser) {
-	if !i.released {
-		i.releaser = releaser
+	if i.err == nil && i.Released() {
+		i.err = ErrIterReleased
 	}
 }
 

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter.go

@@ -7,16 +7,10 @@
 package iterator
 
 import (
-	"errors"
-
 	"github.com/syndtr/goleveldb/leveldb/comparer"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
-var (
-	ErrIterReleased = errors.New("leveldb/iterator: iterator released")
-)
-
 type dir int
 
 const (
@@ -274,9 +268,13 @@ func (i *mergedIterator) Release() {
 }
 
 func (i *mergedIterator) SetReleaser(releaser util.Releaser) {
-	if i.dir != dirReleased {
-		i.releaser = releaser
+	if i.dir == dirReleased {
+		panic(util.ErrReleased)
 	}
+	if i.releaser != nil && releaser != nil {
+		panic(util.ErrHasReleaser)
+	}
+	i.releaser = releaser
 }
 
 func (i *mergedIterator) Error() error {

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal/journal.go

@@ -103,18 +103,18 @@ type flusher interface {
 	Flush() error
 }
 
-// DroppedError is the error type that passed to Dropper.Drop method.
-type DroppedError struct {
+// ErrCorrupted is the error type that generated by corrupted block or chunk.
+type ErrCorrupted struct {
 	Size   int
 	Reason string
 }
 
-func (e DroppedError) Error() string {
-	return fmt.Sprintf("leveldb/journal: dropped %d bytes: %s", e.Size, e.Reason)
+func (e ErrCorrupted) Error() string {
+	return fmt.Sprintf("leveldb/journal: block/chunk corrupted: %s (%d bytes)", e.Reason, e.Size)
 }
 
 // Dropper is the interface that wrap simple Drop method. The Drop
-// method will be called when the journal reader dropping a chunk.
+// method will be called when the journal reader dropping a block or chunk.
 type Dropper interface {
 	Drop(err error)
 }
@@ -158,76 +158,78 @@ func NewReader(r io.Reader, dropper Dropper, strict, checksum bool) *Reader {
 	}
 }
 
+var errSkip = errors.New("leveldb/journal: skipped")
+
+func (r *Reader) corrupt(n int, reason string, skip bool) error {
+	if r.dropper != nil {
+		r.dropper.Drop(ErrCorrupted{n, reason})
+	}
+	if r.strict && !skip {
+		r.err = ErrCorrupted{n, reason}
+		return r.err
+	}
+	return errSkip
+}
+
 // nextChunk sets r.buf[r.i:r.j] to hold the next chunk's payload, reading the
 // next block into the buffer if necessary.
-func (r *Reader) nextChunk(wantFirst, skip bool) error {
+func (r *Reader) nextChunk(first bool) error {
 	for {
 		if r.j+headerSize <= r.n {
 			checksum := binary.LittleEndian.Uint32(r.buf[r.j+0 : r.j+4])
 			length := binary.LittleEndian.Uint16(r.buf[r.j+4 : r.j+6])
 			chunkType := r.buf[r.j+6]
 
-			var err error
 			if checksum == 0 && length == 0 && chunkType == 0 {
 				// Drop entire block.
-				err = DroppedError{r.n - r.j, "zero header"}
+				m := r.n - r.j
 				r.i = r.n
 				r.j = r.n
+				return r.corrupt(m, "zero header", false)
 			} else {
 				m := r.n - r.j
 				r.i = r.j + headerSize
 				r.j = r.j + headerSize + int(length)
 				if r.j > r.n {
 					// Drop entire block.
-					err = DroppedError{m, "chunk length overflows block"}
 					r.i = r.n
 					r.j = r.n
+					return r.corrupt(m, "chunk length overflows block", false)
 				} else if r.checksum && checksum != util.NewCRC(r.buf[r.i-1:r.j]).Value() {
 					// Drop entire block.
-					err = DroppedError{m, "checksum mismatch"}
 					r.i = r.n
 					r.j = r.n
+					return r.corrupt(m, "checksum mismatch", false)
 				}
 			}
-			if wantFirst && err == nil && chunkType != fullChunkType && chunkType != firstChunkType {
-				if skip {
-					// The chunk are intentionally skipped.
-					if chunkType == lastChunkType {
-						skip = false
-					}
-					continue
-				} else {
-					// Drop the chunk.
-					err = DroppedError{r.j - r.i + headerSize, "orphan chunk"}
-				}
+			if first && chunkType != fullChunkType && chunkType != firstChunkType {
+				m := r.j - r.i
+				r.i = r.j
+				// Report the error, but skip it.
+				return r.corrupt(m+headerSize, "orphan chunk", true)
 			}
-			if err == nil {
-				r.last = chunkType == fullChunkType || chunkType == lastChunkType
-			} else {
-				if r.dropper != nil {
-					r.dropper.Drop(err)
-				}
-				if r.strict {
-					r.err = err
-				}
+			r.last = chunkType == fullChunkType || chunkType == lastChunkType
+			return nil
+		}
+
+		// The last block.
+		if r.n < blockSize && r.n > 0 {
+			if !first {
+				return r.corrupt(0, "missing chunk part", false)
 			}
+			r.err = io.EOF
+			return r.err
+		}
+
+		// Read block.
+		n, err := io.ReadFull(r.r, r.buf[:])
+		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 			return err
 		}
-		if r.n < blockSize && r.n > 0 {
-			// This is the last block.
-			if r.j != r.n {
-				r.err = io.ErrUnexpectedEOF
-			} else {
-				r.err = io.EOF
-			}
-			return r.err
-		}
-		n, err := io.ReadFull(r.r, r.buf[:])
-		if err != nil && err != io.ErrUnexpectedEOF {
-			r.err = err
-			return r.err
-		}
 		if n == 0 {
+			if !first {
+				return r.corrupt(0, "missing chunk part", false)
+			}
 			r.err = io.EOF
 			return r.err
 		}
@@ -237,29 +239,26 @@ func (r *Reader) nextChunk(wantFirst, skip bool) error {
 
 // Next returns a reader for the next journal. It returns io.EOF if there are no
 // more journals. The reader returned becomes stale after the next Next call,
-// and should no longer be used.
+// and should no longer be used. If strict is false, the reader will returns
+// io.ErrUnexpectedEOF error when found corrupted journal.
 func (r *Reader) Next() (io.Reader, error) {
 	r.seq++
 	if r.err != nil {
 		return nil, r.err
 	}
-	skip := !r.last
+	r.i = r.j
 	for {
-		r.i = r.j
-		if r.nextChunk(true, skip) != nil {
-			// So that 'orphan chunk' drop will be reported.
-			skip = false
-		} else {
+		if err := r.nextChunk(true); err == nil {
 			break
-		}
-		if r.err != nil {
-			return nil, r.err
+		} else if err != errSkip {
+			return nil, err
 		}
 	}
 	return &singleReader{r, r.seq, nil}, nil
 }
 
-// Reset resets the journal reader, allows reuse of the journal reader.
+// Reset resets the journal reader, allows reuse of the journal reader. Reset returns
+// last accumulated error.
 func (r *Reader) Reset(reader io.Reader, dropper Dropper, strict, checksum bool) error {
 	r.seq++
 	err := r.err
@@ -296,7 +295,11 @@ func (x *singleReader) Read(p []byte) (int, error) {
 		if r.last {
 			return 0, io.EOF
 		}
-		if x.err = r.nextChunk(false, false); x.err != nil {
+		x.err = r.nextChunk(false)
+		if x.err != nil {
+			if x.err == errSkip {
+				x.err = io.ErrUnexpectedEOF
+			}
 			return 0, x.err
 		}
 	}
@@ -320,7 +323,11 @@ func (x *singleReader) ReadByte() (byte, error) {
 		if r.last {
 			return 0, io.EOF
 		}
-		if x.err = r.nextChunk(false, false); x.err != nil {
+		x.err = r.nextChunk(false)
+		if x.err != nil {
+			if x.err == errSkip {
+				x.err = io.ErrUnexpectedEOF
+			}
 			return 0, x.err
 		}
 	}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal/journal_test.go

@@ -12,6 +12,7 @@ package journal
 
 import (
 	"bytes"
+	"encoding/binary"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -326,3 +327,492 @@ func TestStaleWriter(t *testing.T) {
 		t.Fatalf("stale write #1: unexpected error: %v", err)
 	}
 }
+
+func TestCorrupt_MissingLastBlock(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+
+	// First record.
+	ww, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-1024)); err != nil {
+		t.Fatalf("write #0: unexpected error: %v", err)
+	}
+
+	// Second record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
+		t.Fatalf("write #1: unexpected error: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Cut the last block.
+	b := buf.Bytes()[:blockSize]
+	r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
+
+	// First read.
+	rr, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #0: %v", err)
+	}
+	if n != blockSize-1024 {
+		t.Fatalf("read #0: got %d bytes want %d", n, blockSize-1024)
+	}
+
+	// Second read.
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != io.ErrUnexpectedEOF {
+		t.Fatalf("read #1: unexpected error: %v", err)
+	}
+
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("last next: unexpected error: %v", err)
+	}
+}
+
+func TestCorrupt_CorruptedFirstBlock(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+
+	// First record.
+	ww, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
+		t.Fatalf("write #0: unexpected error: %v", err)
+	}
+
+	// Second record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
+		t.Fatalf("write #1: unexpected error: %v", err)
+	}
+
+	// Third record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
+		t.Fatalf("write #2: unexpected error: %v", err)
+	}
+
+	// Fourth record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+2)); err != nil {
+		t.Fatalf("write #3: unexpected error: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	b := buf.Bytes()
+	// Corrupting block #0.
+	for i := 0; i < 1024; i++ {
+		b[i] = '1'
+	}
+
+	r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
+
+	// First read (third record).
+	rr, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #0: %v", err)
+	}
+	if want := int64(blockSize-headerSize) + 1; n != want {
+		t.Fatalf("read #0: got %d bytes want %d", n, want)
+	}
+
+	// Second read (fourth record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #1: %v", err)
+	}
+	if want := int64(blockSize-headerSize) + 2; n != want {
+		t.Fatalf("read #1: got %d bytes want %d", n, want)
+	}
+
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("last next: unexpected error: %v", err)
+	}
+}
+
+func TestCorrupt_CorruptedMiddleBlock(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+
+	// First record.
+	ww, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
+		t.Fatalf("write #0: unexpected error: %v", err)
+	}
+
+	// Second record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
+		t.Fatalf("write #1: unexpected error: %v", err)
+	}
+
+	// Third record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
+		t.Fatalf("write #2: unexpected error: %v", err)
+	}
+
+	// Fourth record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+2)); err != nil {
+		t.Fatalf("write #3: unexpected error: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	b := buf.Bytes()
+	// Corrupting block #1.
+	for i := 0; i < 1024; i++ {
+		b[blockSize+i] = '1'
+	}
+
+	r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
+
+	// First read (first record).
+	rr, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #0: %v", err)
+	}
+	if want := int64(blockSize / 2); n != want {
+		t.Fatalf("read #0: got %d bytes want %d", n, want)
+	}
+
+	// Second read (second record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != io.ErrUnexpectedEOF {
+		t.Fatalf("read #1: unexpected error: %v", err)
+	}
+
+	// Third read (fourth record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #2: %v", err)
+	}
+	if want := int64(blockSize-headerSize) + 2; n != want {
+		t.Fatalf("read #2: got %d bytes want %d", n, want)
+	}
+
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("last next: unexpected error: %v", err)
+	}
+}
+
+func TestCorrupt_CorruptedLastBlock(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+
+	// First record.
+	ww, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
+		t.Fatalf("write #0: unexpected error: %v", err)
+	}
+
+	// Second record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
+		t.Fatalf("write #1: unexpected error: %v", err)
+	}
+
+	// Third record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
+		t.Fatalf("write #2: unexpected error: %v", err)
+	}
+
+	// Fourth record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+2)); err != nil {
+		t.Fatalf("write #3: unexpected error: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	b := buf.Bytes()
+	// Corrupting block #3.
+	for i := len(b) - 1; i > len(b)-1024; i-- {
+		b[i] = '1'
+	}
+
+	r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
+
+	// First read (first record).
+	rr, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #0: %v", err)
+	}
+	if want := int64(blockSize / 2); n != want {
+		t.Fatalf("read #0: got %d bytes want %d", n, want)
+	}
+
+	// Second read (second record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #1: %v", err)
+	}
+	if want := int64(blockSize - headerSize); n != want {
+		t.Fatalf("read #1: got %d bytes want %d", n, want)
+	}
+
+	// Third read (third record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #2: %v", err)
+	}
+	if want := int64(blockSize-headerSize) + 1; n != want {
+		t.Fatalf("read #2: got %d bytes want %d", n, want)
+	}
+
+	// Fourth read (fourth record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != io.ErrUnexpectedEOF {
+		t.Fatalf("read #3: unexpected error: %v", err)
+	}
+
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("last next: unexpected error: %v", err)
+	}
+}
+
+func TestCorrupt_FirstChuckLengthOverflow(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+
+	// First record.
+	ww, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
+		t.Fatalf("write #0: unexpected error: %v", err)
+	}
+
+	// Second record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
+		t.Fatalf("write #1: unexpected error: %v", err)
+	}
+
+	// Third record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
+		t.Fatalf("write #2: unexpected error: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	b := buf.Bytes()
+	// Corrupting record #1.
+	x := blockSize
+	binary.LittleEndian.PutUint16(b[x+4:], 0xffff)
+
+	r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
+
+	// First read (first record).
+	rr, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #0: %v", err)
+	}
+	if want := int64(blockSize / 2); n != want {
+		t.Fatalf("read #0: got %d bytes want %d", n, want)
+	}
+
+	// Second read (second record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != io.ErrUnexpectedEOF {
+		t.Fatalf("read #1: unexpected error: %v", err)
+	}
+
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("last next: unexpected error: %v", err)
+	}
+}
+
+func TestCorrupt_MiddleChuckLengthOverflow(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+
+	// First record.
+	ww, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
+		t.Fatalf("write #0: unexpected error: %v", err)
+	}
+
+	// Second record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
+		t.Fatalf("write #1: unexpected error: %v", err)
+	}
+
+	// Third record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
+		t.Fatalf("write #2: unexpected error: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	b := buf.Bytes()
+	// Corrupting record #1.
+	x := blockSize/2 + headerSize
+	binary.LittleEndian.PutUint16(b[x+4:], 0xffff)
+
+	r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
+
+	// First read (first record).
+	rr, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #0: %v", err)
+	}
+	if want := int64(blockSize / 2); n != want {
+		t.Fatalf("read #0: got %d bytes want %d", n, want)
+	}
+
+	// Second read (third record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #1: %v", err)
+	}
+	if want := int64(blockSize-headerSize) + 1; n != want {
+		t.Fatalf("read #1: got %d bytes want %d", n, want)
+	}
+
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("last next: unexpected error: %v", err)
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go

@@ -8,6 +8,7 @@
 package memdb
 
 import (
+	"errors"
 	"math/rand"
 	"sync"
 
@@ -17,7 +18,8 @@ import (
 )
 
 var (
-	ErrNotFound = util.ErrNotFound
+	ErrNotFound     = util.ErrNotFound
+	ErrIterReleased = errors.New("leveldb/memdb: iterator released")
 )
 
 const tMaxHeight = 12
@@ -29,6 +31,7 @@ type dbIter struct {
 	node       int
 	forward    bool
 	key, value []byte
+	err        error
 }
 
 func (i *dbIter) fill(checkStart, checkLimit bool) bool {
@@ -59,6 +62,11 @@ func (i *dbIter) Valid() bool {
 }
 
 func (i *dbIter) First() bool {
+	if i.Released() {
+		i.err = ErrIterReleased
+		return false
+	}
+
 	i.forward = true
 	i.p.mu.RLock()
 	defer i.p.mu.RUnlock()
@@ -71,9 +79,11 @@ func (i *dbIter) First() bool {
 }
 
 func (i *dbIter) Last() bool {
-	if i.p == nil {
+	if i.Released() {
+		i.err = ErrIterReleased
 		return false
 	}
+
 	i.forward = false
 	i.p.mu.RLock()
 	defer i.p.mu.RUnlock()
@@ -86,9 +96,11 @@ func (i *dbIter) Last() bool {
 }
 
 func (i *dbIter) Seek(key []byte) bool {
-	if i.p == nil {
+	if i.Released() {
+		i.err = ErrIterReleased
 		return false
 	}
+
 	i.forward = true
 	i.p.mu.RLock()
 	defer i.p.mu.RUnlock()
@@ -100,9 +112,11 @@ func (i *dbIter) Seek(key []byte) bool {
 }
 
 func (i *dbIter) Next() bool {
-	if i.p == nil {
+	if i.Released() {
+		i.err = ErrIterReleased
 		return false
 	}
+
 	if i.node == 0 {
 		if !i.forward {
 			return i.First()
@@ -117,9 +131,11 @@ func (i *dbIter) Next() bool {
 }
 
 func (i *dbIter) Prev() bool {
-	if i.p == nil {
+	if i.Released() {
+		i.err = ErrIterReleased
 		return false
 	}
+
 	if i.node == 0 {
 		if i.forward {
 			return i.Last()
@@ -141,10 +157,10 @@ func (i *dbIter) Value() []byte {
 	return i.value
 }
 
-func (i *dbIter) Error() error { return nil }
+func (i *dbIter) Error() error { return i.err }
 
 func (i *dbIter) Release() {
-	if i.p != nil {
+	if !i.Released() {
 		i.p = nil
 		i.node = 0
 		i.key = nil
@@ -437,6 +453,8 @@ func (p *DB) Reset() {
 // New creates a new initalized in-memory key/value DB. The capacity
 // is the initial key/value buffer capacity. The capacity is advisory,
 // not enforced.
+//
+// The returned DB instance is goroutine-safe.
 func New(cmp comparer.BasicComparer, capacity int) *DB {
 	p := &DB{
 		cmp:       cmp,

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb_test.go

@@ -129,7 +129,7 @@ var _ = testutil.Defer(func() {
 				}
 
 				return db
-			})
+			}, nil, nil)
 		})
 	})
 })

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go

@@ -24,16 +24,22 @@ const (
 	DefaultBlockRestartInterval = 16
 	DefaultBlockSize            = 4 * KiB
 	DefaultCompressionType      = SnappyCompression
-	DefaultMaxOpenFiles         = 1000
+	DefaultCachedOpenFiles      = 500
 	DefaultWriteBuffer          = 4 * MiB
 )
 
 type noCache struct{}
 
-func (noCache) SetCapacity(capacity int)               {}
-func (noCache) GetNamespace(id uint64) cache.Namespace { return nil }
-func (noCache) Purge(fin cache.PurgeFin)               {}
-func (noCache) Zap(closed bool)                        {}
+func (noCache) SetCapacity(capacity int)                     {}
+func (noCache) Capacity() int                                { return 0 }
+func (noCache) Used() int                                    { return 0 }
+func (noCache) Size() int                                    { return 0 }
+func (noCache) NumObjects() int                              { return 0 }
+func (noCache) GetNamespace(id uint64) cache.Namespace       { return nil }
+func (noCache) PurgeNamespace(id uint64, fin cache.PurgeFin) {}
+func (noCache) ZapNamespace(id uint64)                       {}
+func (noCache) Purge(fin cache.PurgeFin)                     {}
+func (noCache) Zap()                                         {}
 
 var NoCache cache.Cache = noCache{}
 
@@ -119,6 +125,13 @@ type Options struct {
 	// The default value is 4KiB.
 	BlockSize int
 
+	// CachedOpenFiles defines number of open files to kept around when not
+	// in-use, the counting includes still in-use files.
+	// Set this to negative value to disable caching.
+	//
+	// The default value is 500.
+	CachedOpenFiles int
+
 	// Comparer defines a total ordering over the space of []byte keys: a 'less
 	// than' relationship. The same comparison algorithm must be used for reads
 	// and writes over the lifetime of the DB.
@@ -159,13 +172,6 @@ type Options struct {
 	// The default value is nil.
 	Filter filter.Filter
 
-	// MaxOpenFiles defines maximum number of open files to kept around
-	// (cached). This is not an hard limit, actual open files may exceed
-	// the defined value.
-	//
-	// The default value is 1000.
-	MaxOpenFiles int
-
 	// Strict defines the DB strict level.
 	Strict Strict
 
@@ -207,6 +213,15 @@ func (o *Options) GetBlockSize() int {
 	return o.BlockSize
 }
 
+func (o *Options) GetCachedOpenFiles() int {
+	if o == nil || o.CachedOpenFiles == 0 {
+		return DefaultCachedOpenFiles
+	} else if o.CachedOpenFiles < 0 {
+		return 0
+	}
+	return o.CachedOpenFiles
+}
+
 func (o *Options) GetComparer() comparer.Comparer {
 	if o == nil || o.Comparer == nil {
 		return comparer.DefaultComparer
@@ -242,13 +257,6 @@ func (o *Options) GetFilter() filter.Filter {
 	return o.Filter
 }
 
-func (o *Options) GetMaxOpenFiles() int {
-	if o == nil || o.MaxOpenFiles <= 0 {
-		return DefaultMaxOpenFiles
-	}
-	return o.MaxOpenFiles
-}
-
 func (o *Options) GetStrict(strict Strict) bool {
 	if o == nil || o.Strict == 0 {
 		return DefaultStrict&strict != 0

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go

@@ -39,11 +39,12 @@ type session struct {
 	manifestWriter storage.Writer
 	manifestFile   storage.File
 
-	stCPtrs   [kNumLevels]iKey // compact pointers; need external synchronization
+	stCptrs   [kNumLevels]iKey // compact pointers; need external synchronization
 	stVersion *version         // current version
 	vmu       sync.Mutex
 }
 
+// Creates new initialized session instance.
 func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) {
 	if stor == nil {
 		return nil, os.ErrInvalid
@@ -57,7 +58,7 @@ func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) {
 		storLock: storLock,
 	}
 	s.setOptions(o)
-	s.tops = newTableOps(s, s.o.GetMaxOpenFiles())
+	s.tops = newTableOps(s, s.o.GetCachedOpenFiles())
 	s.setVersion(&version{s: s})
 	s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock D·DeletedEntry L·Level Q·SeqNum T·TimeElapsed")
 	return
@@ -81,6 +82,7 @@ func (s *session) close() {
 	s.stVersion = nil
 }
 
+// Release session lock.
 func (s *session) release() {
 	s.storLock.Release()
 }
@@ -132,8 +134,8 @@ func (s *session) recover() (err error) {
 		err = rec.decode(r)
 		if err == nil {
 			// save compact pointers
-			for _, rp := range rec.compactionPointers {
-				s.stCPtrs[rp.level] = iKey(rp.key)
+			for _, r := range rec.compactionPointers {
+				s.stCptrs[r.level] = iKey(r.ikey)
 			}
 			// commit record to version staging
 			staging.commit(rec)
@@ -195,16 +197,16 @@ func (s *session) pickCompaction() *compaction {
 	var t0 tFiles
 	if v.cScore >= 1 {
 		level = v.cLevel
-		cp := s.stCPtrs[level]
-		tt := v.tables[level]
-		for _, t := range tt {
-			if cp == nil || s.icmp.Compare(t.max, cp) > 0 {
+		cptr := s.stCptrs[level]
+		tables := v.tables[level]
+		for _, t := range tables {
+			if cptr == nil || s.icmp.Compare(t.imax, cptr) > 0 {
 				t0 = append(t0, t)
 				break
 			}
 		}
 		if len(t0) == 0 {
-			t0 = append(t0, tt[0])
+			t0 = append(t0, tables[0])
 		}
 	} else {
 		if p := atomic.LoadPointer(&v.cSeek); p != nil {
@@ -216,11 +218,10 @@ func (s *session) pickCompaction() *compaction {
 		}
 	}
 
-	c := &compaction{s: s, version: v, level: level}
+	c := &compaction{s: s, v: v, level: level}
 	if level == 0 {
-		min, max := t0.getRange(s.icmp)
-		t0 = nil
-		v.tables[0].getOverlaps(min.ukey(), max.ukey(), &t0, false, s.icmp.ucmp)
+		imin, imax := t0.getRange(s.icmp)
+		t0 = v.tables[0].getOverlaps(t0[:0], s.icmp, imin.ukey(), imax.ukey(), true)
 	}
 
 	c.tables[0] = t0
@@ -229,11 +230,10 @@ func (s *session) pickCompaction() *compaction {
 }
 
 // Create compaction from given level and range; need external synchronization.
-func (s *session) getCompactionRange(level int, min, max []byte) *compaction {
+func (s *session) getCompactionRange(level int, umin, umax []byte) *compaction {
 	v := s.version_NB()
 
-	var t0 tFiles
-	v.tables[level].getOverlaps(min, max, &t0, level != 0, s.icmp.ucmp)
+	t0 := v.tables[level].getOverlaps(nil, s.icmp, umin, umax, level == 0)
 	if len(t0) == 0 {
 		return nil
 	}
@@ -255,16 +255,16 @@ func (s *session) getCompactionRange(level int, min, max []byte) *compaction {
 		}
 	}
 
-	c := &compaction{s: s, version: v, level: level}
+	c := &compaction{s: s, v: v, level: level}
 	c.tables[0] = t0
 	c.expand()
 	return c
 }
 
-// compaction represent a compaction state
+// compaction represent a compaction state.
 type compaction struct {
-	s       *session
-	version *version
+	s *session
+	v *version
 
 	level  int
 	tables [2]tFiles
@@ -273,42 +273,36 @@ type compaction struct {
 	gpidx           int
 	seenKey         bool
 	overlappedBytes uint64
-	min, max        iKey
+	imin, imax      iKey
 
 	tPtrs [kNumLevels]int
 }
 
 // Expand compacted tables; need external synchronization.
 func (c *compaction) expand() {
-	s := c.s
-	v := c.version
-
 	level := c.level
-	vt0, vt1 := v.tables[level], v.tables[level+1]
+	vt0, vt1 := c.v.tables[level], c.v.tables[level+1]
 
 	t0, t1 := c.tables[0], c.tables[1]
-	min, max := t0.getRange(s.icmp)
-	vt1.getOverlaps(min.ukey(), max.ukey(), &t1, true, s.icmp.ucmp)
-
-	// Get entire range covered by compaction
-	amin, amax := append(t0, t1...).getRange(s.icmp)
+	imin, imax := t0.getRange(c.s.icmp)
+	t1 = vt1.getOverlaps(t1, c.s.icmp, imin.ukey(), imax.ukey(), false)
+	// Get entire range covered by compaction.
+	amin, amax := append(t0, t1...).getRange(c.s.icmp)
 
 	// See if we can grow the number of inputs in "level" without
 	// changing the number of "level+1" files we pick up.
 	if len(t1) > 0 {
-		var exp0 tFiles
-		vt0.getOverlaps(amin.ukey(), amax.ukey(), &exp0, level != 0, s.icmp.ucmp)
+		exp0 := vt0.getOverlaps(nil, c.s.icmp, amin.ukey(), amax.ukey(), level == 0)
 		if len(exp0) > len(t0) && t1.size()+exp0.size() < kExpCompactionMaxBytes {
-			var exp1 tFiles
-			xmin, xmax := exp0.getRange(s.icmp)
-			vt1.getOverlaps(xmin.ukey(), xmax.ukey(), &exp1, true, s.icmp.ucmp)
+			xmin, xmax := exp0.getRange(c.s.icmp)
+			exp1 := vt1.getOverlaps(nil, c.s.icmp, xmin.ukey(), xmax.ukey(), false)
 			if len(exp1) == len(t1) {
-				s.logf("table@compaction expanding L%d+L%d (F·%d S·%s)+(F·%d S·%s) -> (F·%d S·%s)+(F·%d S·%s)",
+				c.s.logf("table@compaction expanding L%d+L%d (F·%d S·%s)+(F·%d S·%s) -> (F·%d S·%s)+(F·%d S·%s)",
 					level, level+1, len(t0), shortenb(int(t0.size())), len(t1), shortenb(int(t1.size())),
 					len(exp0), shortenb(int(exp0.size())), len(exp1), shortenb(int(exp1.size())))
-				min, max = xmin, xmax
+				imin, imax = xmin, xmax
 				t0, t1 = exp0, exp1
-				amin, amax = append(t0, t1...).getRange(s.icmp)
+				amin, amax = append(t0, t1...).getRange(c.s.icmp)
 			}
 		}
 	}
@@ -316,11 +310,11 @@ func (c *compaction) expand() {
 	// Compute the set of grandparent files that overlap this compaction
 	// (parent == level+1; grandparent == level+2)
 	if level+2 < kNumLevels {
-		v.tables[level+2].getOverlaps(amin.ukey(), amax.ukey(), &c.gp, true, s.icmp.ucmp)
+		c.gp = c.v.tables[level+2].getOverlaps(c.gp, c.s.icmp, amin.ukey(), amax.ukey(), false)
 	}
 
 	c.tables[0], c.tables[1] = t0, t1
-	c.min, c.max = min, max
+	c.imin, c.imax = imin, imax
 }
 
 // Check whether compaction is trivial.
@@ -328,17 +322,14 @@ func (c *compaction) trivial() bool {
 	return len(c.tables[0]) == 1 && len(c.tables[1]) == 0 && c.gp.size() <= kMaxGrandParentOverlapBytes
 }
 
-func (c *compaction) isBaseLevelForKey(key []byte) bool {
-	s := c.s
-	v := c.version
-
-	for level, tt := range v.tables[c.level+2:] {
-		for c.tPtrs[level] < len(tt) {
-			t := tt[c.tPtrs[level]]
-			if s.icmp.uCompare(key, t.max.ukey()) <= 0 {
-				// We've advanced far enough
-				if s.icmp.uCompare(key, t.min.ukey()) >= 0 {
-					// Key falls in this file's range, so definitely not base level
+func (c *compaction) baseLevelForKey(ukey []byte) bool {
+	for level, tables := range c.v.tables[c.level+2:] {
+		for c.tPtrs[level] < len(tables) {
+			t := tables[c.tPtrs[level]]
+			if c.s.icmp.uCompare(ukey, t.imax.ukey()) <= 0 {
+				// We've advanced far enough.
+				if c.s.icmp.uCompare(ukey, t.imin.ukey()) >= 0 {
+					// Key falls in this file's range, so definitely not base level.
 					return false
 				}
 				break
@@ -349,10 +340,10 @@ func (c *compaction) isBaseLevelForKey(key []byte) bool {
 	return true
 }
 
-func (c *compaction) shouldStopBefore(key iKey) bool {
+func (c *compaction) shouldStopBefore(ikey iKey) bool {
 	for ; c.gpidx < len(c.gp); c.gpidx++ {
 		gp := c.gp[c.gpidx]
-		if c.s.icmp.Compare(key, gp.max) <= 0 {
+		if c.s.icmp.Compare(ikey, gp.imax) <= 0 {
 			break
 		}
 		if c.seenKey {
@@ -362,42 +353,44 @@ func (c *compaction) shouldStopBefore(key iKey) bool {
 	c.seenKey = true
 
 	if c.overlappedBytes > kMaxGrandParentOverlapBytes {
-		// Too much overlap for current output; start new output
+		// Too much overlap for current output; start new output.
 		c.overlappedBytes = 0
 		return true
 	}
 	return false
 }
 
+// Creates an iterator.
 func (c *compaction) newIterator() iterator.Iterator {
-	s := c.s
-
-	level := c.level
-	icap := 2
+	// Creates iterator slice.
+	icap := len(c.tables)
 	if c.level == 0 {
+		// Special case for level-0
 		icap = len(c.tables[0]) + 1
 	}
 	its := make([]iterator.Iterator, 0, icap)
 
+	// Options.
 	ro := &opt.ReadOptions{
 		DontFillCache: true,
 	}
-	strict := s.o.GetStrict(opt.StrictIterator)
+	strict := c.s.o.GetStrict(opt.StrictIterator)
 
-	for i, tt := range c.tables {
-		if len(tt) == 0 {
+	for i, tables := range c.tables {
+		if len(tables) == 0 {
 			continue
 		}
 
-		if level+i == 0 {
-			for _, t := range tt {
-				its = append(its, s.tops.newIterator(t, nil, ro))
+		// Level-0 is not sorted and may overlaps each other.
+		if c.level+i == 0 {
+			for _, t := range tables {
+				its = append(its, c.s.tops.newIterator(t, nil, ro))
 			}
 		} else {
-			it := iterator.NewIndexedIterator(tt.newIndexIterator(s.tops, s.icmp, nil, ro), strict, true)
+			it := iterator.NewIndexedIterator(tables.newIndexIterator(c.s.tops, c.s.icmp, nil, ro), strict, true)
 			its = append(its, it)
 		}
 	}
 
-	return iterator.NewMergedIterator(its, s.icmp, true)
+	return iterator.NewMergedIterator(its, c.s.icmp, true)
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record.go

@@ -35,19 +35,19 @@ const (
 
 type cpRecord struct {
 	level int
-	key   iKey
+	ikey  iKey
 }
 
 type ntRecord struct {
 	level int
 	num   uint64
 	size  uint64
-	min   iKey
-	max   iKey
+	imin  iKey
+	imax  iKey
 }
 
 func (r ntRecord) makeFile(s *session) *tFile {
-	return newTFile(s.getTableFile(r.num), r.size, r.min, r.max)
+	return newTableFile(s.getTableFile(r.num), r.size, r.imin, r.imax)
 }
 
 type dtRecord struct {
@@ -98,9 +98,9 @@ func (p *sessionRecord) setSeq(seq uint64) {
 	p.seq = seq
 }
 
-func (p *sessionRecord) addCompactionPointer(level int, key iKey) {
+func (p *sessionRecord) addCompactionPointer(level int, ikey iKey) {
 	p.hasRec |= 1 << recCompactionPointer
-	p.compactionPointers = append(p.compactionPointers, cpRecord{level, key})
+	p.compactionPointers = append(p.compactionPointers, cpRecord{level, ikey})
 }
 
 func (p *sessionRecord) resetCompactionPointers() {
@@ -108,13 +108,13 @@ func (p *sessionRecord) resetCompactionPointers() {
 	p.compactionPointers = p.compactionPointers[:0]
 }
 
-func (p *sessionRecord) addTable(level int, num, size uint64, min, max iKey) {
+func (p *sessionRecord) addTable(level int, num, size uint64, imin, imax iKey) {
 	p.hasRec |= 1 << recNewTable
-	p.addedTables = append(p.addedTables, ntRecord{level, num, size, min, max})
+	p.addedTables = append(p.addedTables, ntRecord{level, num, size, imin, imax})
 }
 
 func (p *sessionRecord) addTableFile(level int, t *tFile) {
-	p.addTable(level, t.file.Num(), t.size, t.min, t.max)
+	p.addTable(level, t.file.Num(), t.size, t.imin, t.imax)
 }
 
 func (p *sessionRecord) resetAddedTables() {
@@ -169,23 +169,23 @@ func (p *sessionRecord) encode(w io.Writer) error {
 		p.putUvarint(w, recSeq)
 		p.putUvarint(w, p.seq)
 	}
-	for _, cp := range p.compactionPointers {
+	for _, r := range p.compactionPointers {
 		p.putUvarint(w, recCompactionPointer)
-		p.putUvarint(w, uint64(cp.level))
-		p.putBytes(w, cp.key)
+		p.putUvarint(w, uint64(r.level))
+		p.putBytes(w, r.ikey)
 	}
-	for _, t := range p.deletedTables {
+	for _, r := range p.deletedTables {
 		p.putUvarint(w, recDeletedTable)
-		p.putUvarint(w, uint64(t.level))
-		p.putUvarint(w, t.num)
+		p.putUvarint(w, uint64(r.level))
+		p.putUvarint(w, r.num)
 	}
-	for _, t := range p.addedTables {
+	for _, r := range p.addedTables {
 		p.putUvarint(w, recNewTable)
-		p.putUvarint(w, uint64(t.level))
-		p.putUvarint(w, t.num)
-		p.putUvarint(w, t.size)
-		p.putBytes(w, t.min)
-		p.putBytes(w, t.max)
+		p.putUvarint(w, uint64(r.level))
+		p.putUvarint(w, r.num)
+		p.putUvarint(w, r.size)
+		p.putBytes(w, r.imin)
+		p.putBytes(w, r.imax)
 	}
 	return p.err
 }
@@ -282,18 +282,18 @@ func (p *sessionRecord) decode(r io.Reader) error {
 			}
 		case recCompactionPointer:
 			level := p.readLevel(br)
-			key := p.readBytes(br)
+			ikey := p.readBytes(br)
 			if p.err == nil {
-				p.addCompactionPointer(level, iKey(key))
+				p.addCompactionPointer(level, iKey(ikey))
 			}
 		case recNewTable:
 			level := p.readLevel(br)
 			num := p.readUvarint(br)
 			size := p.readUvarint(br)
-			min := p.readBytes(br)
-			max := p.readBytes(br)
+			imin := p.readBytes(br)
+			imax := p.readBytes(br)
 			if p.err == nil {
-				p.addTable(level, num, size, min, max)
+				p.addTable(level, num, size, imin, imax)
 			}
 		case recDeletedTable:
 			level := p.readLevel(br)

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go

@@ -14,7 +14,7 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/storage"
 )
 
-// logging
+// Logging.
 
 type dropper struct {
 	s    *session
@@ -22,22 +22,17 @@ type dropper struct {
 }
 
 func (d dropper) Drop(err error) {
-	if e, ok := err.(journal.DroppedError); ok {
+	if e, ok := err.(journal.ErrCorrupted); ok {
 		d.s.logf("journal@drop %s-%d S·%s %q", d.file.Type(), d.file.Num(), shortenb(e.Size), e.Reason)
 	} else {
 		d.s.logf("journal@drop %s-%d %q", d.file.Type(), d.file.Num(), err)
 	}
 }
 
-func (s *session) log(v ...interface{}) {
-	s.stor.Log(fmt.Sprint(v...))
-}
+func (s *session) log(v ...interface{})                 { s.stor.Log(fmt.Sprint(v...)) }
+func (s *session) logf(format string, v ...interface{}) { s.stor.Log(fmt.Sprintf(format, v...)) }
 
-func (s *session) logf(format string, v ...interface{}) {
-	s.stor.Log(fmt.Sprintf(format, v...))
-}
-
-// file utils
+// File utils.
 
 func (s *session) getJournalFile(num uint64) storage.File {
 	return s.stor.GetFile(num, storage.TypeJournal)
@@ -56,7 +51,7 @@ func (s *session) newTemp() storage.File {
 	return s.stor.GetFile(num, storage.TypeTemp)
 }
 
-// session state
+// Session state.
 
 // Get current version.
 func (s *session) version() *version {
@@ -126,7 +121,7 @@ func (s *session) reuseFileNum(num uint64) {
 	}
 }
 
-// manifest related utils
+// Manifest related utils.
 
 // Fill given session record obj with current states; need external
 // synchronization.
@@ -142,7 +137,7 @@ func (s *session) fillRecord(r *sessionRecord, snapshot bool) {
 			r.setSeq(s.stSeq)
 		}
 
-		for level, ik := range s.stCPtrs {
+		for level, ik := range s.stCptrs {
 			if ik != nil {
 				r.addCompactionPointer(level, ik)
 			}
@@ -152,7 +147,7 @@ func (s *session) fillRecord(r *sessionRecord, snapshot bool) {
 	}
 }
 
-// Mark if record has been commited, this will update session state;
+// Mark if record has been committed, this will update session state;
 // need external synchronization.
 func (s *session) recordCommited(r *sessionRecord) {
 	if r.has(recJournalNum) {
@@ -168,7 +163,7 @@ func (s *session) recordCommited(r *sessionRecord) {
 	}
 
 	for _, p := range r.compactionPointers {
-		s.stCPtrs[p.level] = iKey(p.key)
+		s.stCptrs[p.level] = iKey(p.ikey)
 	}
 }
 

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go

@@ -344,19 +344,17 @@ type fileWrap struct {
 }
 
 func (fw fileWrap) Sync() error {
+	if err := fw.File.Sync(); err != nil {
+		return err
+	}
 	if fw.f.Type() == TypeManifest {
 		// Also sync parent directory if file type is manifest.
 		// See: https://code.google.com/p/leveldb/issues/detail?id=190.
-		f, err := os.Open(fw.f.fs.path)
-		if err != nil {
-			return err
-		}
-		defer f.Close()
-		if err := f.Sync(); err != nil {
+		if err := syncDir(fw.f.fs.path); err != nil {
 			return err
 		}
 	}
-	return fw.File.Sync()
+	return nil
 }
 
 func (fw fileWrap) Close() error {

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_plan9.go

@@ -38,3 +38,15 @@ func rename(oldpath, newpath string) error {
 	_, fname := filepath.Split(newpath)
 	return os.Rename(oldpath, fname)
 }
+
+func syncDir(name string) error {
+	f, err := os.Open(name)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if err := f.Sync(); err != nil {
+		return err
+	}
+	return nil
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_solaris.go

@@ -0,0 +1,68 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// +build solaris
+
+package storage
+
+import (
+	"os"
+	"syscall"
+)
+
+type unixFileLock struct {
+	f *os.File
+}
+
+func (fl *unixFileLock) release() error {
+	if err := setFileLock(fl.f, false); err != nil {
+		return err
+	}
+	return fl.f.Close()
+}
+
+func newFileLock(path string) (fl fileLock, err error) {
+	f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
+	if err != nil {
+		return
+	}
+	err = setFileLock(f, true)
+	if err != nil {
+		f.Close()
+		return
+	}
+	fl = &unixFileLock{f: f}
+	return
+}
+
+func setFileLock(f *os.File, lock bool) error {
+	flock := syscall.Flock_t{
+		Type:   syscall.F_UNLCK,
+		Start:  0,
+		Len:    0,
+		Whence: 1,
+	}
+	if lock {
+		flock.Type = syscall.F_WRLCK
+	}
+	return syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &flock)
+}
+
+func rename(oldpath, newpath string) error {
+	return os.Rename(oldpath, newpath)
+}
+
+func syncDir(name string) error {
+	f, err := os.Open(name)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if err := f.Sync(); err != nil {
+		return err
+	}
+	return nil
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_unix.go

@@ -4,7 +4,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-// +build darwin freebsd linux netbsd openbsd
+// +build darwin dragonfly freebsd linux netbsd openbsd
 
 package storage
 
@@ -49,3 +49,15 @@ func setFileLock(f *os.File, lock bool) error {
 func rename(oldpath, newpath string) error {
 	return os.Rename(oldpath, newpath)
 }
+
+func syncDir(name string) error {
+	f, err := os.Open(name)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if err := f.Sync(); err != nil {
+		return err
+	}
+	return nil
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_windows.go

@@ -65,3 +65,5 @@ func rename(oldpath, newpath string) error {
 	}
 	return moveFileEx(from, to, _MOVEFILE_REPLACE_EXISTING)
 }
+
+func syncDir(name string) error { return nil }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/storage.go

@@ -67,7 +67,7 @@ type Writer interface {
 	Syncer
 }
 
-// File is the file.
+// File is the file. A file instance must be goroutine-safe.
 type File interface {
 	// Open opens the file for read. Returns os.ErrNotExist error
 	// if the file does not exist.
@@ -94,7 +94,7 @@ type File interface {
 	Remove() error
 }
 
-// Storage is the storage.
+// Storage is the storage. A storage instance must be goroutine-safe.
 type Storage interface {
 	// Lock locks the storage. Any subsequent attempt to call Lock will fail
 	// until the last lock released.

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go

@@ -7,11 +7,12 @@
 package leveldb
 
 import (
+	"fmt"
 	"sort"
+	"sync"
 	"sync/atomic"
 
 	"github.com/syndtr/goleveldb/leveldb/cache"
-	"github.com/syndtr/goleveldb/leveldb/comparer"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	"github.com/syndtr/goleveldb/leveldb/storage"
@@ -19,34 +20,41 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
-// table file
+// tFile holds basic information about a table.
 type tFile struct {
-	file     storage.File
-	seekLeft int32
-	size     uint64
-	min, max iKey
+	file       storage.File
+	seekLeft   int32
+	size       uint64
+	imin, imax iKey
 }
 
-// test if key is after t
-func (t *tFile) isAfter(key []byte, ucmp comparer.BasicComparer) bool {
-	return key != nil && ucmp.Compare(key, t.max.ukey()) > 0
+// Returns true if given key is after largest key of this table.
+func (t *tFile) after(icmp *iComparer, ukey []byte) bool {
+	return ukey != nil && icmp.uCompare(ukey, t.imax.ukey()) > 0
 }
 
-// test if key is before t
-func (t *tFile) isBefore(key []byte, ucmp comparer.BasicComparer) bool {
-	return key != nil && ucmp.Compare(key, t.min.ukey()) < 0
+// Returns true if given key is before smallest key of this table.
+func (t *tFile) before(icmp *iComparer, ukey []byte) bool {
+	return ukey != nil && icmp.uCompare(ukey, t.imin.ukey()) < 0
 }
 
-func (t *tFile) incrSeek() int32 {
+// Returns true if given key range overlaps with this table key range.
+func (t *tFile) overlaps(icmp *iComparer, umin, umax []byte) bool {
+	return !t.after(icmp, umin) && !t.before(icmp, umax)
+}
+
+// Cosumes one seek and return current seeks left.
+func (t *tFile) consumeSeek() int32 {
 	return atomic.AddInt32(&t.seekLeft, -1)
 }
 
-func newTFile(file storage.File, size uint64, min, max iKey) *tFile {
+// Creates new tFile.
+func newTableFile(file storage.File, size uint64, imin, imax iKey) *tFile {
 	f := &tFile{
 		file: file,
 		size: size,
-		min:  min,
-		max:  max,
+		imin: imin,
+		imax: imax,
 	}
 
 	// We arrange to automatically compact this file after
@@ -70,33 +78,40 @@ func newTFile(file storage.File, size uint64, min, max iKey) *tFile {
 	return f
 }
 
-// table files
+// tFiles hold multiple tFile.
 type tFiles []*tFile
 
 func (tf tFiles) Len() int      { return len(tf) }
 func (tf tFiles) Swap(i, j int) { tf[i], tf[j] = tf[j], tf[i] }
 
+// Returns true if i smallest key is less than j.
+// This used for sort by key in ascending order.
 func (tf tFiles) lessByKey(icmp *iComparer, i, j int) bool {
 	a, b := tf[i], tf[j]
-	n := icmp.Compare(a.min, b.min)
+	n := icmp.Compare(a.imin, b.imin)
 	if n == 0 {
 		return a.file.Num() < b.file.Num()
 	}
 	return n < 0
 }
 
+// Returns true if i file number is greater than j.
+// This used for sort by file number in descending order.
 func (tf tFiles) lessByNum(i, j int) bool {
 	return tf[i].file.Num() > tf[j].file.Num()
 }
 
+// Sorts tables by key in ascending order.
 func (tf tFiles) sortByKey(icmp *iComparer) {
 	sort.Sort(&tFilesSortByKey{tFiles: tf, icmp: icmp})
 }
 
+// Sorts tables by file number in descending order.
 func (tf tFiles) sortByNum() {
 	sort.Sort(&tFilesSortByNum{tFiles: tf})
 }
 
+// Returns sum of all tables size.
 func (tf tFiles) size() (sum uint64) {
 	for _, t := range tf {
 		sum += t.size
@@ -104,94 +119,106 @@ func (tf tFiles) size() (sum uint64) {
 	return sum
 }
 
-func (tf tFiles) searchMin(key iKey, icmp *iComparer) int {
+// Searches smallest index of tables whose its smallest
+// key is after or equal with given key.
+func (tf tFiles) searchMin(icmp *iComparer, ikey iKey) int {
 	return sort.Search(len(tf), func(i int) bool {
-		return icmp.Compare(tf[i].min, key) >= 0
+		return icmp.Compare(tf[i].imin, ikey) >= 0
 	})
 }
 
-func (tf tFiles) searchMax(key iKey, icmp *iComparer) int {
+// Searches smallest index of tables whose its largest
+// key is after or equal with given key.
+func (tf tFiles) searchMax(icmp *iComparer, ikey iKey) int {
 	return sort.Search(len(tf), func(i int) bool {
-		return icmp.Compare(tf[i].max, key) >= 0
+		return icmp.Compare(tf[i].imax, ikey) >= 0
 	})
 }
 
-func (tf tFiles) isOverlaps(min, max []byte, disjSorted bool, icmp *iComparer) bool {
-	if !disjSorted {
-		// Need to check against all files
+// Returns true if given key range overlaps with one or more
+// tables key range. If unsorted is true then binary search will not be used.
+func (tf tFiles) overlaps(icmp *iComparer, umin, umax []byte, unsorted bool) bool {
+	if unsorted {
+		// Check against all files.
 		for _, t := range tf {
-			if !t.isAfter(min, icmp.ucmp) && !t.isBefore(max, icmp.ucmp) {
+			if t.overlaps(icmp, umin, umax) {
 				return true
 			}
 		}
 		return false
 	}
 
-	var idx int
-	if len(min) > 0 {
-		// Find the earliest possible internal key for min
-		idx = tf.searchMax(newIKey(min, kMaxSeq, tSeek), icmp)
+	i := 0
+	if len(umin) > 0 {
+		// Find the earliest possible internal key for min.
+		i = tf.searchMax(icmp, newIKey(umin, kMaxSeq, tSeek))
 	}
-
-	if idx >= len(tf) {
-		// beginning of range is after all files, so no overlap
+	if i >= len(tf) {
+		// Beginning of range is after all files, so no overlap.
 		return false
 	}
-	return !tf[idx].isBefore(max, icmp.ucmp)
+	return !tf[i].before(icmp, umax)
 }
 
-func (tf tFiles) getOverlaps(min, max []byte, r *tFiles, disjSorted bool, ucmp comparer.BasicComparer) {
+// Returns tables whose its key range overlaps with given key range.
+// If overlapped is true then the search will be expanded to tables that
+// overlaps with each other.
+func (tf tFiles) getOverlaps(dst tFiles, icmp *iComparer, umin, umax []byte, overlapped bool) tFiles {
+	x := len(dst)
 	for i := 0; i < len(tf); {
 		t := tf[i]
-		i++
-		if t.isAfter(min, ucmp) || t.isBefore(max, ucmp) {
-			continue
-		}
-
-		*r = append(*r, t)
-		if !disjSorted {
-			// Level-0 files may overlap each other.  So check if the newly
-			// added file has expanded the range.  If so, restart search.
-			if min != nil && ucmp.Compare(t.min.ukey(), min) < 0 {
-				min = t.min.ukey()
-				*r = nil
-				i = 0
-			} else if max != nil && ucmp.Compare(t.max.ukey(), max) > 0 {
-				max = t.max.ukey()
-				*r = nil
-				i = 0
+		if t.overlaps(icmp, umin, umax) {
+			if overlapped {
+				// For overlapped files, check if the newly added file has
+				// expanded the range. If so, restart search.
+				if umin != nil && icmp.uCompare(t.imin.ukey(), umin) < 0 {
+					umin = t.imin.ukey()
+					dst = dst[:x]
+					i = 0
+					continue
+				} else if umax != nil && icmp.uCompare(t.imax.ukey(), umax) > 0 {
+					umax = t.imax.ukey()
+					dst = dst[:x]
+					i = 0
+					continue
+				}
 			}
+
+			dst = append(dst, t)
 		}
+		i++
 	}
 
-	return
+	return dst
 }
 
-func (tf tFiles) getRange(icmp *iComparer) (min, max iKey) {
+// Returns tables key range.
+func (tf tFiles) getRange(icmp *iComparer) (imin, imax iKey) {
 	for i, t := range tf {
 		if i == 0 {
-			min, max = t.min, t.max
+			imin, imax = t.imin, t.imax
 			continue
 		}
-		if icmp.Compare(t.min, min) < 0 {
-			min = t.min
+		if icmp.Compare(t.imin, imin) < 0 {
+			imin = t.imin
 		}
-		if icmp.Compare(t.max, max) > 0 {
-			max = t.max
+		if icmp.Compare(t.imax, imax) > 0 {
+			imax = t.imax
 		}
 	}
 
 	return
 }
 
+// Creates iterator index from tables.
 func (tf tFiles) newIndexIterator(tops *tOps, icmp *iComparer, slice *util.Range, ro *opt.ReadOptions) iterator.IteratorIndexer {
 	if slice != nil {
 		var start, limit int
 		if slice.Start != nil {
-			start = tf.searchMax(iKey(slice.Start), icmp)
+			start = tf.searchMax(icmp, iKey(slice.Start))
 		}
 		if slice.Limit != nil {
-			limit = tf.searchMin(iKey(slice.Limit), icmp)
+			limit = tf.searchMin(icmp, iKey(slice.Limit))
 		} else {
 			limit = tf.Len()
 		}
@@ -206,6 +233,7 @@ func (tf tFiles) newIndexIterator(tops *tOps, icmp *iComparer, slice *util.Range
 	})
 }
 
+// Tables iterator index.
 type tFilesArrayIndexer struct {
 	tFiles
 	tops  *tOps
@@ -215,7 +243,7 @@ type tFilesArrayIndexer struct {
 }
 
 func (a *tFilesArrayIndexer) Search(key []byte) int {
-	return a.searchMax(iKey(key), a.icmp)
+	return a.searchMax(a.icmp, iKey(key))
 }
 
 func (a *tFilesArrayIndexer) Get(i int) iterator.Iterator {
@@ -225,6 +253,7 @@ func (a *tFilesArrayIndexer) Get(i int) iterator.Iterator {
 	return a.tops.newIterator(a.tFiles[i], nil, a.ro)
 }
 
+// Helper type for sortByKey.
 type tFilesSortByKey struct {
 	tFiles
 	icmp *iComparer
@@ -234,6 +263,7 @@ func (x *tFilesSortByKey) Less(i, j int) bool {
 	return x.lessByKey(x.icmp, i, j)
 }
 
+// Helper type for sortByNum.
 type tFilesSortByNum struct {
 	tFiles
 }
@@ -242,19 +272,17 @@ func (x *tFilesSortByNum) Less(i, j int) bool {
 	return x.lessByNum(i, j)
 }
 
-// table operations
+// Table operations.
 type tOps struct {
 	s       *session
 	cache   cache.Cache
 	cacheNS cache.Namespace
+	bpool   *util.BufferPool
+	mu      sync.Mutex
+	closed  bool
 }
 
-func newTableOps(s *session, cacheCap int) *tOps {
-	c := cache.NewLRUCache(cacheCap)
-	ns := c.GetNamespace(0)
-	return &tOps{s, c, ns}
-}
-
+// Creates an empty table and returns table writer.
 func (t *tOps) create() (*tWriter, error) {
 	file := t.s.getTableFile(t.s.allocFileNum())
 	fw, err := file.Create()
@@ -269,10 +297,11 @@ func (t *tOps) create() (*tWriter, error) {
 	}, nil
 }
 
+// Builds table from src iterator.
 func (t *tOps) createFrom(src iterator.Iterator) (f *tFile, n int, err error) {
 	w, err := t.create()
 	if err != nil {
-		return f, n, err
+		return
 	}
 
 	defer func() {
@@ -282,7 +311,7 @@ func (t *tOps) createFrom(src iterator.Iterator) (f *tFile, n int, err error) {
 	}()
 
 	for src.Next() {
-		err = w.add(src.Key(), src.Value())
+		err = w.append(src.Key(), src.Value())
 		if err != nil {
 			return
 		}
@@ -297,84 +326,142 @@ func (t *tOps) createFrom(src iterator.Iterator) (f *tFile, n int, err error) {
 	return
 }
 
-func (t *tOps) lookup(f *tFile) (c cache.Object, err error) {
+type trWrapper struct {
+	*table.Reader
+	t   *tOps
+	ref int
+}
+
+func (w *trWrapper) Release() {
+	if w.ref != 0 && !w.t.closed {
+		panic(fmt.Sprintf("BUG: invalid ref %d, refer to issue #72", w.ref))
+	}
+	w.Reader.Release()
+}
+
+type trCacheHandleWrapper struct {
+	cache.Handle
+	t        *tOps
+	released bool
+}
+
+func (w *trCacheHandleWrapper) Release() {
+	w.t.mu.Lock()
+	defer w.t.mu.Unlock()
+
+	if !w.released {
+		w.released = true
+		w.Value().(*trWrapper).ref--
+	}
+	w.Handle.Release()
+}
+
+// Opens table. It returns a cache handle, which should
+// be released after use.
+func (t *tOps) open(f *tFile) (ch cache.Handle, err error) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
 	num := f.file.Num()
-	c, ok := t.cacheNS.Get(num, func() (ok bool, value interface{}, charge int, fin cache.SetFin) {
+	ch = t.cacheNS.Get(num, func() (charge int, value interface{}) {
 		var r storage.Reader
 		r, err = f.file.Open()
 		if err != nil {
-			return
+			return 0, nil
 		}
 
-		o := t.s.o
-
-		var cacheNS cache.Namespace
-		if bc := o.GetBlockCache(); bc != nil {
-			cacheNS = bc.GetNamespace(num)
+		var bcacheNS cache.Namespace
+		if bc := t.s.o.GetBlockCache(); bc != nil {
+			bcacheNS = bc.GetNamespace(num)
 		}
-
-		ok = true
-		value = table.NewReader(r, int64(f.size), cacheNS, o)
-		charge = 1
-		fin = func() {
-			r.Close()
-		}
-		return
+		return 1, &trWrapper{table.NewReader(r, int64(f.size), bcacheNS, t.bpool, t.s.o), t, 0}
 	})
-	if !ok && err == nil {
+	if ch == nil && err == nil {
 		err = ErrClosed
 	}
+	ch.Value().(*trWrapper).ref++
+	ch = &trCacheHandleWrapper{ch, t, false}
 	return
 }
 
-func (t *tOps) get(f *tFile, key []byte, ro *opt.ReadOptions) (rkey, rvalue []byte, err error) {
-	c, err := t.lookup(f)
+// Finds key/value pair whose key is greater than or equal to the
+// given key.
+func (t *tOps) find(f *tFile, key []byte, ro *opt.ReadOptions) (rkey, rvalue []byte, err error) {
+	ch, err := t.open(f)
 	if err != nil {
 		return nil, nil, err
 	}
-	defer c.Release()
-	return c.Value().(*table.Reader).Find(key, ro)
+	defer ch.Release()
+	return ch.Value().(*trWrapper).Find(key, ro)
 }
 
+// Returns approximate offset of the given key.
 func (t *tOps) offsetOf(f *tFile, key []byte) (offset uint64, err error) {
-	c, err := t.lookup(f)
+	ch, err := t.open(f)
 	if err != nil {
 		return
 	}
-	_offset, err := c.Value().(*table.Reader).OffsetOf(key)
-	offset = uint64(_offset)
-	c.Release()
-	return
+	defer ch.Release()
+	offset_, err := ch.Value().(*trWrapper).OffsetOf(key)
+	return uint64(offset_), err
 }
 
+// Creates an iterator from the given table.
 func (t *tOps) newIterator(f *tFile, slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
-	c, err := t.lookup(f)
+	ch, err := t.open(f)
 	if err != nil {
 		return iterator.NewEmptyIterator(err)
 	}
-	iter := c.Value().(*table.Reader).NewIterator(slice, ro)
-	iter.SetReleaser(c)
+	iter := ch.Value().(*trWrapper).NewIterator(slice, ro)
+	iter.SetReleaser(ch)
 	return iter
 }
 
+// Removes table from persistent storage. It waits until
+// no one use the the table.
 func (t *tOps) remove(f *tFile) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
 	num := f.file.Num()
-	t.cacheNS.Delete(num, func(exist bool) {
-		if err := f.file.Remove(); err != nil {
-			t.s.logf("table@remove removing @%d %q", num, err)
-		} else {
-			t.s.logf("table@remove removed @%d", num)
-		}
-		if bc := t.s.o.GetBlockCache(); bc != nil {
-			bc.GetNamespace(num).Zap(false)
+	t.cacheNS.Delete(num, func(exist, pending bool) {
+		if !pending {
+			if err := f.file.Remove(); err != nil {
+				t.s.logf("table@remove removing @%d %q", num, err)
+			} else {
+				t.s.logf("table@remove removed @%d", num)
+			}
+			if bc := t.s.o.GetBlockCache(); bc != nil {
+				bc.ZapNamespace(num)
+			}
 		}
 	})
 }
 
+// Closes the table ops instance. It will close all tables,
+// regadless still used or not.
 func (t *tOps) close() {
-	t.cache.Zap(true)
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	t.closed = true
+	t.cache.Zap()
+	t.bpool.Close()
 }
 
+// Creates new initialized table ops instance.
+func newTableOps(s *session, cacheCap int) *tOps {
+	c := cache.NewLRUCache(cacheCap)
+	return &tOps{
+		s:       s,
+		cache:   c,
+		cacheNS: c.GetNamespace(0),
+		bpool:   util.NewBufferPool(s.o.GetBlockSize() + 5),
+	}
+}
+
+// tWriter wraps the table writer. It keep track of file descriptor
+// and added key range.
 type tWriter struct {
 	t *tOps
 
@@ -385,7 +472,8 @@ type tWriter struct {
 	first, last []byte
 }
 
-func (w *tWriter) add(key, value []byte) error {
+// Append key/value pair to the table.
+func (w *tWriter) append(key, value []byte) error {
 	if w.first == nil {
 		w.first = append([]byte{}, key...)
 	}
@@ -393,10 +481,12 @@ func (w *tWriter) add(key, value []byte) error {
 	return w.tw.Append(key, value)
 }
 
+// Returns true if the table is empty.
 func (w *tWriter) empty() bool {
 	return w.first == nil
 }
 
+// Finalizes the table and returns table file.
 func (w *tWriter) finish() (f *tFile, err error) {
 	err = w.tw.Close()
 	if err != nil {
@@ -408,10 +498,11 @@ func (w *tWriter) finish() (f *tFile, err error) {
 		return
 	}
 	w.w.Close()
-	f = newTFile(w.file, uint64(w.tw.BytesLen()), iKey(w.first), iKey(w.last))
+	f = newTableFile(w.file, uint64(w.tw.BytesLen()), iKey(w.first), iKey(w.last))
 	return
 }
 
+// Drops the table.
 func (w *tWriter) drop() {
 	w.w.Close()
 	w.file.Remove()

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/block_test.go

@@ -40,7 +40,7 @@ var _ = testutil.Defer(func() {
 			data := bw.buf.Bytes()
 			restartsLen := int(binary.LittleEndian.Uint32(data[len(data)-4:]))
 			return &block{
-				cmp:            comparer.DefaultComparer,
+				tr:             &Reader{cmp: comparer.DefaultComparer},
 				data:           data,
 				restartsLen:    restartsLen,
 				restartsOffset: len(data) - (restartsLen+1)*4,
@@ -59,7 +59,7 @@ var _ = testutil.Defer(func() {
 						// Make block.
 						br := Build(kv, restartInterval)
 						// Do testing.
-						testutil.KeyValueTesting(nil, br, kv.Clone())
+						testutil.KeyValueTesting(nil, kv.Clone(), br, nil, nil)
 					}
 
 					Describe(Text(), Test)

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/reader.go

@@ -13,6 +13,7 @@ import (
 	"io"
 	"sort"
 	"strings"
+	"sync"
 
 	"code.google.com/p/snappy-go/snappy"
 
@@ -25,8 +26,9 @@ import (
 )
 
 var (
-	ErrNotFound     = util.ErrNotFound
-	ErrIterReleased = errors.New("leveldb/table: iterator released")
+	ErrNotFound       = util.ErrNotFound
+	ErrReaderReleased = errors.New("leveldb/table: reader released")
+	ErrIterReleased   = errors.New("leveldb/table: iterator released")
 )
 
 func max(x, y int) int {
@@ -37,7 +39,7 @@ func max(x, y int) int {
 }
 
 type block struct {
-	cmp            comparer.BasicComparer
+	tr             *Reader
 	data           []byte
 	restartsLen    int
 	restartsOffset int
@@ -46,31 +48,25 @@ type block struct {
 }
 
 func (b *block) seek(rstart, rlimit int, key []byte) (index, offset int, err error) {
-	n := b.restartsOffset
-	data := b.data
-	cmp := b.cmp
-
 	index = sort.Search(b.restartsLen-rstart-(b.restartsLen-rlimit), func(i int) bool {
-		offset := int(binary.LittleEndian.Uint32(data[n+4*(rstart+i):]))
-		offset += 1                               // shared always zero, since this is a restart point
-		v1, n1 := binary.Uvarint(data[offset:])   // key length
-		_, n2 := binary.Uvarint(data[offset+n1:]) // value length
+		offset := int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*(rstart+i):]))
+		offset += 1                                 // shared always zero, since this is a restart point
+		v1, n1 := binary.Uvarint(b.data[offset:])   // key length
+		_, n2 := binary.Uvarint(b.data[offset+n1:]) // value length
 		m := offset + n1 + n2
-		return cmp.Compare(data[m:m+int(v1)], key) > 0
+		return b.tr.cmp.Compare(b.data[m:m+int(v1)], key) > 0
 	}) + rstart - 1
 	if index < rstart {
 		// The smallest key is greater-than key sought.
 		index = rstart
 	}
-	offset = int(binary.LittleEndian.Uint32(data[n+4*index:]))
+	offset = int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*index:]))
 	return
 }
 
 func (b *block) restartIndex(rstart, rlimit, offset int) int {
-	n := b.restartsOffset
-	data := b.data
 	return sort.Search(b.restartsLen-rstart-(b.restartsLen-rlimit), func(i int) bool {
-		return int(binary.LittleEndian.Uint32(data[n+4*(rstart+i):])) > offset
+		return int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*(rstart+i):])) > offset
 	}) + rstart - 1
 }
 
@@ -139,6 +135,12 @@ func (b *block) newIterator(slice *util.Range, inclLimit bool, cache util.Releas
 	return bi
 }
 
+func (b *block) Release() {
+	b.tr.bpool.Put(b.data)
+	b.tr = nil
+	b.data = nil
+}
+
 type dir int
 
 const (
@@ -261,7 +263,7 @@ func (i *blockIter) Seek(key []byte) bool {
 		i.dir = dirForward
 	}
 	for i.Next() {
-		if i.block.cmp.Compare(i.key, key) >= 0 {
+		if i.block.tr.cmp.Compare(i.key, key) >= 0 {
 			return true
 		}
 	}
@@ -437,25 +439,32 @@ func (i *blockIter) Value() []byte {
 }
 
 func (i *blockIter) Release() {
-	i.prevNode = nil
-	i.prevKeys = nil
-	i.key = nil
-	i.value = nil
-	i.dir = dirReleased
-	if i.cache != nil {
-		i.cache.Release()
-		i.cache = nil
-	}
-	if i.releaser != nil {
-		i.releaser.Release()
-		i.releaser = nil
+	if i.dir != dirReleased {
+		i.block = nil
+		i.prevNode = nil
+		i.prevKeys = nil
+		i.key = nil
+		i.value = nil
+		i.dir = dirReleased
+		if i.cache != nil {
+			i.cache.Release()
+			i.cache = nil
+		}
+		if i.releaser != nil {
+			i.releaser.Release()
+			i.releaser = nil
+		}
 	}
 }
 
 func (i *blockIter) SetReleaser(releaser util.Releaser) {
-	if i.dir > dirReleased {
-		i.releaser = releaser
+	if i.dir == dirReleased {
+		panic(util.ErrReleased)
 	}
+	if i.releaser != nil && releaser != nil {
+		panic(util.ErrHasReleaser)
+	}
+	i.releaser = releaser
 }
 
 func (i *blockIter) Valid() bool {
@@ -467,7 +476,7 @@ func (i *blockIter) Error() error {
 }
 
 type filterBlock struct {
-	filter     filter.Filter
+	tr         *Reader
 	data       []byte
 	oOffset    int
 	baseLg     uint
@@ -481,7 +490,7 @@ func (b *filterBlock) contains(offset uint64, key []byte) bool {
 		n := int(binary.LittleEndian.Uint32(o))
 		m := int(binary.LittleEndian.Uint32(o[4:]))
 		if n < m && m <= b.oOffset {
-			return b.filter.Contains(b.data[n:m], key)
+			return b.tr.filter.Contains(b.data[n:m], key)
 		} else if n == m {
 			return false
 		}
@@ -489,10 +498,15 @@ func (b *filterBlock) contains(offset uint64, key []byte) bool {
 	return true
 }
 
+func (b *filterBlock) Release() {
+	b.tr.bpool.Put(b.data)
+	b.tr = nil
+	b.data = nil
+}
+
 type indexIter struct {
-	blockIter
-	tableReader *Reader
-	slice       *util.Range
+	*blockIter
+	slice *util.Range
 	// Options
 	checksum  bool
 	fillCache bool
@@ -507,27 +521,31 @@ func (i *indexIter) Get() iterator.Iterator {
 	if n == 0 {
 		return iterator.NewEmptyIterator(errors.New("leveldb/table: Reader: invalid table (bad data block handle)"))
 	}
+
 	var slice *util.Range
 	if i.slice != nil && (i.blockIter.isFirst() || i.blockIter.isLast()) {
 		slice = i.slice
 	}
-	return i.tableReader.getDataIter(dataBH, slice, i.checksum, i.fillCache)
+	return i.blockIter.block.tr.getDataIterErr(dataBH, slice, i.checksum, i.fillCache)
 }
 
 // Reader is a table reader.
 type Reader struct {
+	mu     sync.RWMutex
 	reader io.ReaderAt
 	cache  cache.Namespace
 	err    error
+	bpool  *util.BufferPool
 	// Options
 	cmp        comparer.Comparer
 	filter     filter.Filter
 	checksum   bool
 	strictIter bool
 
-	dataEnd     int64
-	indexBlock  *block
-	filterBlock *filterBlock
+	dataEnd           int64
+	indexBH, filterBH blockHandle
+	indexBlock        *block
+	filterBlock       *filterBlock
 }
 
 func verifyChecksum(data []byte) bool {
@@ -538,12 +556,13 @@ func verifyChecksum(data []byte) bool {
 }
 
 func (r *Reader) readRawBlock(bh blockHandle, checksum bool) ([]byte, error) {
-	data := make([]byte, bh.length+blockTrailerLen)
+	data := r.bpool.Get(int(bh.length + blockTrailerLen))
 	if _, err := r.reader.ReadAt(data, int64(bh.offset)); err != nil && err != io.EOF {
 		return nil, err
 	}
 	if checksum || r.checksum {
 		if !verifyChecksum(data) {
+			r.bpool.Put(data)
 			return nil, errors.New("leveldb/table: Reader: invalid block (checksum mismatch)")
 		}
 	}
@@ -551,12 +570,18 @@ func (r *Reader) readRawBlock(bh blockHandle, checksum bool) ([]byte, error) {
 	case blockTypeNoCompression:
 		data = data[:bh.length]
 	case blockTypeSnappyCompression:
-		var err error
-		data, err = snappy.Decode(nil, data[:bh.length])
+		decLen, err := snappy.DecodedLen(data[:bh.length])
+		if err != nil {
+			return nil, err
+		}
+		tmp := data
+		data, err = snappy.Decode(r.bpool.Get(decLen), tmp[:bh.length])
+		r.bpool.Put(tmp)
 		if err != nil {
 			return nil, err
 		}
 	default:
+		r.bpool.Put(data)
 		return nil, fmt.Errorf("leveldb/table: Reader: unknown block compression type: %d", data[bh.length])
 	}
 	return data, nil
@@ -569,7 +594,7 @@ func (r *Reader) readBlock(bh blockHandle, checksum bool) (*block, error) {
 	}
 	restartsLen := int(binary.LittleEndian.Uint32(data[len(data)-4:]))
 	b := &block{
-		cmp:            r.cmp,
+		tr:             r,
 		data:           data,
 		restartsLen:    restartsLen,
 		restartsOffset: len(data) - (restartsLen+1)*4,
@@ -578,7 +603,44 @@ func (r *Reader) readBlock(bh blockHandle, checksum bool) (*block, error) {
 	return b, nil
 }
 
-func (r *Reader) readFilterBlock(bh blockHandle, filter filter.Filter) (*filterBlock, error) {
+func (r *Reader) readBlockCached(bh blockHandle, checksum, fillCache bool) (*block, util.Releaser, error) {
+	if r.cache != nil {
+		var err error
+		ch := r.cache.Get(bh.offset, func() (charge int, value interface{}) {
+			if !fillCache {
+				return 0, nil
+			}
+			var b *block
+			b, err = r.readBlock(bh, checksum)
+			if err != nil {
+				return 0, nil
+			}
+			return cap(b.data), b
+		})
+		if ch != nil {
+			b, ok := ch.Value().(*block)
+			if !ok {
+				ch.Release()
+				return nil, nil, errors.New("leveldb/table: Reader: inconsistent block type")
+			}
+			if !b.checksum && (r.checksum || checksum) {
+				if !verifyChecksum(b.data) {
+					ch.Release()
+					return nil, nil, errors.New("leveldb/table: Reader: invalid block (checksum mismatch)")
+				}
+				b.checksum = true
+			}
+			return b, ch, err
+		} else if err != nil {
+			return nil, nil, err
+		}
+	}
+
+	b, err := r.readBlock(bh, checksum)
+	return b, b, err
+}
+
+func (r *Reader) readFilterBlock(bh blockHandle) (*filterBlock, error) {
 	data, err := r.readRawBlock(bh, true)
 	if err != nil {
 		return nil, err
@@ -593,7 +655,7 @@ func (r *Reader) readFilterBlock(bh blockHandle, filter filter.Filter) (*filterB
 		return nil, errors.New("leveldb/table: Reader: invalid filter block (invalid offset)")
 	}
 	b := &filterBlock{
-		filter:     filter,
+		tr:         r,
 		data:       data,
 		oOffset:    oOffset,
 		baseLg:     uint(data[n-1]),
@@ -602,44 +664,67 @@ func (r *Reader) readFilterBlock(bh blockHandle, filter filter.Filter) (*filterB
 	return b, nil
 }
 
-func (r *Reader) getDataIter(dataBH blockHandle, slice *util.Range, checksum, fillCache bool) iterator.Iterator {
+func (r *Reader) readFilterBlockCached(bh blockHandle, fillCache bool) (*filterBlock, util.Releaser, error) {
 	if r.cache != nil {
-		// Get/set block cache.
 		var err error
-		cache, ok := r.cache.Get(dataBH.offset, func() (ok bool, value interface{}, charge int, fin cache.SetFin) {
+		ch := r.cache.Get(bh.offset, func() (charge int, value interface{}) {
 			if !fillCache {
-				return
+				return 0, nil
 			}
-			var dataBlock *block
-			dataBlock, err = r.readBlock(dataBH, checksum)
-			if err == nil {
-				ok = true
-				value = dataBlock
-				charge = int(dataBH.length)
+			var b *filterBlock
+			b, err = r.readFilterBlock(bh)
+			if err != nil {
+				return 0, nil
 			}
-			return
+			return cap(b.data), b
 		})
-		if err != nil {
-			return iterator.NewEmptyIterator(err)
-		}
-		if ok {
-			dataBlock := cache.Value().(*block)
-			if !dataBlock.checksum && (r.checksum || checksum) {
-				if !verifyChecksum(dataBlock.data) {
-					return iterator.NewEmptyIterator(errors.New("leveldb/table: Reader: invalid block (checksum mismatch)"))
-				}
-				dataBlock.checksum = true
+		if ch != nil {
+			b, ok := ch.Value().(*filterBlock)
+			if !ok {
+				ch.Release()
+				return nil, nil, errors.New("leveldb/table: Reader: inconsistent block type")
 			}
-			iter := dataBlock.newIterator(slice, false, cache)
-			return iter
+			return b, ch, err
+		} else if err != nil {
+			return nil, nil, err
 		}
 	}
-	dataBlock, err := r.readBlock(dataBH, checksum)
+
+	b, err := r.readFilterBlock(bh)
+	return b, b, err
+}
+
+func (r *Reader) getIndexBlock(fillCache bool) (b *block, rel util.Releaser, err error) {
+	if r.indexBlock == nil {
+		return r.readBlockCached(r.indexBH, true, fillCache)
+	}
+	return r.indexBlock, util.NoopReleaser{}, nil
+}
+
+func (r *Reader) getFilterBlock(fillCache bool) (*filterBlock, util.Releaser, error) {
+	if r.filterBlock == nil {
+		return r.readFilterBlockCached(r.filterBH, fillCache)
+	}
+	return r.filterBlock, util.NoopReleaser{}, nil
+}
+
+func (r *Reader) getDataIter(dataBH blockHandle, slice *util.Range, checksum, fillCache bool) iterator.Iterator {
+	b, rel, err := r.readBlockCached(dataBH, checksum, fillCache)
 	if err != nil {
 		return iterator.NewEmptyIterator(err)
 	}
-	iter := dataBlock.newIterator(slice, false, nil)
-	return iter
+	return b.newIterator(slice, false, rel)
+}
+
+func (r *Reader) getDataIterErr(dataBH blockHandle, slice *util.Range, checksum, fillCache bool) iterator.Iterator {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
+	if r.err != nil {
+		return iterator.NewEmptyIterator(r.err)
+	}
+
+	return r.getDataIter(dataBH, slice, checksum, fillCache)
 }
 
 // NewIterator creates an iterator from the table.
@@ -653,20 +738,26 @@ func (r *Reader) getDataIter(dataBH blockHandle, slice *util.Range, checksum, fi
 // when not used.
 //
 // Also read Iterator documentation of the leveldb/iterator package.
-
 func (r *Reader) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
 	if r.err != nil {
 		return iterator.NewEmptyIterator(r.err)
 	}
 
-	index := &indexIter{
-		blockIter:   *r.indexBlock.newIterator(slice, true, nil),
-		tableReader: r,
-		slice:       slice,
-		checksum:    ro.GetStrict(opt.StrictBlockChecksum),
-		fillCache:   !ro.GetDontFillCache(),
+	fillCache := !ro.GetDontFillCache()
+	indexBlock, rel, err := r.getIndexBlock(fillCache)
+	if err != nil {
+		return iterator.NewEmptyIterator(err)
 	}
-	return iterator.NewIndexedIterator(index, r.strictIter || ro.GetStrict(opt.StrictIterator), false)
+	index := &indexIter{
+		blockIter: indexBlock.newIterator(slice, true, rel),
+		slice:     slice,
+		checksum:  ro.GetStrict(opt.StrictBlockChecksum),
+		fillCache: !ro.GetDontFillCache(),
+	}
+	return iterator.NewIndexedIterator(index, r.strictIter || ro.GetStrict(opt.StrictIterator), true)
 }
 
 // Find finds key/value pair whose key is greater than or equal to the
@@ -676,12 +767,21 @@ func (r *Reader) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.It
 // The caller should not modify the contents of the returned slice, but
 // it is safe to modify the contents of the argument after Find returns.
 func (r *Reader) Find(key []byte, ro *opt.ReadOptions) (rkey, value []byte, err error) {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
 	if r.err != nil {
 		err = r.err
 		return
 	}
 
-	index := r.indexBlock.newIterator(nil, true, nil)
+	indexBlock, rel, err := r.getIndexBlock(true)
+	if err != nil {
+		return
+	}
+	defer rel.Release()
+
+	index := indexBlock.newIterator(nil, true, nil)
 	defer index.Release()
 	if !index.Seek(key) {
 		err = index.Error()
@@ -695,9 +795,15 @@ func (r *Reader) Find(key []byte, ro *opt.ReadOptions) (rkey, value []byte, err
 		err = errors.New("leveldb/table: Reader: invalid table (bad data block handle)")
 		return
 	}
-	if r.filterBlock != nil && !r.filterBlock.contains(dataBH.offset, key) {
-		err = ErrNotFound
-		return
+	if r.filter != nil {
+		filterBlock, rel, ferr := r.getFilterBlock(true)
+		if ferr == nil {
+			if !filterBlock.contains(dataBH.offset, key) {
+				rel.Release()
+				return nil, nil, ErrNotFound
+			}
+			rel.Release()
+		}
 	}
 	data := r.getDataIter(dataBH, nil, ro.GetStrict(opt.StrictBlockChecksum), !ro.GetDontFillCache())
 	defer data.Release()
@@ -708,8 +814,15 @@ func (r *Reader) Find(key []byte, ro *opt.ReadOptions) (rkey, value []byte, err
 		}
 		return
 	}
+	// Don't use block buffer, no need to copy the buffer.
 	rkey = data.Key()
-	value = data.Value()
+	if r.bpool == nil {
+		value = data.Value()
+	} else {
+		// Use block buffer, and since the buffer will be recycled, the buffer
+		// need to be copied.
+		value = append([]byte{}, data.Value()...)
+	}
 	return
 }
 
@@ -719,6 +832,9 @@ func (r *Reader) Find(key []byte, ro *opt.ReadOptions) (rkey, value []byte, err
 // The caller should not modify the contents of the returned slice, but
 // it is safe to modify the contents of the argument after Get returns.
 func (r *Reader) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
 	if r.err != nil {
 		err = r.err
 		return
@@ -736,12 +852,21 @@ func (r *Reader) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error)
 //
 // It is safe to modify the contents of the argument after Get returns.
 func (r *Reader) OffsetOf(key []byte) (offset int64, err error) {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
 	if r.err != nil {
 		err = r.err
 		return
 	}
 
-	index := r.indexBlock.newIterator(nil, true, nil)
+	indexBlock, rel, err := r.readBlockCached(r.indexBH, true, true)
+	if err != nil {
+		return
+	}
+	defer rel.Release()
+
+	index := indexBlock.newIterator(nil, true, nil)
 	defer index.Release()
 	if index.Seek(key) {
 		dataBH, n := decodeBlockHandle(index.Value())
@@ -759,12 +884,38 @@ func (r *Reader) OffsetOf(key []byte) (offset int64, err error) {
 	return
 }
 
+// Release implements util.Releaser.
+// It also close the file if it is an io.Closer.
+func (r *Reader) Release() {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	if closer, ok := r.reader.(io.Closer); ok {
+		closer.Close()
+	}
+	if r.indexBlock != nil {
+		r.indexBlock.Release()
+		r.indexBlock = nil
+	}
+	if r.filterBlock != nil {
+		r.filterBlock.Release()
+		r.filterBlock = nil
+	}
+	r.reader = nil
+	r.cache = nil
+	r.bpool = nil
+	r.err = ErrReaderReleased
+}
+
 // NewReader creates a new initialized table reader for the file.
-// The cache is optional and can be nil.
-func NewReader(f io.ReaderAt, size int64, cache cache.Namespace, o *opt.Options) *Reader {
+// The cache and bpool is optional and can be nil.
+//
+// The returned table reader instance is goroutine-safe.
+func NewReader(f io.ReaderAt, size int64, cache cache.Namespace, bpool *util.BufferPool, o *opt.Options) *Reader {
 	r := &Reader{
 		reader:     f,
 		cache:      cache,
+		bpool:      bpool,
 		cmp:        o.GetComparer(),
 		checksum:   o.GetStrict(opt.StrictBlockChecksum),
 		strictIter: o.GetStrict(opt.StrictIterator),
@@ -792,16 +943,11 @@ func NewReader(f io.ReaderAt, size int64, cache cache.Namespace, o *opt.Options)
 		return r
 	}
 	// Decode the index block handle.
-	indexBH, n := decodeBlockHandle(footer[n:])
+	r.indexBH, n = decodeBlockHandle(footer[n:])
 	if n == 0 {
 		r.err = errors.New("leveldb/table: Reader: invalid table (bad index block handle)")
 		return r
 	}
-	// Read index block.
-	r.indexBlock, r.err = r.readBlock(indexBH, true)
-	if r.err != nil {
-		return r
-	}
 	// Read metaindex block.
 	metaBlock, err := r.readBlock(metaBH, true)
 	if err != nil {
@@ -817,32 +963,45 @@ func NewReader(f io.ReaderAt, size int64, cache cache.Namespace, o *opt.Options)
 			continue
 		}
 		fn := key[7:]
-		var filter filter.Filter
 		if f0 := o.GetFilter(); f0 != nil && f0.Name() == fn {
-			filter = f0
+			r.filter = f0
 		} else {
 			for _, f0 := range o.GetAltFilters() {
 				if f0.Name() == fn {
-					filter = f0
+					r.filter = f0
 					break
 				}
 			}
 		}
-		if filter != nil {
+		if r.filter != nil {
 			filterBH, n := decodeBlockHandle(metaIter.Value())
 			if n == 0 {
 				continue
 			}
+			r.filterBH = filterBH
 			// Update data end.
 			r.dataEnd = int64(filterBH.offset)
-			filterBlock, err := r.readFilterBlock(filterBH, filter)
-			if err != nil {
-				continue
-			}
-			r.filterBlock = filterBlock
 			break
 		}
 	}
 	metaIter.Release()
+	metaBlock.Release()
+
+	// Cache index and filter block locally, since we don't have global cache.
+	if cache == nil {
+		r.indexBlock, r.err = r.readBlock(r.indexBH, true)
+		if r.err != nil {
+			return r
+		}
+		if r.filter != nil {
+			r.filterBlock, err = r.readFilterBlock(r.filterBH)
+			if err != nil {
+				// Don't use filter then.
+				r.filter = nil
+				r.filterBH = blockHandle{}
+			}
+		}
+	}
+
 	return r
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table_test.go

@@ -59,7 +59,7 @@ var _ = testutil.Defer(func() {
 			It("Should be able to approximate offset of a key correctly", func() {
 				Expect(err).ShouldNot(HaveOccurred())
 
-				tr := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, o)
+				tr := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, nil, o)
 				CheckOffset := func(key string, expect, threshold int) {
 					offset, err := tr.OffsetOf([]byte(key))
 					Expect(err).ShouldNot(HaveOccurred())
@@ -95,7 +95,7 @@ var _ = testutil.Defer(func() {
 				tw.Close()
 
 				// Opening the table.
-				tr := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, o)
+				tr := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, nil, o)
 				return tableWrapper{tr}
 			}
 			Test := func(kv *testutil.KeyValue, body func(r *Reader)) func() {
@@ -104,14 +104,16 @@ var _ = testutil.Defer(func() {
 					if body != nil {
 						body(db.(tableWrapper).Reader)
 					}
-					testutil.KeyValueTesting(nil, db, *kv)
+					testutil.KeyValueTesting(nil, *kv, db, nil, nil)
 				}
 			}
 
-			testutil.AllKeyValueTesting(nil, Build)
+			testutil.AllKeyValueTesting(nil, Build, nil, nil)
 			Describe("with one key per block", Test(testutil.KeyValue_Generate(nil, 9, 1, 10, 512, 512), func(r *Reader) {
 				It("should have correct blocks number", func() {
-					Expect(r.indexBlock.restartsLen).Should(Equal(9))
+					indexBlock, err := r.readBlock(r.indexBH, true)
+					Expect(err).To(BeNil())
+					Expect(indexBlock.restartsLen).Should(Equal(9))
 				})
 			}))
 		})

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/kvtest.go

@@ -16,13 +16,22 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
-func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
+func KeyValueTesting(rnd *rand.Rand, kv KeyValue, p DB, setup func(KeyValue) DB, teardown func(DB)) {
 	if rnd == nil {
 		rnd = NewRand()
 	}
 
-	if db, ok := p.(Find); ok {
-		It("Should find all keys with Find", func() {
+	if p == nil {
+		BeforeEach(func() {
+			p = setup(kv)
+		})
+		AfterEach(func() {
+			teardown(p)
+		})
+	}
+
+	It("Should find all keys with Find", func() {
+		if db, ok := p.(Find); ok {
 			ShuffledIndex(nil, kv.Len(), 1, func(i int) {
 				key_, key, value := kv.IndexInexact(i)
 
@@ -38,9 +47,11 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
 				Expect(rkey).Should(Equal(key))
 				Expect(rvalue).Should(Equal(value), "Value for key %q (%q)", key_, key)
 			})
-		})
+		}
+	})
 
-		It("Should return error if the key is not present", func() {
+	It("Should return error if the key is not present", func() {
+		if db, ok := p.(Find); ok {
 			var key []byte
 			if kv.Len() > 0 {
 				key_, _ := kv.Index(kv.Len() - 1)
@@ -49,11 +60,11 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
 			rkey, _, err := db.TestFind(key)
 			Expect(err).Should(HaveOccurred(), "Find for key %q yield key %q", key, rkey)
 			Expect(err).Should(Equal(util.ErrNotFound))
-		})
-	}
+		}
+	})
 
-	if db, ok := p.(Get); ok {
-		It("Should only find exact key with Get", func() {
+	It("Should only find exact key with Get", func() {
+		if db, ok := p.(Get); ok {
 			ShuffledIndex(nil, kv.Len(), 1, func(i int) {
 				key_, key, value := kv.IndexInexact(i)
 
@@ -69,11 +80,11 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
 					Expect(err).Should(Equal(util.ErrNotFound))
 				}
 			})
-		})
-	}
+		}
+	})
 
-	if db, ok := p.(NewIterator); ok {
-		TestIter := func(r *util.Range, _kv KeyValue) {
+	TestIter := func(r *util.Range, _kv KeyValue) {
+		if db, ok := p.(NewIterator); ok {
 			iter := db.TestNewIterator(r)
 			Expect(iter.Error()).ShouldNot(HaveOccurred())
 
@@ -84,45 +95,48 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
 
 			DoIteratorTesting(&t)
 		}
+	}
 
-		It("Should iterates and seeks correctly", func(done Done) {
-			TestIter(nil, kv.Clone())
-			done <- true
-		}, 3.0)
+	It("Should iterates and seeks correctly", func(done Done) {
+		TestIter(nil, kv.Clone())
+		done <- true
+	}, 3.0)
 
-		RandomIndex(rnd, kv.Len(), kv.Len(), func(i int) {
-			type slice struct {
-				r            *util.Range
-				start, limit int
-			}
+	RandomIndex(rnd, kv.Len(), kv.Len(), func(i int) {
+		type slice struct {
+			r            *util.Range
+			start, limit int
+		}
 
-			key_, _, _ := kv.IndexInexact(i)
-			for _, x := range []slice{
-				{&util.Range{Start: key_, Limit: nil}, i, kv.Len()},
-				{&util.Range{Start: nil, Limit: key_}, 0, i},
-			} {
-				It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", x.start, x.limit), func(done Done) {
-					TestIter(x.r, kv.Slice(x.start, x.limit))
-					done <- true
-				}, 3.0)
-			}
-		})
-
-		RandomRange(rnd, kv.Len(), kv.Len(), func(start, limit int) {
-			It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", start, limit), func(done Done) {
-				r := kv.Range(start, limit)
-				TestIter(&r, kv.Slice(start, limit))
+		key_, _, _ := kv.IndexInexact(i)
+		for _, x := range []slice{
+			{&util.Range{Start: key_, Limit: nil}, i, kv.Len()},
+			{&util.Range{Start: nil, Limit: key_}, 0, i},
+		} {
+			It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", x.start, x.limit), func(done Done) {
+				TestIter(x.r, kv.Slice(x.start, x.limit))
 				done <- true
 			}, 3.0)
-		})
-	}
+		}
+	})
+
+	RandomRange(rnd, kv.Len(), kv.Len(), func(start, limit int) {
+		It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", start, limit), func(done Done) {
+			r := kv.Range(start, limit)
+			TestIter(&r, kv.Slice(start, limit))
+			done <- true
+		}, 3.0)
+	})
 }
 
-func AllKeyValueTesting(rnd *rand.Rand, body func(kv KeyValue) DB) {
+func AllKeyValueTesting(rnd *rand.Rand, body, setup func(KeyValue) DB, teardown func(DB)) {
 	Test := func(kv *KeyValue) func() {
 		return func() {
-			db := body(*kv)
-			KeyValueTesting(rnd, db, *kv)
+			var p DB
+			if body != nil {
+				p = body(*kv)
+			}
+			KeyValueTesting(rnd, *kv, p, setup, teardown)
 		}
 	}
 

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil_test.go

@@ -48,6 +48,7 @@ func (t *testingDB) TestClose() {
 func newTestingDB(o *opt.Options, ro *opt.ReadOptions, wo *opt.WriteOptions) *testingDB {
 	stor := testutil.NewStorage()
 	db, err := Open(stor, o)
+	// FIXME: This may be called from outside It, which may cause panic.
 	Expect(err).NotTo(HaveOccurred())
 	return &testingDB{
 		DB:   db,

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go

@@ -0,0 +1,221 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package util
+
+import (
+	"fmt"
+	"sync/atomic"
+	"time"
+)
+
+type buffer struct {
+	b    []byte
+	miss int
+}
+
+// BufferPool is a 'buffer pool'.
+type BufferPool struct {
+	pool       [6]chan []byte
+	size       [5]uint32
+	sizeMiss   [5]uint32
+	sizeHalf   [5]uint32
+	baseline   [4]int
+	baselinex0 int
+	baselinex1 int
+	baseline0  int
+	baseline1  int
+	baseline2  int
+	close      chan struct{}
+
+	get     uint32
+	put     uint32
+	half    uint32
+	less    uint32
+	equal   uint32
+	greater uint32
+	miss    uint32
+}
+
+func (p *BufferPool) poolNum(n int) int {
+	if n <= p.baseline0 && n > p.baseline0/2 {
+		return 0
+	}
+	for i, x := range p.baseline {
+		if n <= x {
+			return i + 1
+		}
+	}
+	return len(p.baseline) + 1
+}
+
+// Get returns buffer with length of n.
+func (p *BufferPool) Get(n int) []byte {
+	if p == nil {
+		return make([]byte, n)
+	}
+
+	atomic.AddUint32(&p.get, 1)
+
+	poolNum := p.poolNum(n)
+	pool := p.pool[poolNum]
+	if poolNum == 0 {
+		// Fast path.
+		select {
+		case b := <-pool:
+			switch {
+			case cap(b) > n:
+				if cap(b)-n >= n {
+					atomic.AddUint32(&p.half, 1)
+					select {
+					case pool <- b:
+					default:
+					}
+					return make([]byte, n)
+				} else {
+					atomic.AddUint32(&p.less, 1)
+					return b[:n]
+				}
+			case cap(b) == n:
+				atomic.AddUint32(&p.equal, 1)
+				return b[:n]
+			default:
+				atomic.AddUint32(&p.greater, 1)
+			}
+		default:
+			atomic.AddUint32(&p.miss, 1)
+		}
+
+		return make([]byte, n, p.baseline0)
+	} else {
+		sizePtr := &p.size[poolNum-1]
+
+		select {
+		case b := <-pool:
+			switch {
+			case cap(b) > n:
+				if cap(b)-n >= n {
+					atomic.AddUint32(&p.half, 1)
+					sizeHalfPtr := &p.sizeHalf[poolNum-1]
+					if atomic.AddUint32(sizeHalfPtr, 1) == 20 {
+						atomic.StoreUint32(sizePtr, uint32(cap(b)/2))
+						atomic.StoreUint32(sizeHalfPtr, 0)
+					} else {
+						select {
+						case pool <- b:
+						default:
+						}
+					}
+					return make([]byte, n)
+				} else {
+					atomic.AddUint32(&p.less, 1)
+					return b[:n]
+				}
+			case cap(b) == n:
+				atomic.AddUint32(&p.equal, 1)
+				return b[:n]
+			default:
+				atomic.AddUint32(&p.greater, 1)
+				if uint32(cap(b)) >= atomic.LoadUint32(sizePtr) {
+					select {
+					case pool <- b:
+					default:
+					}
+				}
+			}
+		default:
+			atomic.AddUint32(&p.miss, 1)
+		}
+
+		if size := atomic.LoadUint32(sizePtr); uint32(n) > size {
+			if size == 0 {
+				atomic.CompareAndSwapUint32(sizePtr, 0, uint32(n))
+			} else {
+				sizeMissPtr := &p.sizeMiss[poolNum-1]
+				if atomic.AddUint32(sizeMissPtr, 1) == 20 {
+					atomic.StoreUint32(sizePtr, uint32(n))
+					atomic.StoreUint32(sizeMissPtr, 0)
+				}
+			}
+			return make([]byte, n)
+		} else {
+			return make([]byte, n, size)
+		}
+	}
+}
+
+// Put adds given buffer to the pool.
+func (p *BufferPool) Put(b []byte) {
+	if p == nil {
+		return
+	}
+
+	atomic.AddUint32(&p.put, 1)
+
+	pool := p.pool[p.poolNum(cap(b))]
+	select {
+	case pool <- b:
+	default:
+	}
+
+}
+
+func (p *BufferPool) Close() {
+	if p == nil {
+		return
+	}
+
+	select {
+	case p.close <- struct{}{}:
+	default:
+	}
+}
+
+func (p *BufferPool) String() string {
+	if p == nil {
+		return "<nil>"
+	}
+
+	return fmt.Sprintf("BufferPool{B·%d Z·%v Zm·%v Zh·%v G·%d P·%d H·%d <·%d =·%d >·%d M·%d}",
+		p.baseline0, p.size, p.sizeMiss, p.sizeHalf, p.get, p.put, p.half, p.less, p.equal, p.greater, p.miss)
+}
+
+func (p *BufferPool) drain() {
+	ticker := time.NewTicker(2 * time.Second)
+	for {
+		select {
+		case <-ticker.C:
+			for _, ch := range p.pool {
+				select {
+				case <-ch:
+				default:
+				}
+			}
+		case <-p.close:
+			for _, ch := range p.pool {
+				close(ch)
+			}
+			return
+		}
+	}
+}
+
+// NewBufferPool creates a new initialized 'buffer pool'.
+func NewBufferPool(baseline int) *BufferPool {
+	if baseline <= 0 {
+		panic("baseline can't be <= 0")
+	}
+	p := &BufferPool{
+		baseline0: baseline,
+		baseline:  [...]int{baseline / 4, baseline / 2, baseline * 2, baseline * 4},
+		close:     make(chan struct{}, 1),
+	}
+	for i, cap := range []int{2, 2, 4, 4, 2, 1} {
+		p.pool[i] = make(chan []byte, cap)
+	}
+	go p.drain()
+	return p
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/pool.go

@@ -0,0 +1,21 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// +build go1.3
+
+package util
+
+import (
+	"sync"
+)
+
+type Pool struct {
+	sync.Pool
+}
+
+func NewPool(cap int) *Pool {
+	return &Pool{}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/pool_legacy.go

@@ -0,0 +1,33 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// +build !go1.3
+
+package util
+
+type Pool struct {
+	pool chan interface{}
+}
+
+func (p *Pool) Get() interface{} {
+	select {
+	case x := <-p.pool:
+		return x
+	default:
+		return nil
+	}
+}
+
+func (p *Pool) Put(x interface{}) {
+	select {
+	case p.pool <- x:
+	default:
+	}
+}
+
+func NewPool(cap int) *Pool {
+	return &Pool{pool: make(chan interface{}, cap)}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/range.go

@@ -14,3 +14,19 @@ type Range struct {
 	// Limit of the key range, not include in the range.
 	Limit []byte
 }
+
+// BytesPrefix returns key range that satisfy the given prefix.
+// This only applicable for the standard 'bytes comparer'.
+func BytesPrefix(prefix []byte) *Range {
+	var limit []byte
+	for i := len(prefix) - 1; i >= 0; i-- {
+		c := prefix[i]
+		if c < 0xff {
+			limit = make([]byte, i+1)
+			copy(limit, prefix)
+			limit[i] = c + 1
+			break
+		}
+	}
+	return &Range{prefix, limit}
+}