Fix broken upgrades (fixes #1175)

Hash blocks after receipt, try multiple peers (fixes #1166)

.gitignore

@@ -4,8 +4,14 @@ syncthing.exe
 *.zip
 *.asc
 *.sublime*
+.idea/
 .jshintrc
 coverage.out
 files/pidx
 bin
 perfstats*.csv
+coverage.xml
+!gui/scripts/syncthing
+.DS_Store
+syncthing.md5
+syncthing.exe.md5

.mailmap

@@ -0,0 +1 @@
+NICKS

.travis.yml

@@ -1,18 +0,0 @@
-language: go
-
-go:
-   - tip
-
-install:
-   - export PATH=$PATH:$HOME/gopath/bin
-   - ./build.sh setup
-
-script:
-   - ./build.sh test-cov
-
-after_success:
-   - goveralls -coverprofile=coverage.out -service=travis-ci -package=syncthing/syncthing -repotoken="$COVERALLS_TOKEN"
-
-env:
-   global:
-      secure: "TSPJDsokGCQhKLjgG3c58qHn8Qxhh4zEkWFf0XIOOY2nlDVzdgXDsC+Nq0YaP4106Ee4FgkSefsUTQV5lq/IyYW8elgqlgghjOtOi6RJa14eIS9Yy5Bkx6MXn0QfZX/lG+sy42pKSNk43y9GWx/qrt4nkfTtTvI5cXgwDGYdmX8="

AUTHORS

@@ -0,0 +1,35 @@
+# This is the official list of Syncthing authors for copyright purposes.
+
+Aaron Bieber <qbit@deftly.net>
+Alexander Graf <register-github@alex-graf.de>
+Andrew Dunham <andrew@du.nham.ca>
+Audrius Butkevicius <audrius.butkevicius@gmail.com>
+Arthur Axel fREW Schmidt <frew@afoolishmanifesto.com> <frioux@gmail.com>
+Ben Schulz <ueomkail@gmail.com> <uok@users.noreply.github.com>
+Ben Sidhom <bsidhom@gmail.com>
+Brandon Philips <brandon@ifup.org>
+Caleb Callaway <enlightened.despot@gmail.com>
+Cathryne Linenweaver <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
+Chris Joel <chris@scriptolo.gy>
+Daniel Martí <mvdan@mvdan.cc>
+Dennis Wilson <dw@risu.io>
+Dominik Heidler <dominik@heidler.eu>
+Emil Hessman <emil@hessman.se>
+Felix Ableitner <me@nutomic.com>
+Felix Unterpaintner <bigbear2nd@gmail.com>
+Gilli Sigurdsson <gilli@vx.is>
+Jakob Borg <jakob@nym.se>
+James Patterson <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
+Jens Diemer <github.com@jensdiemer.de> <git@jensdiemer.de>
+Jochen Voss <voss@seehuhn.de>
+Lode Hoste <zillode@zillode.be>
+Marcin Dziadus <dziadus.marcin@gmail.com>
+Michael Tilli <pyfisch@gmail.com>
+Philippe Schommers <philippe@schommers.be>
+Phill Luby <phill.luby@newredo.com>
+Piotr Bejda <piotrb10@gmail.com>
+Ryan Sullivan <kayoticsully@gmail.com>
+Tomas Cerveny <kozec@kozec.com>
+Tully Robinson <tully@tojr.org>
+Veeti Paananen <veeti.paananen@rojekti.fi>
+Vil Brekin <vilbrekin@gmail.com>

CONDUCT.md

@@ -0,0 +1,92 @@
+## Conduct
+
+* We are committed to providing a friendly, safe and welcoming
+  environment for all, regardless of gender, sexual orientation,
+  disability, ethnicity, religion, or similar personal characteristic.
+
+* On IRC, please avoid using overtly sexual nicknames or other nicknames
+  that might detract from a friendly, safe and welcoming environment for
+  all.
+
+* Please be kind and courteous. There's no need to be mean or rude.
+
+* Respect that people have differences of opinion and that every design
+  or implementation choice carries a trade-off and numerous costs. There
+  is seldom a right answer.
+
+* Please keep unstructured critique to a minimum. If you have solid
+  ideas you want to experiment with, make a fork and see how it works.
+
+* We will exclude you from interaction if you insult, demean or harass
+  anyone. That is not welcome behaviour. We interpret the term
+  "harassment" as including the definition in the <a
+  href="http://citizencodeofconduct.org/">Citizen Code of Conduct</a>;
+  if you have any lack of clarity about what might be included in that
+  concept, please read their definition. In particular, we don't
+  tolerate behavior that excludes people in socially marginalized
+  groups.
+
+* Private harassment is also unacceptable. No matter who you are, if you
+  feel you have been or are being harassed or made uncomfortable by a
+  community member, please contact one of the channel ops or any of the
+  Syncthing core team immediately. Whether you're a regular  contributor
+  or a newcomer, we care about making this community a safe place for
+  you and we've got your back.
+
+* Likewise any spamming, trolling, flaming, baiting or other
+  attention-stealing behaviour is not welcome.
+
+## Moderation
+
+These are the policies for upholding our community's standards of
+conduct in our communication channels, most notably in Syncthing-related
+IRC channels and on the web forum.
+
+1. Remarks that violate the Syncthing standards of conduct, including
+   hateful, hurtful, oppressive, or exclusionary remarks, are not
+   allowed. (Cursing is allowed, but never targeting another user, and
+   never in a hateful manner.)
+
+2. Remarks that moderators find inappropriate, whether listed in the
+   code of conduct or not, are also not allowed.
+
+3. Moderators will first respond to such remarks with a warning.
+
+4. If the warning is unheeded, the user will be "kicked," i.e., kicked
+   out of the communication channel to cool off.
+
+5. If the user comes back and continues to make trouble, they will be
+   banned, i.e., indefinitely excluded.
+
+6. Moderators may choose at their discretion to un-ban the user if it
+   was a first offense and they offer the offended party a genuine
+   apology.
+
+7. If a moderator bans someone and you think it was unjustified, please
+   take it up with that moderator, or with a different moderator, **in
+   private**. Complaints about bans in-channel are not allowed.
+
+8. Moderators are held to a higher standard than other community
+   members. If a moderator creates an inappropriate situation, they
+   should expect less leeway than others.
+
+In the Syncthing community we strive to go the extra step to look out
+for each other. Don't just aim to be technically unimpeachable, try to
+be your best self. In particular, avoid flirting with offensive or
+sensitive issues, particularly if they're off-topic; this all too
+often leads to unnecessary fights, hurt feelings, and damaged trust;
+worse, it can drive people away from the community entirely.
+
+And if someone takes issue with something you said or did, resist the
+urge to be defensive. Just stop doing what it was they complained about
+and apologize. Even if you feel you were misinterpreted or unfairly
+accused, chances are good there was something you could've communicated
+better — remember that it's your responsibility to make your fellow
+community members comfortable. Everyone wants to get along and we are
+all here first and foremost because we want to talk about cool
+technology. You will find that people will be eager to assume good
+intent and forgive as long as you earn their trust.
+
+*Adapted from the [Rust Code of Conduct](https://github.com/rust-lang/rust/wiki/Note-development-policy#conduct)*
+
+*Adapted from the [Node.js Policy on Trolling](http://blog.izs.me/post/30036893703/policy-on-trolling)*

CONTRIBUTING.md

@@ -1,23 +1,112 @@
-Please do contribute! If you want to contribute but are unsure where to
-start, the [Contributions Needed
-topic](http://discourse.syncthing.net/t/contributions-needed/49)
-lists areas in need of attention.
+## Reporting Bugs
+
+Please file bugs in the [Github Issue
+Tracker](https://github.com/syncthing/syncthing/issues). Include at
+least the following:
+
+ - What happened
+
+ - What did you expect to happen instead of what *did* happen, if it's
+   not crazy obvious
+
+ - What operating system, operating system version and version of
+   Syncthing you are running
+
+ - The same for other connected devices, where relevant
+
+ - Screenshot if the issue concerns something visible in the GUI
+
+ - Console log entries, where possible and relevant
+
+If you're not sure whether something is relevant, erring on the side of
+too much information will never get you yelled at. :)
+
+## Contributing Translations
+
+All translations are done via
+[Transifex](https://www.transifex.com/projects/p/syncthing/). If you
+wish to contribute to a translation, just head over there and sign up.
+Before every release, the language resources are updated from the
+latest info on Transifex.
+
+## Contributing Code
+
+Every contribution is welcome. If you want to contribute but are unsure
+where to start, any open issues are fair game! Be prepared for a
+[certain amount of review](https://discourse.syncthing.net/t/733); it's
+all in the name of quality. :) Following the points below will make this
+a smoother process.
+
+Individuals making significant and valuable contributions are given
+commit-access to the project. If you make a significant contribution and
+are not considered for commit-access, please contact any of the
+Syncthing core team members.
+
+All nontrivial contributions should go through the pull request
+mechanism for internal review. Determining what is "nontrivial" is left
+at the discretion of the contributor.
+
+### Authorship
+
+All code authors are listed in the AUTHORS file. Commits must be made
+with the same name and email as listed in the AUTHORS file. To
+accomplish this, ensure that your git configuration is set correctly
+prior to making your first commit;
+
+    $ git config --global user.name "Jane Doe"
+    $ git config --global user.email janedoe@example.com
+
+You must be reachable on the given email address. If you do not wish to
+use your real name for whatever reason, using a nickname or pseudonym is
+perfectly acceptable.
+
+### Core Team
+
+The Syncthing core team currently consists of the following members;
+
+ - Jakob Borg (@calmh)
+ - Audrius Butkevicius (@AudriusButkevicius)
+
+## Coding Style
+
+- Follow the conventions laid out in [Effective Go](https://golang.org/doc/effective_go.html)
+  as much as makes sense.
+
+- All text files use Unix line endings.
+
+- Each commit should be `go fmt` clean.
+
+- The commit message subject should be a single short sentence
+  describing the change, starting with a capital letter.
+
+- Commits that resolve an existing issue must include the issue number
+  as `(fixes #123)` at the end of the commit message subject.
+
+- Imports are grouped per `goimports` standard; that is, standard
+  library first, then third party libraries after a blank line.
+
+- A contribution solving a single issue or introducing a single new
+  feature should probably be a single commit based on the current
+  `master` branch. You may be asked to "rebase" or "squash" your pull
+  request to make sure this is the case, especially if there have been
+  amendments during review.
 
 ## Licensing
 
-All contributions are made under the same MIT License as the rest of the
-project, except documentation which is licensed under the Creative
-Commons Attribution 4.0 International License. You retain the copyright
-to code you have written.
+All contributions are made under the same GPL license as the rest of the
+project, except documentation, user interface text and translation
+strings which are licensed under the Creative Commons Attribution 4.0
+International License. You retain the copyright to code you have
+written.
 
 When accepting your first contribution, the maintainer of the project
-will ensure that you are added to the CONTRIBUTORS file. You are welcome
-to add yourself as a separate commit in your first pull request.
+will ensure that you are added to the AUTHORS file. You are welcome to
+add yourself as a separate commit in your first pull request.
 
 ## Building
 
-[See the
-documentation](http://discourse.syncthing.net/t/building-syncthing/44)
+[See the documentation](http://discourse.syncthing.net/t/44) on how to
+get started with a build environment.
 
 ## Branches
 
@@ -42,15 +131,10 @@ signed by GPG key BCE524C7.
 
 Yes please!
 
-## Style
-
-`go fmt`
-
 ## Documentation
 
 [Over here!](http://discourse.syncthing.net/category/documentation)
 
 ## License
 
-MIT
-
+GPLv3

CONTRIBUTORS

@@ -1,11 +0,0 @@
-Aaron Bieber <qbit@deftly.net>
-Andrew Dunham <andrew@du.nham.ca>
-Audrius Butkevicius <audrius.butkevicius@gmail.com>
-Arthur Axel fREW Schmidt <frew@afoolishmanifesto.com>
-Ben Sidhom <bsidhom@gmail.com>
-Brandon Philips <brandon@ifup.org>
-James Patterson <jamespatterson@operamail.com>
-Jens Diemer <github.com@jensdiemer.de>
-Philippe Schommers <philippe@schommers.be>
-Ryan Sullivan <kayoticsully@gmail.com>
-Veeti Paananen <veeti.paananen@rojekti.fi>

Godeps/Godeps.json

@@ -1,47 +1,29 @@
 {
 	"ImportPath": "github.com/syncthing/syncthing",
-	"GoVersion": "go1.3",
+	"GoVersion": "go1.4",
 	"Packages": [
 		"./cmd/..."
 	],
 	"Deps": [
 		{
-			"ImportPath": "bitbucket.org/kardianos/osext",
-			"Comment": "null-13",
-			"Rev": "5d3ddcf53a508cc2f7404eaebf546ef2cb5cdb6e"
-		},
-		{
-			"ImportPath": "code.google.com/p/go.crypto/bcrypt",
-			"Comment": "null-213",
-			"Rev": "aa2644fe4aa50e3b38d75187b4799b1f0c9ddcef"
-		},
-		{
-			"ImportPath": "code.google.com/p/go.crypto/blowfish",
-			"Comment": "null-213",
-			"Rev": "aa2644fe4aa50e3b38d75187b4799b1f0c9ddcef"
-		},
-		{
-			"ImportPath": "code.google.com/p/go.text/transform",
-			"Comment": "null-89",
-			"Rev": "df15baaf13e3f62b6b7a901e74caa3818a7c0a7c"
-		},
-		{
-			"ImportPath": "code.google.com/p/go.text/unicode/norm",
-			"Comment": "null-89",
-			"Rev": "df15baaf13e3f62b6b7a901e74caa3818a7c0a7c"
-		},
-		{
-			"ImportPath": "code.google.com/p/snappy-go/snappy",
-			"Comment": "null-15",
-			"Rev": "12e4b4183793ac4b061921e7980845e750679fd0"
+			"ImportPath": "github.com/AudriusButkevicius/lfu-go",
+			"Rev": "164bcecceb92fd6037f4d18a8d97b495ec6ef669"
 		},
 		{
 			"ImportPath": "github.com/bkaradzic/go-lz4",
-			"Rev": "77e2ba877bde9da31213bec75dbbe197fa507c21"
+			"Rev": "93a831dcee242be64a9cc9803dda84af25932de7"
+		},
+		{
+			"ImportPath": "github.com/calmh/logger",
+			"Rev": "f50d32b313bec2933a3e1049f7416a29f3413d29"
+		},
+		{
+			"ImportPath": "github.com/calmh/osext",
+			"Rev": "9bf61584e5f1f172e8766ddc9022d9c401faaa5e"
 		},
 		{
 			"ImportPath": "github.com/calmh/xdr",
-			"Rev": "694859acb207675085232438780db923ceb43e96"
+			"Rev": "45c46b7db7ff83b8b9ee09bbd95f36ab50043ece"
 		},
 		{
 			"ImportPath": "github.com/juju/ratelimit",
@@ -49,7 +31,11 @@
 		},
 		{
 			"ImportPath": "github.com/syndtr/goleveldb/leveldb",
-			"Rev": "c5955912e3287376475731c5bc59c79a5a799105"
+			"Rev": "63c9e642efad852f49e20a6f90194cae112fd2ac"
+		},
+		{
+			"ImportPath": "github.com/syndtr/gosnappy/snappy",
+			"Rev": "ce8acff4829e0c2458a67ead32390ac0a381c862"
 		},
 		{
 			"ImportPath": "github.com/vitrun/qart/coding",
@@ -62,6 +48,22 @@
 		{
 			"ImportPath": "github.com/vitrun/qart/qr",
 			"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
+		},
+		{
+			"ImportPath": "golang.org/x/crypto/bcrypt",
+			"Rev": "731db29863ea7213d9556d0170afb38987f401d4"
+		},
+		{
+			"ImportPath": "golang.org/x/crypto/blowfish",
+			"Rev": "731db29863ea7213d9556d0170afb38987f401d4"
+		},
+		{
+			"ImportPath": "golang.org/x/text/transform",
+			"Rev": "985ee5acfaf1ff6712c7c99438752f8e09416ccb"
+		},
+		{
+			"ImportPath": "golang.org/x/text/unicode/norm",
+			"Rev": "985ee5acfaf1ff6712c7c99438752f8e09416ccb"
 		}
 	]
 }

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/maketesttables.go

@@ -1,45 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build ignore
-
-// Generate test data for trie code.
-
-package main
-
-import (
-	"fmt"
-)
-
-func main() {
-	printTestTables()
-}
-
-// We take the smallest, largest and an arbitrary value for each
-// of the UTF-8 sequence lengths.
-var testRunes = []rune{
-	0x01, 0x0C, 0x7F, // 1-byte sequences
-	0x80, 0x100, 0x7FF, // 2-byte sequences
-	0x800, 0x999, 0xFFFF, // 3-byte sequences
-	0x10000, 0x10101, 0x10FFFF, // 4-byte sequences
-	0x200, 0x201, 0x202, 0x210, 0x215, // five entries in one sparse block
-}
-
-const fileHeader = `// Generated by running
-//	maketesttables
-// DO NOT EDIT
-
-package norm
-
-`
-
-func printTestTables() {
-	fmt.Print(fileHeader)
-	fmt.Printf("var testRunes = %#v\n\n", testRunes)
-	t := newNode()
-	for i, r := range testRunes {
-		t.insert(r, uint16(i))
-	}
-	t.printTables("testdata")
-}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/trie.go

@@ -1,232 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package norm
-
-type valueRange struct {
-	value  uint16 // header: value:stride
-	lo, hi byte   // header: lo:n
-}
-
-type trie struct {
-	index        []uint8
-	values       []uint16
-	sparse       []valueRange
-	sparseOffset []uint16
-	cutoff       uint8 // indices >= cutoff are sparse
-}
-
-// lookupValue determines the type of block n and looks up the value for b.
-// For n < t.cutoff, the block is a simple lookup table. Otherwise, the block
-// is a list of ranges with an accompanying value. Given a matching range r,
-// the value for b is by r.value + (b - r.lo) * stride.
-func (t *trie) lookupValue(n uint8, b byte) uint16 {
-	if n < t.cutoff {
-		return t.values[uint16(n)<<6+uint16(b)]
-	}
-	offset := t.sparseOffset[n-t.cutoff]
-	header := t.sparse[offset]
-	lo := offset + 1
-	hi := lo + uint16(header.lo)
-	for lo < hi {
-		m := lo + (hi-lo)/2
-		r := t.sparse[m]
-		if r.lo <= b && b <= r.hi {
-			return r.value + uint16(b-r.lo)*header.value
-		}
-		if b < r.lo {
-			hi = m
-		} else {
-			lo = m + 1
-		}
-	}
-	return 0
-}
-
-const (
-	t1 = 0x00 // 0000 0000
-	tx = 0x80 // 1000 0000
-	t2 = 0xC0 // 1100 0000
-	t3 = 0xE0 // 1110 0000
-	t4 = 0xF0 // 1111 0000
-	t5 = 0xF8 // 1111 1000
-	t6 = 0xFC // 1111 1100
-	te = 0xFE // 1111 1110
-)
-
-// lookup returns the trie value for the first UTF-8 encoding in s and
-// the width in bytes of this encoding. The size will be 0 if s does not
-// hold enough bytes to complete the encoding. len(s) must be greater than 0.
-func (t *trie) lookup(s []byte) (v uint16, sz int) {
-	c0 := s[0]
-	switch {
-	case c0 < tx:
-		return t.values[c0], 1
-	case c0 < t2:
-		return 0, 1
-	case c0 < t3:
-		if len(s) < 2 {
-			return 0, 0
-		}
-		i := t.index[c0]
-		c1 := s[1]
-		if c1 < tx || t2 <= c1 {
-			return 0, 1
-		}
-		return t.lookupValue(i, c1), 2
-	case c0 < t4:
-		if len(s) < 3 {
-			return 0, 0
-		}
-		i := t.index[c0]
-		c1 := s[1]
-		if c1 < tx || t2 <= c1 {
-			return 0, 1
-		}
-		o := uint16(i)<<6 + uint16(c1)
-		i = t.index[o]
-		c2 := s[2]
-		if c2 < tx || t2 <= c2 {
-			return 0, 2
-		}
-		return t.lookupValue(i, c2), 3
-	case c0 < t5:
-		if len(s) < 4 {
-			return 0, 0
-		}
-		i := t.index[c0]
-		c1 := s[1]
-		if c1 < tx || t2 <= c1 {
-			return 0, 1
-		}
-		o := uint16(i)<<6 + uint16(c1)
-		i = t.index[o]
-		c2 := s[2]
-		if c2 < tx || t2 <= c2 {
-			return 0, 2
-		}
-		o = uint16(i)<<6 + uint16(c2)
-		i = t.index[o]
-		c3 := s[3]
-		if c3 < tx || t2 <= c3 {
-			return 0, 3
-		}
-		return t.lookupValue(i, c3), 4
-	}
-	// Illegal rune
-	return 0, 1
-}
-
-// lookupString returns the trie value for the first UTF-8 encoding in s and
-// the width in bytes of this encoding. The size will be 0 if s does not
-// hold enough bytes to complete the encoding. len(s) must be greater than 0.
-func (t *trie) lookupString(s string) (v uint16, sz int) {
-	c0 := s[0]
-	switch {
-	case c0 < tx:
-		return t.values[c0], 1
-	case c0 < t2:
-		return 0, 1
-	case c0 < t3:
-		if len(s) < 2 {
-			return 0, 0
-		}
-		i := t.index[c0]
-		c1 := s[1]
-		if c1 < tx || t2 <= c1 {
-			return 0, 1
-		}
-		return t.lookupValue(i, c1), 2
-	case c0 < t4:
-		if len(s) < 3 {
-			return 0, 0
-		}
-		i := t.index[c0]
-		c1 := s[1]
-		if c1 < tx || t2 <= c1 {
-			return 0, 1
-		}
-		o := uint16(i)<<6 + uint16(c1)
-		i = t.index[o]
-		c2 := s[2]
-		if c2 < tx || t2 <= c2 {
-			return 0, 2
-		}
-		return t.lookupValue(i, c2), 3
-	case c0 < t5:
-		if len(s) < 4 {
-			return 0, 0
-		}
-		i := t.index[c0]
-		c1 := s[1]
-		if c1 < tx || t2 <= c1 {
-			return 0, 1
-		}
-		o := uint16(i)<<6 + uint16(c1)
-		i = t.index[o]
-		c2 := s[2]
-		if c2 < tx || t2 <= c2 {
-			return 0, 2
-		}
-		o = uint16(i)<<6 + uint16(c2)
-		i = t.index[o]
-		c3 := s[3]
-		if c3 < tx || t2 <= c3 {
-			return 0, 3
-		}
-		return t.lookupValue(i, c3), 4
-	}
-	// Illegal rune
-	return 0, 1
-}
-
-// lookupUnsafe returns the trie value for the first UTF-8 encoding in s.
-// s must hold a full encoding.
-func (t *trie) lookupUnsafe(s []byte) uint16 {
-	c0 := s[0]
-	if c0 < tx {
-		return t.values[c0]
-	}
-	if c0 < t2 {
-		return 0
-	}
-	i := t.index[c0]
-	if c0 < t3 {
-		return t.lookupValue(i, s[1])
-	}
-	i = t.index[uint16(i)<<6+uint16(s[1])]
-	if c0 < t4 {
-		return t.lookupValue(i, s[2])
-	}
-	i = t.index[uint16(i)<<6+uint16(s[2])]
-	if c0 < t5 {
-		return t.lookupValue(i, s[3])
-	}
-	return 0
-}
-
-// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s.
-// s must hold a full encoding.
-func (t *trie) lookupStringUnsafe(s string) uint16 {
-	c0 := s[0]
-	if c0 < tx {
-		return t.values[c0]
-	}
-	if c0 < t2 {
-		return 0
-	}
-	i := t.index[c0]
-	if c0 < t3 {
-		return t.lookupValue(i, s[1])
-	}
-	i = t.index[uint16(i)<<6+uint16(s[1])]
-	if c0 < t4 {
-		return t.lookupValue(i, s[2])
-	}
-	i = t.index[uint16(i)<<6+uint16(s[2])]
-	if c0 < t5 {
-		return t.lookupValue(i, s[3])
-	}
-	return 0
-}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/trie_test.go

@@ -1,152 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package norm
-
-import (
-	"testing"
-	"unicode/utf8"
-)
-
-// Test data is located in triedata_test.go; generated by maketesttables.
-var testdata = testdataTrie
-
-type rangeTest struct {
-	block   uint8
-	lookup  byte
-	result  uint16
-	table   []valueRange
-	offsets []uint16
-}
-
-var range1Off = []uint16{0, 2}
-var range1 = []valueRange{
-	{0, 1, 0},
-	{1, 0x80, 0x80},
-	{0, 2, 0},
-	{1, 0x80, 0x80},
-	{9, 0xff, 0xff},
-}
-
-var rangeTests = []rangeTest{
-	{10, 0x80, 1, range1, range1Off},
-	{10, 0x00, 0, range1, range1Off},
-	{11, 0x80, 1, range1, range1Off},
-	{11, 0xff, 9, range1, range1Off},
-	{11, 0x00, 0, range1, range1Off},
-}
-
-func TestLookupSparse(t *testing.T) {
-	for i, test := range rangeTests {
-		n := trie{sparse: test.table, sparseOffset: test.offsets, cutoff: 10}
-		v := n.lookupValue(test.block, test.lookup)
-		if v != test.result {
-			t.Errorf("LookupSparse:%d: found %X; want %X", i, v, test.result)
-		}
-	}
-}
-
-// Test cases for illegal runes.
-type trietest struct {
-	size  int
-	bytes []byte
-}
-
-var tests = []trietest{
-	// illegal runes
-	{1, []byte{0x80}},
-	{1, []byte{0xFF}},
-	{1, []byte{t2, tx - 1}},
-	{1, []byte{t2, t2}},
-	{2, []byte{t3, tx, tx - 1}},
-	{2, []byte{t3, tx, t2}},
-	{1, []byte{t3, tx - 1, tx}},
-	{3, []byte{t4, tx, tx, tx - 1}},
-	{3, []byte{t4, tx, tx, t2}},
-	{1, []byte{t4, t2, tx, tx - 1}},
-	{2, []byte{t4, tx, t2, tx - 1}},
-
-	// short runes
-	{0, []byte{t2}},
-	{0, []byte{t3, tx}},
-	{0, []byte{t4, tx, tx}},
-
-	// we only support UTF-8 up to utf8.UTFMax bytes (4 bytes)
-	{1, []byte{t5, tx, tx, tx, tx}},
-	{1, []byte{t6, tx, tx, tx, tx, tx}},
-}
-
-func mkUTF8(r rune) ([]byte, int) {
-	var b [utf8.UTFMax]byte
-	sz := utf8.EncodeRune(b[:], r)
-	return b[:sz], sz
-}
-
-func TestLookup(t *testing.T) {
-	for i, tt := range testRunes {
-		b, szg := mkUTF8(tt)
-		v, szt := testdata.lookup(b)
-		if int(v) != i {
-			t.Errorf("lookup(%U): found value %#x, expected %#x", tt, v, i)
-		}
-		if szt != szg {
-			t.Errorf("lookup(%U): found size %d, expected %d", tt, szt, szg)
-		}
-	}
-	for i, tt := range tests {
-		v, sz := testdata.lookup(tt.bytes)
-		if v != 0 {
-			t.Errorf("lookup of illegal rune, case %d: found value %#x, expected 0", i, v)
-		}
-		if sz != tt.size {
-			t.Errorf("lookup of illegal rune, case %d: found size %d, expected %d", i, sz, tt.size)
-		}
-	}
-	// Verify defaults.
-	if v, _ := testdata.lookup([]byte{0xC1, 0x8C}); v != 0 {
-		t.Errorf("lookup of non-existing rune should be 0; found %X", v)
-	}
-}
-
-func TestLookupUnsafe(t *testing.T) {
-	for i, tt := range testRunes {
-		b, _ := mkUTF8(tt)
-		v := testdata.lookupUnsafe(b)
-		if int(v) != i {
-			t.Errorf("lookupUnsafe(%U): found value %#x, expected %#x", i, v, i)
-		}
-	}
-}
-
-func TestLookupString(t *testing.T) {
-	for i, tt := range testRunes {
-		b, szg := mkUTF8(tt)
-		v, szt := testdata.lookupString(string(b))
-		if int(v) != i {
-			t.Errorf("lookup(%U): found value %#x, expected %#x", i, v, i)
-		}
-		if szt != szg {
-			t.Errorf("lookup(%U): found size %d, expected %d", i, szt, szg)
-		}
-	}
-	for i, tt := range tests {
-		v, sz := testdata.lookupString(string(tt.bytes))
-		if int(v) != 0 {
-			t.Errorf("lookup of illegal rune, case %d: found value %#x, expected 0", i, v)
-		}
-		if sz != tt.size {
-			t.Errorf("lookup of illegal rune, case %d: found size %d, expected %d", i, sz, tt.size)
-		}
-	}
-}
-
-func TestLookupStringUnsafe(t *testing.T) {
-	for i, tt := range testRunes {
-		b, _ := mkUTF8(tt)
-		v := testdata.lookupStringUnsafe(string(b))
-		if int(v) != i {
-			t.Errorf("lookupUnsafe(%U): found value %#x, expected %#x", i, v, i)
-		}
-	}
-}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/triedata_test.go

@@ -1,85 +0,0 @@
-// Generated by running
-//	maketesttables
-// DO NOT EDIT
-
-package norm
-
-var testRunes = []int32{1, 12, 127, 128, 256, 2047, 2048, 2457, 65535, 65536, 65793, 1114111, 512, 513, 514, 528, 533}
-
-// testdataValues: 192 entries, 384 bytes
-// Block 2 is the null block.
-var testdataValues = [192]uint16{
-	// Block 0x0, offset 0x0
-	0x000c: 0x0001,
-	// Block 0x1, offset 0x40
-	0x007f: 0x0002,
-	// Block 0x2, offset 0x80
-}
-
-// testdataSparseOffset: 10 entries, 20 bytes
-var testdataSparseOffset = []uint16{0x0, 0x2, 0x4, 0x8, 0xa, 0xc, 0xe, 0x10, 0x12, 0x14}
-
-// testdataSparseValues: 22 entries, 88 bytes
-var testdataSparseValues = [22]valueRange{
-	// Block 0x0, offset 0x1
-	{value: 0x0000, lo: 0x01},
-	{value: 0x0003, lo: 0x80, hi: 0x80},
-	// Block 0x1, offset 0x2
-	{value: 0x0000, lo: 0x01},
-	{value: 0x0004, lo: 0x80, hi: 0x80},
-	// Block 0x2, offset 0x3
-	{value: 0x0001, lo: 0x03},
-	{value: 0x000c, lo: 0x80, hi: 0x82},
-	{value: 0x000f, lo: 0x90, hi: 0x90},
-	{value: 0x0010, lo: 0x95, hi: 0x95},
-	// Block 0x3, offset 0x4
-	{value: 0x0000, lo: 0x01},
-	{value: 0x0005, lo: 0xbf, hi: 0xbf},
-	// Block 0x4, offset 0x5
-	{value: 0x0000, lo: 0x01},
-	{value: 0x0006, lo: 0x80, hi: 0x80},
-	// Block 0x5, offset 0x6
-	{value: 0x0000, lo: 0x01},
-	{value: 0x0007, lo: 0x99, hi: 0x99},
-	// Block 0x6, offset 0x7
-	{value: 0x0000, lo: 0x01},
-	{value: 0x0008, lo: 0xbf, hi: 0xbf},
-	// Block 0x7, offset 0x8
-	{value: 0x0000, lo: 0x01},
-	{value: 0x0009, lo: 0x80, hi: 0x80},
-	// Block 0x8, offset 0x9
-	{value: 0x0000, lo: 0x01},
-	{value: 0x000a, lo: 0x81, hi: 0x81},
-	// Block 0x9, offset 0xa
-	{value: 0x0000, lo: 0x01},
-	{value: 0x000b, lo: 0xbf, hi: 0xbf},
-}
-
-// testdataLookup: 640 bytes
-// Block 0 is the null block.
-var testdataLookup = [640]uint8{
-	// Block 0x0, offset 0x0
-	// Block 0x1, offset 0x40
-	// Block 0x2, offset 0x80
-	// Block 0x3, offset 0xc0
-	0x0c2: 0x01, 0x0c4: 0x02,
-	0x0c8: 0x03,
-	0x0df: 0x04,
-	0x0e0: 0x02,
-	0x0ef: 0x03,
-	0x0f0: 0x05, 0x0f4: 0x07,
-	// Block 0x4, offset 0x100
-	0x120: 0x05, 0x126: 0x06,
-	// Block 0x5, offset 0x140
-	0x17f: 0x07,
-	// Block 0x6, offset 0x180
-	0x180: 0x08, 0x184: 0x09,
-	// Block 0x7, offset 0x1c0
-	0x1d0: 0x04,
-	// Block 0x8, offset 0x200
-	0x23f: 0x0a,
-	// Block 0x9, offset 0x240
-	0x24f: 0x06,
-}
-
-var testdataTrie = trie{testdataLookup[:], testdataValues[:], testdataSparseValues[:], testdataSparseOffset[:], 1}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/triegen.go

@@ -1,317 +0,0 @@
-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build ignore
-
-// Trie table generator.
-// Used by make*tables tools to generate a go file with trie data structures
-// for mapping UTF-8 to a 16-bit value. All but the last byte in a UTF-8 byte
-// sequence are used to lookup offsets in the index table to be used for the
-// next byte. The last byte is used to index into a table with 16-bit values.
-
-package main
-
-import (
-	"fmt"
-	"hash/crc32"
-	"log"
-	"unicode/utf8"
-)
-
-const (
-	blockSize        = 64
-	blockOffset      = 2 // Subtract two blocks to compensate for the 0x80 added to continuation bytes.
-	maxSparseEntries = 16
-)
-
-// Intermediate trie structure
-type trieNode struct {
-	table [256]*trieNode
-	value int
-	b     byte
-	leaf  bool
-}
-
-func newNode() *trieNode {
-	return new(trieNode)
-}
-
-func (n trieNode) String() string {
-	s := fmt.Sprint("trieNode{table: { non-nil at index: ")
-	for i, v := range n.table {
-		if v != nil {
-			s += fmt.Sprintf("%d, ", i)
-		}
-	}
-	s += fmt.Sprintf("}, value:%#x, b:%#x leaf:%v}", n.value, n.b, n.leaf)
-	return s
-}
-
-func (n trieNode) isInternal() bool {
-	internal := true
-	for i := 0; i < 256; i++ {
-		if nn := n.table[i]; nn != nil {
-			if !internal && !nn.leaf {
-				log.Fatalf("triegen: isInternal: node contains both leaf and non-leaf children (%v)", n)
-			}
-			internal = internal && !nn.leaf
-		}
-	}
-	return internal
-}
-
-func (n trieNode) mostFrequentStride() int {
-	counts := make(map[int]int)
-	v := 0
-	for _, t := range n.table[0x80 : 0x80+blockSize] {
-		if t != nil {
-			if stride := t.value - v; v != 0 && stride >= 0 {
-				counts[stride]++
-			}
-			v = t.value
-		} else {
-			v = 0
-		}
-	}
-	var maxs, maxc int
-	for stride, cnt := range counts {
-		if cnt > maxc || (cnt == maxc && stride < maxs) {
-			maxs, maxc = stride, cnt
-		}
-	}
-	return maxs
-}
-
-func (n trieNode) countSparseEntries() int {
-	stride := n.mostFrequentStride()
-	var count, v int
-	for _, t := range n.table[0x80 : 0x80+blockSize] {
-		tv := 0
-		if t != nil {
-			tv = t.value
-		}
-		if tv-v != stride {
-			if tv != 0 {
-				count++
-			}
-		}
-		v = tv
-	}
-	return count
-}
-
-func (n *trieNode) insert(r rune, value uint16) {
-	var p [utf8.UTFMax]byte
-	sz := utf8.EncodeRune(p[:], r)
-
-	for i := 0; i < sz; i++ {
-		if n.leaf {
-			log.Fatalf("triegen: insert: node (%#v) should not be a leaf", n)
-		}
-		nn := n.table[p[i]]
-		if nn == nil {
-			nn = newNode()
-			nn.b = p[i]
-			n.table[p[i]] = nn
-		}
-		n = nn
-	}
-	n.value = int(value)
-	n.leaf = true
-}
-
-type nodeIndex struct {
-	lookupBlocks []*trieNode
-	valueBlocks  []*trieNode
-	sparseBlocks []*trieNode
-	sparseOffset []uint16
-	sparseCount  int
-
-	lookupBlockIdx map[uint32]int
-	valueBlockIdx  map[uint32]int
-}
-
-func newIndex() *nodeIndex {
-	index := &nodeIndex{}
-	index.lookupBlocks = make([]*trieNode, 0)
-	index.valueBlocks = make([]*trieNode, 0)
-	index.sparseBlocks = make([]*trieNode, 0)
-	index.sparseOffset = make([]uint16, 1)
-	index.lookupBlockIdx = make(map[uint32]int)
-	index.valueBlockIdx = make(map[uint32]int)
-	return index
-}
-
-func computeOffsets(index *nodeIndex, n *trieNode) int {
-	if n.leaf {
-		return n.value
-	}
-	hasher := crc32.New(crc32.MakeTable(crc32.IEEE))
-	// We only index continuation bytes.
-	for i := 0; i < blockSize; i++ {
-		v := 0
-		if nn := n.table[0x80+i]; nn != nil {
-			v = computeOffsets(index, nn)
-		}
-		hasher.Write([]byte{uint8(v >> 8), uint8(v)})
-	}
-	h := hasher.Sum32()
-	if n.isInternal() {
-		v, ok := index.lookupBlockIdx[h]
-		if !ok {
-			v = len(index.lookupBlocks) - blockOffset
-			index.lookupBlocks = append(index.lookupBlocks, n)
-			index.lookupBlockIdx[h] = v
-		}
-		n.value = v
-	} else {
-		v, ok := index.valueBlockIdx[h]
-		if !ok {
-			if c := n.countSparseEntries(); c > maxSparseEntries {
-				v = len(index.valueBlocks) - blockOffset
-				index.valueBlocks = append(index.valueBlocks, n)
-				index.valueBlockIdx[h] = v
-			} else {
-				v = -len(index.sparseOffset)
-				index.sparseBlocks = append(index.sparseBlocks, n)
-				index.sparseOffset = append(index.sparseOffset, uint16(index.sparseCount))
-				index.sparseCount += c + 1
-				index.valueBlockIdx[h] = v
-			}
-		}
-		n.value = v
-	}
-	return n.value
-}
-
-func printValueBlock(nr int, n *trieNode, offset int) {
-	boff := nr * blockSize
-	fmt.Printf("\n// Block %#x, offset %#x", nr, boff)
-	var printnewline bool
-	for i := 0; i < blockSize; i++ {
-		if i%6 == 0 {
-			printnewline = true
-		}
-		v := 0
-		if nn := n.table[i+offset]; nn != nil {
-			v = nn.value
-		}
-		if v != 0 {
-			if printnewline {
-				fmt.Printf("\n")
-				printnewline = false
-			}
-			fmt.Printf("%#04x:%#04x, ", boff+i, v)
-		}
-	}
-}
-
-func printSparseBlock(nr int, n *trieNode) {
-	boff := -n.value
-	fmt.Printf("\n// Block %#x, offset %#x", nr, boff)
-	v := 0
-	//stride := f(n)
-	stride := n.mostFrequentStride()
-	c := n.countSparseEntries()
-	fmt.Printf("\n{value:%#04x,lo:%#02x},", stride, uint8(c))
-	for i, nn := range n.table[0x80 : 0x80+blockSize] {
-		nv := 0
-		if nn != nil {
-			nv = nn.value
-		}
-		if nv-v != stride {
-			if v != 0 {
-				fmt.Printf(",hi:%#02x},", 0x80+i-1)
-			}
-			if nv != 0 {
-				fmt.Printf("\n{value:%#04x,lo:%#02x", nv, nn.b)
-			}
-		}
-		v = nv
-	}
-	if v != 0 {
-		fmt.Printf(",hi:%#02x},", 0x80+blockSize-1)
-	}
-}
-
-func printLookupBlock(nr int, n *trieNode, offset, cutoff int) {
-	boff := nr * blockSize
-	fmt.Printf("\n// Block %#x, offset %#x", nr, boff)
-	var printnewline bool
-	for i := 0; i < blockSize; i++ {
-		if i%8 == 0 {
-			printnewline = true
-		}
-		v := 0
-		if nn := n.table[i+offset]; nn != nil {
-			v = nn.value
-		}
-		if v != 0 {
-			if v < 0 {
-				v = -v - 1 + cutoff
-			}
-			if printnewline {
-				fmt.Printf("\n")
-				printnewline = false
-			}
-			fmt.Printf("%#03x:%#02x, ", boff+i, v)
-		}
-	}
-}
-
-// printTables returns the size in bytes of the generated tables.
-func (t *trieNode) printTables(name string) int {
-	index := newIndex()
-	// Values for 7-bit ASCII are stored in first two block, followed by nil block.
-	index.valueBlocks = append(index.valueBlocks, nil, nil, nil)
-	// First byte of multi-byte UTF-8 codepoints are indexed in 4th block.
-	index.lookupBlocks = append(index.lookupBlocks, nil, nil, nil, nil)
-	// Index starter bytes of multi-byte UTF-8.
-	for i := 0xC0; i < 0x100; i++ {
-		if t.table[i] != nil {
-			computeOffsets(index, t.table[i])
-		}
-	}
-
-	nv := len(index.valueBlocks) * blockSize
-	fmt.Printf("// %sValues: %d entries, %d bytes\n", name, nv, nv*2)
-	fmt.Printf("// Block 2 is the null block.\n")
-	fmt.Printf("var %sValues = [%d]uint16 {", name, nv)
-	printValueBlock(0, t, 0)
-	printValueBlock(1, t, 64)
-	printValueBlock(2, newNode(), 0)
-	for i := 3; i < len(index.valueBlocks); i++ {
-		printValueBlock(i, index.valueBlocks[i], 0x80)
-	}
-	fmt.Print("\n}\n\n")
-
-	ls := len(index.sparseBlocks)
-	fmt.Printf("// %sSparseOffset: %d entries, %d bytes\n", name, ls, ls*2)
-	fmt.Printf("var %sSparseOffset = %#v\n\n", name, index.sparseOffset[1:])
-
-	ns := index.sparseCount
-	fmt.Printf("// %sSparseValues: %d entries, %d bytes\n", name, ns, ns*4)
-	fmt.Printf("var %sSparseValues = [%d]valueRange {", name, ns)
-	for i, n := range index.sparseBlocks {
-		printSparseBlock(i, n)
-	}
-	fmt.Print("\n}\n\n")
-
-	cutoff := len(index.valueBlocks) - blockOffset
-	ni := len(index.lookupBlocks) * blockSize
-	fmt.Printf("// %sLookup: %d bytes\n", name, ni)
-	fmt.Printf("// Block 0 is the null block.\n")
-	fmt.Printf("var %sLookup = [%d]uint8 {", name, ni)
-	printLookupBlock(0, newNode(), 0, cutoff)
-	printLookupBlock(1, newNode(), 0, cutoff)
-	printLookupBlock(2, newNode(), 0, cutoff)
-	printLookupBlock(3, t, 0xC0, cutoff)
-	for i := 4; i < len(index.lookupBlocks); i++ {
-		printLookupBlock(i, index.lookupBlocks[i], 0x80, cutoff)
-	}
-	fmt.Print("\n}\n\n")
-	fmt.Printf("var %sTrie = trie{ %sLookup[:], %sValues[:], %sSparseValues[:], %sSparseOffset[:], %d}\n\n",
-		name, name, name, name, name, cutoff)
-	return nv*2 + ns*4 + ni + ls*2
-}

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2012 Dave Grijalva
+
+Permission is hereby granted, free of charge, to any person obtaining a 
+copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation 
+the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the 
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included 
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 
+THE SOFTWARE.

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/README.md

@@ -0,0 +1,19 @@
+A simple LFU cache for golang.  Based on the paper [An O(1) algorithm for implementing the LFU cache eviction scheme](http://dhruvbird.com/lfu.pdf).
+
+Usage:
+
+```go
+import "github.com/dgrijalva/lfu-go"
+
+// Make a new thing
+c := lfu.New()
+
+// Set some values
+c.Set("myKey", myValue)
+
+// Retrieve some values
+myValue = c.Get("myKey")
+
+// Evict some values
+c.Evict(1)
+```

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/lfu.go

@@ -0,0 +1,156 @@
+package lfu
+
+import (
+	"container/list"
+	"sync"
+)
+
+type Eviction struct {
+	Key   string
+	Value interface{}
+}
+
+type Cache struct {
+	// If len > UpperBound, cache will automatically evict
+	// down to LowerBound.  If either value is 0, this behavior
+	// is disabled.
+	UpperBound      int
+	LowerBound      int
+	values          map[string]*cacheEntry
+	freqs           *list.List
+	len             int
+	lock            *sync.Mutex
+	EvictionChannel chan<- Eviction
+}
+
+type cacheEntry struct {
+	key      string
+	value    interface{}
+	freqNode *list.Element
+}
+
+type listEntry struct {
+	entries map[*cacheEntry]byte
+	freq    int
+}
+
+func New() *Cache {
+	c := new(Cache)
+	c.values = make(map[string]*cacheEntry)
+	c.freqs = list.New()
+	c.lock = new(sync.Mutex)
+	return c
+}
+
+func (c *Cache) Get(key string) interface{} {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	if e, ok := c.values[key]; ok {
+		c.increment(e)
+		return e.value
+	}
+	return nil
+}
+
+func (c *Cache) Set(key string, value interface{}) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	if e, ok := c.values[key]; ok {
+		// value already exists for key.  overwrite
+		e.value = value
+		c.increment(e)
+	} else {
+		// value doesn't exist.  insert
+		e := new(cacheEntry)
+		e.key = key
+		e.value = value
+		c.values[key] = e
+		c.increment(e)
+		c.len++
+		// bounds mgmt
+		if c.UpperBound > 0 && c.LowerBound > 0 {
+			if c.len > c.UpperBound {
+				c.evict(c.len - c.LowerBound)
+			}
+		}
+	}
+}
+
+func (c *Cache) Len() int {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return c.len
+}
+
+func (c *Cache) Evict(count int) int {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return c.evict(count)
+}
+
+func (c *Cache) evict(count int) int {
+	// No lock here so it can be called
+	// from within the lock (during Set)
+	var evicted int
+	for i := 0; i < count; {
+		if place := c.freqs.Front(); place != nil {
+			for entry, _ := range place.Value.(*listEntry).entries {
+				if i < count {
+					if c.EvictionChannel != nil {
+						c.EvictionChannel <- Eviction{
+							Key:   entry.key,
+							Value: entry.value,
+						}
+					}
+					delete(c.values, entry.key)
+					c.remEntry(place, entry)
+					evicted++
+					c.len--
+					i++
+				}
+			}
+		}
+	}
+	return evicted
+}
+
+func (c *Cache) increment(e *cacheEntry) {
+	currentPlace := e.freqNode
+	var nextFreq int
+	var nextPlace *list.Element
+	if currentPlace == nil {
+		// new entry
+		nextFreq = 1
+		nextPlace = c.freqs.Front()
+	} else {
+		// move up
+		nextFreq = currentPlace.Value.(*listEntry).freq + 1
+		nextPlace = currentPlace.Next()
+	}
+
+	if nextPlace == nil || nextPlace.Value.(*listEntry).freq != nextFreq {
+		// create a new list entry
+		li := new(listEntry)
+		li.freq = nextFreq
+		li.entries = make(map[*cacheEntry]byte)
+		if currentPlace != nil {
+			nextPlace = c.freqs.InsertAfter(li, currentPlace)
+		} else {
+			nextPlace = c.freqs.PushFront(li)
+		}
+	}
+	e.freqNode = nextPlace
+	nextPlace.Value.(*listEntry).entries[e] = 1
+	if currentPlace != nil {
+		// remove from current position
+		c.remEntry(currentPlace, e)
+	}
+}
+
+func (c *Cache) remEntry(place *list.Element, entry *cacheEntry) {
+	entries := place.Value.(*listEntry).entries
+	delete(entries, entry)
+	if len(entries) == 0 {
+		c.freqs.Remove(place)
+	}
+}

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/lfu_test.go

@@ -0,0 +1,68 @@
+package lfu
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestLFU(t *testing.T) {
+	c := New()
+	c.Set("a", "a")
+	if v := c.Get("a"); v != "a" {
+		t.Errorf("Value was not saved: %v != 'a'", v)
+	}
+	if l := c.Len(); l != 1 {
+		t.Errorf("Length was not updated: %v != 1", l)
+	}
+
+	c.Set("b", "b")
+	if v := c.Get("b"); v != "b" {
+		t.Errorf("Value was not saved: %v != 'b'", v)
+	}
+	if l := c.Len(); l != 2 {
+		t.Errorf("Length was not updated: %v != 2", l)
+	}
+
+	c.Get("a")
+	evicted := c.Evict(1)
+	if v := c.Get("a"); v != "a" {
+		t.Errorf("Value was improperly evicted: %v != 'a'", v)
+	}
+	if v := c.Get("b"); v != nil {
+		t.Errorf("Value was not evicted: %v", v)
+	}
+	if l := c.Len(); l != 1 {
+		t.Errorf("Length was not updated: %v != 1", l)
+	}
+	if evicted != 1 {
+		t.Errorf("Number of evicted items is wrong: %v != 1", evicted)
+	}
+}
+
+func TestBoundsMgmt(t *testing.T) {
+	c := New()
+	c.UpperBound = 10
+	c.LowerBound = 5
+
+	for i := 0; i < 100; i++ {
+		c.Set(fmt.Sprintf("%v", i), i)
+	}
+	if c.Len() > 10 {
+		t.Errorf("Bounds management failed to evict properly: %v", c.Len())
+	}
+}
+
+func TestEviction(t *testing.T) {
+	ch := make(chan Eviction, 1)
+
+	c := New()
+	c.EvictionChannel = ch
+	c.Set("a", "b")
+	c.Evict(1)
+
+	ev := <-ch
+
+	if ev.Key != "a" || ev.Value.(string) != "b" {
+		t.Error("Incorrect item")
+	}
+}

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/lz4-example/main.go

@@ -72,10 +72,18 @@ func main() {
 
 	if *decompress {
 		data, _ = ioutil.ReadAll(input)
-		data, _ = lz4.Decode(nil, data)
+		data, err = lz4.Decode(nil, data)
+		if err != nil {
+			fmt.Println("Failed to decode:", err)
+			return
+		}
 	} else {
 		data, _ = ioutil.ReadAll(input)
-		data, _ = lz4.Encode(nil, data)
+		data, err = lz4.Encode(nil, data)
+		if err != nil {
+			fmt.Println("Failed to encode:", err)
+			return
+		}
 	}
 
 	err = ioutil.WriteFile(args[1], data, 0644)

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/writer.go

@@ -121,7 +121,7 @@ func Encode(dst, src []byte) ([]byte, error) {
 	)
 
 	for {
-		if int(e.pos)+4 >= len(e.src) {
+		if int(e.pos)+12 >= len(e.src) {
 			e.writeLiterals(uint32(len(e.src))-e.anchor, 0, e.anchor)
 			return e.dst[:e.dpos], nil
 		}
@@ -158,7 +158,7 @@ func Encode(dst, src []byte) ([]byte, error) {
 		ref += minMatch
 		e.anchor = e.pos
 
-		for int(e.pos) < len(e.src) && e.src[e.pos] == e.src[ref] {
+		for int(e.pos) < len(e.src)-5 && e.src[e.pos] == e.src[ref] {
 			e.pos++
 			ref++
 		}

Godeps/_workspace/src/github.com/calmh/logger/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2013 Jakob Borg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+- The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Godeps/_workspace/src/github.com/calmh/logger/README.md

@@ -0,0 +1,15 @@
+logger
+======
+
+A small wrapper around `log` to provide log levels.
+
+Documentation
+-------------
+
+http://godoc.org/github.com/calmh/logger
+
+License
+-------
+
+MIT
+

Godeps/_workspace/src/github.com/calmh/logger/logger.go

@@ -1,6 +1,5 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
 
 // Package logger implements a standardized logger with callback functionality
 package logger
@@ -24,6 +23,7 @@ const (
 	NumLevels
 )
 
+// A MessageHandler is called with the log level and message text.
 type MessageHandler func(l LogLevel, msg string)
 
 type Logger struct {
@@ -32,6 +32,7 @@ type Logger struct {
 	mut      sync.Mutex
 }
 
+// The default logger logs to standard output with a time prefix.
 var DefaultLogger = New()
 
 func New() *Logger {
@@ -40,16 +41,20 @@ func New() *Logger {
 	}
 }
 
+// AddHandler registers a new MessageHandler to receive messages with the
+// specified log level or above.
 func (l *Logger) AddHandler(level LogLevel, h MessageHandler) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
 	l.handlers[level] = append(l.handlers[level], h)
 }
 
+// See log.SetFlags
 func (l *Logger) SetFlags(flag int) {
 	l.logger.SetFlags(flag)
 }
 
+// See log.SetPrefix
 func (l *Logger) SetPrefix(prefix string) {
 	l.logger.SetPrefix(prefix)
 }
@@ -60,6 +65,7 @@ func (l *Logger) callHandlers(level LogLevel, s string) {
 	}
 }
 
+// Debugln logs a line with a DEBUG prefix.
 func (l *Logger) Debugln(vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -68,6 +74,7 @@ func (l *Logger) Debugln(vals ...interface{}) {
 	l.callHandlers(LevelDebug, s)
 }
 
+// Debugf logs a formatted line with a DEBUG prefix.
 func (l *Logger) Debugf(format string, vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -75,6 +82,8 @@ func (l *Logger) Debugf(format string, vals ...interface{}) {
 	l.logger.Output(2, "DEBUG: "+s)
 	l.callHandlers(LevelDebug, s)
 }
+
+// Infoln logs a line with an INFO prefix.
 func (l *Logger) Infoln(vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -83,6 +92,7 @@ func (l *Logger) Infoln(vals ...interface{}) {
 	l.callHandlers(LevelInfo, s)
 }
 
+// Infof logs a formatted line with an INFO prefix.
 func (l *Logger) Infof(format string, vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -91,6 +101,7 @@ func (l *Logger) Infof(format string, vals ...interface{}) {
 	l.callHandlers(LevelInfo, s)
 }
 
+// Okln logs a line with an OK prefix.
 func (l *Logger) Okln(vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -99,6 +110,7 @@ func (l *Logger) Okln(vals ...interface{}) {
 	l.callHandlers(LevelOK, s)
 }
 
+// Okf logs a formatted line with an OK prefix.
 func (l *Logger) Okf(format string, vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -107,6 +119,7 @@ func (l *Logger) Okf(format string, vals ...interface{}) {
 	l.callHandlers(LevelOK, s)
 }
 
+// Warnln logs a formatted line with a WARNING prefix.
 func (l *Logger) Warnln(vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -115,6 +128,7 @@ func (l *Logger) Warnln(vals ...interface{}) {
 	l.callHandlers(LevelWarn, s)
 }
 
+// Warnf logs a formatted line with a WARNING prefix.
 func (l *Logger) Warnf(format string, vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -123,6 +137,8 @@ func (l *Logger) Warnf(format string, vals ...interface{}) {
 	l.callHandlers(LevelWarn, s)
 }
 
+// Fatalln logs a line with a FATAL prefix and exits the process with exit
+// code 1.
 func (l *Logger) Fatalln(vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -132,6 +148,8 @@ func (l *Logger) Fatalln(vals ...interface{}) {
 	os.Exit(1)
 }
 
+// Fatalf logs a formatted line with a FATAL prefix and exits the process with
+// exit code 1.
 func (l *Logger) Fatalf(format string, vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -140,14 +158,3 @@ func (l *Logger) Fatalf(format string, vals ...interface{}) {
 	l.callHandlers(LevelFatal, s)
 	os.Exit(1)
 }
-
-func (l *Logger) FatalErr(err error) {
-	if err != nil {
-		l.mut.Lock()
-		defer l.mut.Unlock()
-		l.logger.SetFlags(l.logger.Flags() | log.Lshortfile)
-		l.logger.Output(2, "FATAL: "+err.Error())
-		l.callHandlers(LevelFatal, err.Error())
-		os.Exit(1)
-	}
-}

Godeps/_workspace/src/github.com/calmh/logger/logger_test.go

@@ -1,6 +1,5 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
 
 package logger
 

Godeps/_workspace/src/github.com/calmh/osext/osext_procfs.go

@@ -2,12 +2,13 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build linux netbsd openbsd
+// +build linux netbsd openbsd solaris
 
 package osext
 
 import (
 	"errors"
+	"fmt"
 	"os"
 	"runtime"
 )
@@ -20,6 +21,8 @@ func executable() (string, error) {
 		return os.Readlink("/proc/curproc/exe")
 	case "openbsd":
 		return os.Readlink("/proc/curproc/file")
+	case "solaris":
+		return os.Readlink(fmt.Sprintf("/proc/%d/path/a.out", os.Getpid()))
 	}
 	return "", errors.New("ExecPath not implemented for " + runtime.GOOS)
 }

Godeps/_workspace/src/github.com/calmh/xdr/bench_test.go

@@ -7,6 +7,8 @@ import (
 	"io"
 	"io/ioutil"
 	"testing"
+
+	"github.com/calmh/xdr"
 )
 
 type XDRBenchStruct struct {
@@ -31,11 +33,15 @@ var s = XDRBenchStruct{
 	S0:  "Hello World! String one.",
 	S1:  "Hello World! String two.",
 }
-var e = s.MarshalXDR()
+var e []byte
+
+func init() {
+	e, _ = s.MarshalXDR()
+}
 
 func BenchmarkThisMarshal(b *testing.B) {
 	for i := 0; i < b.N; i++ {
-		res = s.MarshalXDR()
+		res, _ = s.MarshalXDR()
 	}
 }
 
@@ -58,6 +64,16 @@ func BenchmarkThisEncode(b *testing.B) {
 	}
 }
 
+func BenchmarkThisEncoder(b *testing.B) {
+	w := xdr.NewWriter(ioutil.Discard)
+	for i := 0; i < b.N; i++ {
+		_, err := s.encodeXDR(w)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 type repeatReader struct {
 	data []byte
 }
@@ -86,3 +102,16 @@ func BenchmarkThisDecode(b *testing.B) {
 		rr.Reset(e)
 	}
 }
+
+func BenchmarkThisDecoder(b *testing.B) {
+	rr := &repeatReader{e}
+	r := xdr.NewReader(rr)
+	var t XDRBenchStruct
+	for i := 0; i < b.N; i++ {
+		err := t.decodeXDR(r)
+		if err != nil {
+			b.Fatal(err)
+		}
+		rr.Reset(e)
+	}
+}

Godeps/_workspace/src/github.com/calmh/xdr/bench_xdr_test.go

@@ -72,15 +72,23 @@ func (o XDRBenchStruct) EncodeXDR(w io.Writer) (int, error) {
 	return o.encodeXDR(xw)
 }
 
-func (o XDRBenchStruct) MarshalXDR() []byte {
+func (o XDRBenchStruct) MarshalXDR() ([]byte, error) {
 	return o.AppendXDR(make([]byte, 0, 128))
 }
 
-func (o XDRBenchStruct) AppendXDR(bs []byte) []byte {
+func (o XDRBenchStruct) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o XDRBenchStruct) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	o.encodeXDR(xw)
-	return []byte(aw)
+	_, err := o.encodeXDR(xw)
+	return []byte(aw), err
 }
 
 func (o XDRBenchStruct) encodeXDR(xw *xdr.Writer) (int, error) {
@@ -88,13 +96,13 @@ func (o XDRBenchStruct) encodeXDR(xw *xdr.Writer) (int, error) {
 	xw.WriteUint32(o.I2)
 	xw.WriteUint16(o.I3)
 	xw.WriteUint8(o.I4)
-	if len(o.Bs0) > 128 {
-		return xw.Tot(), xdr.ErrElementSizeExceeded
+	if l := len(o.Bs0); l > 128 {
+		return xw.Tot(), xdr.ElementSizeExceeded("Bs0", l, 128)
 	}
 	xw.WriteBytes(o.Bs0)
 	xw.WriteBytes(o.Bs1)
-	if len(o.S0) > 128 {
-		return xw.Tot(), xdr.ErrElementSizeExceeded
+	if l := len(o.S0); l > 128 {
+		return xw.Tot(), xdr.ElementSizeExceeded("S0", l, 128)
 	}
 	xw.WriteString(o.S0)
 	xw.WriteString(o.S1)
@@ -150,15 +158,23 @@ func (o repeatReader) EncodeXDR(w io.Writer) (int, error) {
 	return o.encodeXDR(xw)
 }
 
-func (o repeatReader) MarshalXDR() []byte {
+func (o repeatReader) MarshalXDR() ([]byte, error) {
 	return o.AppendXDR(make([]byte, 0, 128))
 }
 
-func (o repeatReader) AppendXDR(bs []byte) []byte {
+func (o repeatReader) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o repeatReader) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	o.encodeXDR(xw)
-	return []byte(aw)
+	_, err := o.encodeXDR(xw)
+	return []byte(aw), err
 }
 
 func (o repeatReader) encodeXDR(xw *xdr.Writer) (int, error) {

Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go

@@ -11,6 +11,8 @@ import (
 	"go/format"
 	"go/parser"
 	"go/token"
+	"io"
+	"log"
 	"os"
 	"regexp"
 	"strconv"
@@ -53,15 +55,23 @@ func (o {{.TypeName}}) EncodeXDR(w io.Writer) (int, error) {
 	return o.encodeXDR(xw)
 }//+n
 
-func (o {{.TypeName}}) MarshalXDR() []byte {
+func (o {{.TypeName}}) MarshalXDR() ([]byte, error) {
 	return o.AppendXDR(make([]byte, 0, 128))
 }//+n
 
-func (o {{.TypeName}}) AppendXDR(bs []byte) []byte {
+func (o {{.TypeName}}) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}//+n
+
+func (o {{.TypeName}}) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	o.encodeXDR(xw)
-	return []byte(aw)
+	_, err := o.encodeXDR(xw)
+	return []byte(aw), err
 }//+n
 
 func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
@@ -71,8 +81,8 @@ func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
 				xw.Write{{$fieldInfo.Encoder}}({{$fieldInfo.Convert}}(o.{{$fieldInfo.Name}}))
 			{{else if $fieldInfo.IsBasic}}
 				{{if ge $fieldInfo.Max 1}}
-					if len(o.{{$fieldInfo.Name}}) > {{$fieldInfo.Max}} {
-						return xw.Tot(), xdr.ErrElementSizeExceeded
+					if l := len(o.{{$fieldInfo.Name}}); l > {{$fieldInfo.Max}} {
+						return xw.Tot(), xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", l, {{$fieldInfo.Max}})
 					}
 				{{end}}
 				xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}})
@@ -84,8 +94,8 @@ func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
 			{{end}}
 		{{else}}
 			{{if ge $fieldInfo.Max 1}}
-				if len(o.{{$fieldInfo.Name}}) > {{$fieldInfo.Max}} {
-					return xw.Tot(), xdr.ErrElementSizeExceeded
+				if l := len(o.{{$fieldInfo.Name}}); l > {{$fieldInfo.Max}} {
+					return xw.Tot(), xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", l, {{$fieldInfo.Max}})
 				}
 			{{end}}
 			xw.WriteUint32(uint32(len(o.{{$fieldInfo.Name}})))
@@ -135,7 +145,7 @@ func (o *{{.TypeName}}) decodeXDR(xr *xdr.Reader) error {
 			_{{$fieldInfo.Name}}Size := int(xr.ReadUint32())
 			{{if ge $fieldInfo.Max 1}}
 				if _{{$fieldInfo.Name}}Size > {{$fieldInfo.Max}} {
-					return xdr.ErrElementSizeExceeded
+					return xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", _{{$fieldInfo.Name}}Size, {{$fieldInfo.Max}})
 				}
 			{{end}}
 			o.{{$fieldInfo.Name}} = make([]{{$fieldInfo.FieldType}}, _{{$fieldInfo.Name}}Size)
@@ -261,7 +271,7 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 	return fs
 }
 
-func generateCode(s structInfo) {
+func generateCode(output io.Writer, s structInfo) {
 	name := s.Name
 	fs := s.Fields
 
@@ -278,7 +288,7 @@ func generateCode(s structInfo) {
 	if err != nil {
 		panic(err)
 	}
-	fmt.Println(string(bs))
+	fmt.Fprintln(output, string(bs))
 }
 
 func uncamelize(s string) string {
@@ -287,16 +297,16 @@ func uncamelize(s string) string {
 	})
 }
 
-func generateDiagram(s structInfo) {
+func generateDiagram(output io.Writer, s structInfo) {
 	sn := s.Name
 	fs := s.Fields
 
-	fmt.Println(sn + " Structure:")
-	fmt.Println()
-	fmt.Println(" 0                   1                   2                   3")
-	fmt.Println(" 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1")
+	fmt.Fprintln(output, sn+" Structure:")
+	fmt.Fprintln(output)
+	fmt.Fprintln(output, " 0                   1                   2                   3")
+	fmt.Fprintln(output, " 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1")
 	line := "+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+"
-	fmt.Println(line)
+	fmt.Fprintln(output, line)
 
 	for _, f := range fs {
 		tn := f.FieldType
@@ -304,52 +314,52 @@ func generateDiagram(s structInfo) {
 		name := uncamelize(f.Name)
 
 		if sl {
-			fmt.Printf("| %s |\n", center("Number of "+name, 61))
-			fmt.Println(line)
+			fmt.Fprintf(output, "| %s |\n", center("Number of "+name, 61))
+			fmt.Fprintln(output, line)
 		}
 		switch tn {
 		case "bool":
-			fmt.Printf("| %s |V|\n", center(name+" (V=0 or 1)", 59))
-			fmt.Println(line)
+			fmt.Fprintf(output, "| %s |V|\n", center(name+" (V=0 or 1)", 59))
+			fmt.Fprintln(output, line)
 		case "uint16":
-			fmt.Printf("| %s | %s |\n", center("0x0000", 29), center(name, 29))
-			fmt.Println(line)
+			fmt.Fprintf(output, "| %s | %s |\n", center("0x0000", 29), center(name, 29))
+			fmt.Fprintln(output, line)
 		case "uint32":
-			fmt.Printf("| %s |\n", center(name, 61))
-			fmt.Println(line)
+			fmt.Fprintf(output, "| %s |\n", center(name, 61))
+			fmt.Fprintln(output, line)
 		case "int64", "uint64":
-			fmt.Printf("| %-61s |\n", "")
-			fmt.Printf("+ %s +\n", center(name+" (64 bits)", 61))
-			fmt.Printf("| %-61s |\n", "")
-			fmt.Println(line)
+			fmt.Fprintf(output, "| %-61s |\n", "")
+			fmt.Fprintf(output, "+ %s +\n", center(name+" (64 bits)", 61))
+			fmt.Fprintf(output, "| %-61s |\n", "")
+			fmt.Fprintln(output, line)
 		case "string", "byte": // XXX We assume slice of byte!
-			fmt.Printf("| %s |\n", center("Length of "+name, 61))
-			fmt.Println(line)
-			fmt.Printf("/ %61s /\n", "")
-			fmt.Printf("\\ %s \\\n", center(name+" (variable length)", 61))
-			fmt.Printf("/ %61s /\n", "")
-			fmt.Println(line)
+			fmt.Fprintf(output, "| %s |\n", center("Length of "+name, 61))
+			fmt.Fprintln(output, line)
+			fmt.Fprintf(output, "/ %61s /\n", "")
+			fmt.Fprintf(output, "\\ %s \\\n", center(name+" (variable length)", 61))
+			fmt.Fprintf(output, "/ %61s /\n", "")
+			fmt.Fprintln(output, line)
 		default:
 			if sl {
 				tn = "Zero or more " + tn + " Structures"
-				fmt.Printf("/ %s /\n", center("", 61))
-				fmt.Printf("\\ %s \\\n", center(tn, 61))
-				fmt.Printf("/ %s /\n", center("", 61))
+				fmt.Fprintf(output, "/ %s /\n", center("", 61))
+				fmt.Fprintf(output, "\\ %s \\\n", center(tn, 61))
+				fmt.Fprintf(output, "/ %s /\n", center("", 61))
 			} else {
-				fmt.Printf("| %s |\n", center(tn, 61))
+				fmt.Fprintf(output, "| %s |\n", center(tn, 61))
 			}
-			fmt.Println(line)
+			fmt.Fprintln(output, line)
 		}
 	}
-	fmt.Println()
-	fmt.Println()
+	fmt.Fprintln(output)
+	fmt.Fprintln(output)
 }
 
-func generateXdr(s structInfo) {
+func generateXdr(output io.Writer, s structInfo) {
 	sn := s.Name
 	fs := s.Fields
 
-	fmt.Printf("struct %s {\n", sn)
+	fmt.Fprintf(output, "struct %s {\n", sn)
 
 	for _, f := range fs {
 		tn := f.FieldType
@@ -365,21 +375,21 @@ func generateXdr(s structInfo) {
 
 		switch tn {
 		case "uint16", "uint32":
-			fmt.Printf("\tunsigned int %s%s;\n", fn, suf)
+			fmt.Fprintf(output, "\tunsigned int %s%s;\n", fn, suf)
 		case "int64":
-			fmt.Printf("\thyper %s%s;\n", fn, suf)
+			fmt.Fprintf(output, "\thyper %s%s;\n", fn, suf)
 		case "uint64":
-			fmt.Printf("\tunsigned hyper %s%s;\n", fn, suf)
+			fmt.Fprintf(output, "\tunsigned hyper %s%s;\n", fn, suf)
 		case "string":
-			fmt.Printf("\tstring %s<%s>;\n", fn, l)
+			fmt.Fprintf(output, "\tstring %s<%s>;\n", fn, l)
 		case "byte":
-			fmt.Printf("\topaque %s<%s>;\n", fn, l)
+			fmt.Fprintf(output, "\topaque %s<%s>;\n", fn, l)
 		default:
-			fmt.Printf("\t%s %s%s;\n", tn, fn, suf)
+			fmt.Fprintf(output, "\t%s %s%s;\n", tn, fn, suf)
 		}
 	}
-	fmt.Println("}")
-	fmt.Println()
+	fmt.Fprintln(output, "}")
+	fmt.Fprintln(output)
 }
 
 func center(s string, w int) string {
@@ -410,25 +420,35 @@ func inspector(structs *[]structInfo) func(ast.Node) bool {
 }
 
 func main() {
+	outputFile := flag.String("o", "", "Output file, blank for stdout")
 	flag.Parse()
 	fname := flag.Arg(0)
 
 	fset := token.NewFileSet()
 	f, err := parser.ParseFile(fset, fname, nil, parser.ParseComments)
 	if err != nil {
-		panic(err)
+		log.Fatal(err)
 	}
 
 	var structs []structInfo
 	i := inspector(&structs)
 	ast.Inspect(f, i)
 
-	headerTpl.Execute(os.Stdout, map[string]string{"Package": f.Name.Name})
+	var output io.Writer = os.Stdout
+	if *outputFile != "" {
+		fd, err := os.Create(*outputFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		output = fd
+	}
+
+	headerTpl.Execute(output, map[string]string{"Package": f.Name.Name})
 	for _, s := range structs {
-		fmt.Printf("\n/*\n\n")
-		generateDiagram(s)
-		generateXdr(s)
-		fmt.Printf("*/\n")
-		generateCode(s)
+		fmt.Fprintf(output, "\n/*\n\n")
+		generateDiagram(output, s)
+		generateXdr(output, s)
+		fmt.Fprintf(output, "*/\n")
+		generateCode(output, s)
 	}
 }

Godeps/_workspace/src/github.com/calmh/xdr/encdec_test.go

@@ -24,9 +24,10 @@ type TestStruct struct {
 	UI32 uint32
 	I64  int64
 	UI64 uint64
-	BS   []byte
-	S    string
+	BS   []byte // max:1024
+	S    string // max:1024
 	C    Opaque
+	SS   []string // max:1024
 }
 
 type Opaque [32]byte
@@ -49,9 +50,12 @@ func (Opaque) Generate(rand *rand.Rand, size int) reflect.Value {
 
 func TestEncDec(t *testing.T) {
 	fn := func(t0 TestStruct) bool {
-		bs := t0.MarshalXDR()
+		bs, err := t0.MarshalXDR()
+		if err != nil {
+			t.Fatal(err)
+		}
 		var t1 TestStruct
-		err := t1.UnmarshalXDR(bs)
+		err = t1.UnmarshalXDR(bs)
 		if err != nil {
 			t.Fatal(err)
 		}

Godeps/_workspace/src/github.com/calmh/xdr/encdec_xdr_test.go

@@ -54,6 +54,14 @@ TestStruct Structure:
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |                            Opaque                             |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Number of SS                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of SS                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     SS (variable length)                      \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 
 struct TestStruct {
@@ -66,9 +74,10 @@ struct TestStruct {
 	unsigned int UI32;
 	hyper I64;
 	unsigned hyper UI64;
-	opaque BS<>;
-	string S<>;
+	opaque BS<1024>;
+	string S<1024>;
 	Opaque C;
+	string SS<1024>;
 }
 
 */
@@ -78,15 +87,23 @@ func (o TestStruct) EncodeXDR(w io.Writer) (int, error) {
 	return o.encodeXDR(xw)
 }
 
-func (o TestStruct) MarshalXDR() []byte {
+func (o TestStruct) MarshalXDR() ([]byte, error) {
 	return o.AppendXDR(make([]byte, 0, 128))
 }
 
-func (o TestStruct) AppendXDR(bs []byte) []byte {
+func (o TestStruct) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o TestStruct) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	o.encodeXDR(xw)
-	return []byte(aw)
+	_, err := o.encodeXDR(xw)
+	return []byte(aw), err
 }
 
 func (o TestStruct) encodeXDR(xw *xdr.Writer) (int, error) {
@@ -99,12 +116,25 @@ func (o TestStruct) encodeXDR(xw *xdr.Writer) (int, error) {
 	xw.WriteUint32(o.UI32)
 	xw.WriteUint64(uint64(o.I64))
 	xw.WriteUint64(o.UI64)
+	if l := len(o.BS); l > 1024 {
+		return xw.Tot(), xdr.ElementSizeExceeded("BS", l, 1024)
+	}
 	xw.WriteBytes(o.BS)
+	if l := len(o.S); l > 1024 {
+		return xw.Tot(), xdr.ElementSizeExceeded("S", l, 1024)
+	}
 	xw.WriteString(o.S)
 	_, err := o.C.encodeXDR(xw)
 	if err != nil {
 		return xw.Tot(), err
 	}
+	if l := len(o.SS); l > 1024 {
+		return xw.Tot(), xdr.ElementSizeExceeded("SS", l, 1024)
+	}
+	xw.WriteUint32(uint32(len(o.SS)))
+	for i := range o.SS {
+		xw.WriteString(o.SS[i])
+	}
 	return xw.Tot(), xw.Error()
 }
 
@@ -129,8 +159,16 @@ func (o *TestStruct) decodeXDR(xr *xdr.Reader) error {
 	o.UI32 = xr.ReadUint32()
 	o.I64 = int64(xr.ReadUint64())
 	o.UI64 = xr.ReadUint64()
-	o.BS = xr.ReadBytes()
-	o.S = xr.ReadString()
+	o.BS = xr.ReadBytesMax(1024)
+	o.S = xr.ReadStringMax(1024)
 	(&o.C).decodeXDR(xr)
+	_SSSize := int(xr.ReadUint32())
+	if _SSSize > 1024 {
+		return xdr.ElementSizeExceeded("SS", _SSSize, 1024)
+	}
+	o.SS = make([]string, _SSSize)
+	for i := range o.SS {
+		o.SS[i] = xr.ReadString()
+	}
 	return xr.Error()
 }

Godeps/_workspace/src/github.com/calmh/xdr/reader.go

@@ -5,17 +5,16 @@
 package xdr
 
 import (
-	"errors"
+	"fmt"
 	"io"
+	"reflect"
+	"unsafe"
 )
 
-var ErrElementSizeExceeded = errors.New("element size exceeded")
-
 type Reader struct {
 	r   io.Reader
 	err error
 	b   [8]byte
-	sb  []byte
 }
 
 func NewReader(r io.Reader) *Reader {
@@ -35,23 +34,17 @@ func (r *Reader) ReadRaw(bs []byte) (int, error) {
 }
 
 func (r *Reader) ReadString() string {
-	if r.sb == nil {
-		r.sb = make([]byte, 64)
-	} else {
-		r.sb = r.sb[:cap(r.sb)]
-	}
-	r.sb = r.ReadBytesInto(r.sb)
-	return string(r.sb)
+	return r.ReadStringMax(0)
 }
 
 func (r *Reader) ReadStringMax(max int) string {
-	if r.sb == nil {
-		r.sb = make([]byte, 64)
-	} else {
-		r.sb = r.sb[:cap(r.sb)]
+	buf := r.ReadBytesMaxInto(max, nil)
+	bh := (*reflect.SliceHeader)(unsafe.Pointer(&buf))
+	sh := reflect.StringHeader{
+		Data: bh.Data,
+		Len:  bh.Len,
 	}
-	r.sb = r.ReadBytesMaxInto(max, r.sb)
-	return string(r.sb)
+	return *((*string)(unsafe.Pointer(&sh)))
 }
 
 func (r *Reader) ReadBytes() []byte {
@@ -76,14 +69,14 @@ func (r *Reader) ReadBytesMaxInto(max int, dst []byte) []byte {
 		return nil
 	}
 	if max > 0 && l > max {
-		r.err = ErrElementSizeExceeded
+		r.err = ElementSizeExceeded("bytes field", l, max)
 		return nil
 	}
 
-	if l+pad(l) > len(dst) {
-		dst = make([]byte, l+pad(l))
+	if fullLen := l + pad(l); fullLen > len(dst) {
+		dst = make([]byte, fullLen)
 	} else {
-		dst = dst[:l+pad(l)]
+		dst = dst[:fullLen]
 	}
 
 	var n int
@@ -167,3 +160,7 @@ func (r *Reader) Error() error {
 	}
 	return XDRError{"read", r.err}
 }
+
+func ElementSizeExceeded(field string, size, limit int) error {
+	return fmt.Errorf("%s exceeds size limit; %d > %d", field, size, limit)
+}

Godeps/_workspace/src/github.com/calmh/xdr/reader_ipdr.go

@@ -22,7 +22,7 @@ func (r *Reader) ReadUint8() uint8 {
 	}
 
 	if debug {
-		dl.Printf("rd uint8=%d (0x%08x)", r.b[0], r.b[0])
+		dl.Printf("rd uint8=%d (0x%02x)", r.b[0], r.b[0])
 	}
 	return r.b[0]
 }
@@ -43,7 +43,7 @@ func (r *Reader) ReadUint16() uint16 {
 	v := uint16(r.b[1]) | uint16(r.b[0])<<8
 
 	if debug {
-		dl.Printf("rd uint16=%d (0x%08x)", v, v)
+		dl.Printf("rd uint16=%d (0x%04x)", v, v)
 	}
 	return v
 }

Godeps/_workspace/src/github.com/calmh/xdr/writer.go

@@ -3,7 +3,11 @@
 
 package xdr
 
-import "io"
+import (
+	"io"
+	"reflect"
+	"unsafe"
+)
 
 var padBytes = []byte{0, 0, 0}
 
@@ -38,7 +42,13 @@ func (w *Writer) WriteRaw(bs []byte) (int, error) {
 }
 
 func (w *Writer) WriteString(s string) (int, error) {
-	return w.WriteBytes([]byte(s))
+	sh := *((*reflect.StringHeader)(unsafe.Pointer(&s)))
+	bh := reflect.SliceHeader{
+		Data: sh.Data,
+		Len:  sh.Len,
+		Cap:  sh.Len,
+	}
+	return w.WriteBytes(*(*[]byte)(unsafe.Pointer(&bh)))
 }
 
 func (w *Writer) WriteBytes(bs []byte) (int, error) {

Godeps/_workspace/src/github.com/calmh/xdr/xdr_test.go

@@ -5,6 +5,7 @@ package xdr
 
 import (
 	"bytes"
+	"strings"
 	"testing"
 	"testing/quick"
 )
@@ -60,7 +61,7 @@ func TestReadBytesMaxInto(t *testing.T) {
 				if read := len(bs); read != tot {
 					t.Errorf("Incorrect read bytes, wrote=%d, buf=%d, max=%d, read=%d", tot, tot+diff, max, read)
 				}
-			} else if r.err != ErrElementSizeExceeded {
+			} else if !strings.Contains(r.err.Error(), "exceeds size") {
 				t.Errorf("Unexpected non-ErrElementSizeExceeded error for wrote=%d, max=%d: %v", tot, max, r.err)
 			}
 		}
@@ -84,7 +85,7 @@ func TestReadStringMax(t *testing.T) {
 				if read != tot {
 					t.Errorf("Incorrect read bytes, wrote=%d, max=%d, read=%d", tot, max, read)
 				}
-			} else if r.err != ErrElementSizeExceeded {
+			} else if !strings.Contains(r.err.Error(), "exceeds size") {
 				t.Errorf("Unexpected non-ErrElementSizeExceeded error for wrote=%d, max=%d, read=%d: %v", tot, max, read, r.err)
 			}
 		}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch.go

@@ -8,65 +8,84 @@ package leveldb
 
 import (
 	"encoding/binary"
-	"errors"
+	"fmt"
 
+	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/memdb"
 )
 
-var (
-	errBatchTooShort  = errors.New("leveldb: batch is too short")
-	errBatchBadRecord = errors.New("leveldb: bad record in batch")
+type ErrBatchCorrupted struct {
+	Reason string
+}
+
+func (e *ErrBatchCorrupted) Error() string {
+	return fmt.Sprintf("leveldb: batch corrupted: %s", e.Reason)
+}
+
+func newErrBatchCorrupted(reason string) error {
+	return errors.NewErrCorrupted(nil, &ErrBatchCorrupted{reason})
+}
+
+const (
+	batchHdrLen  = 8 + 4
+	batchGrowRec = 3000
 )
 
-const kBatchHdrLen = 8 + 4
-
-type batchReplay interface {
-	put(key, value []byte, seq uint64)
-	delete(key []byte, seq uint64)
+type BatchReplay interface {
+	Put(key, value []byte)
+	Delete(key []byte)
 }
 
 // Batch is a write batch.
 type Batch struct {
-	buf        []byte
+	data       []byte
 	rLen, bLen int
 	seq        uint64
 	sync       bool
 }
 
 func (b *Batch) grow(n int) {
-	off := len(b.buf)
+	off := len(b.data)
 	if off == 0 {
-		// include headers
-		off = kBatchHdrLen
-		n += off
+		off = batchHdrLen
+		if b.data != nil {
+			b.data = b.data[:off]
+		}
 	}
-	if cap(b.buf)-off >= n {
-		return
+	if cap(b.data)-off < n {
+		if b.data == nil {
+			b.data = make([]byte, off, off+n)
+		} else {
+			odata := b.data
+			div := 1
+			if b.rLen > batchGrowRec {
+				div = b.rLen / batchGrowRec
+			}
+			b.data = make([]byte, off, off+n+(off-batchHdrLen)/div)
+			copy(b.data, odata)
+		}
 	}
-	buf := make([]byte, 2*cap(b.buf)+n)
-	copy(buf, b.buf)
-	b.buf = buf[:off]
 }
 
-func (b *Batch) appendRec(t vType, key, value []byte) {
+func (b *Batch) appendRec(kt kType, key, value []byte) {
 	n := 1 + binary.MaxVarintLen32 + len(key)
-	if t == tVal {
+	if kt == ktVal {
 		n += binary.MaxVarintLen32 + len(value)
 	}
 	b.grow(n)
-	off := len(b.buf)
-	buf := b.buf[:off+n]
-	buf[off] = byte(t)
+	off := len(b.data)
+	data := b.data[:off+n]
+	data[off] = byte(kt)
 	off += 1
-	off += binary.PutUvarint(buf[off:], uint64(len(key)))
-	copy(buf[off:], key)
+	off += binary.PutUvarint(data[off:], uint64(len(key)))
+	copy(data[off:], key)
 	off += len(key)
-	if t == tVal {
-		off += binary.PutUvarint(buf[off:], uint64(len(value)))
-		copy(buf[off:], value)
+	if kt == ktVal {
+		off += binary.PutUvarint(data[off:], uint64(len(value)))
+		copy(data[off:], value)
 		off += len(value)
 	}
-	b.buf = buf[:off]
+	b.data = data[:off]
 	b.rLen++
 	//  Include 8-byte ikey header
 	b.bLen += len(key) + len(value) + 8
@@ -75,18 +94,51 @@ func (b *Batch) appendRec(t vType, key, value []byte) {
 // Put appends 'put operation' of the given key/value pair to the batch.
 // It is safe to modify the contents of the argument after Put returns.
 func (b *Batch) Put(key, value []byte) {
-	b.appendRec(tVal, key, value)
+	b.appendRec(ktVal, key, value)
 }
 
 // Delete appends 'delete operation' of the given key to the batch.
 // It is safe to modify the contents of the argument after Delete returns.
 func (b *Batch) Delete(key []byte) {
-	b.appendRec(tDel, key, nil)
+	b.appendRec(ktDel, key, nil)
+}
+
+// Dump dumps batch contents. The returned slice can be loaded into the
+// batch using Load method.
+// The returned slice is not its own copy, so the contents should not be
+// modified.
+func (b *Batch) Dump() []byte {
+	return b.encode()
+}
+
+// Load loads given slice into the batch. Previous contents of the batch
+// will be discarded.
+// The given slice will not be copied and will be used as batch buffer, so
+// it is not safe to modify the contents of the slice.
+func (b *Batch) Load(data []byte) error {
+	return b.decode(0, data)
+}
+
+// Replay replays batch contents.
+func (b *Batch) Replay(r BatchReplay) error {
+	return b.decodeRec(func(i int, kt kType, key, value []byte) {
+		switch kt {
+		case ktVal:
+			r.Put(key, value)
+		case ktDel:
+			r.Delete(key)
+		}
+	})
+}
+
+// Len returns number of records in the batch.
+func (b *Batch) Len() int {
+	return b.rLen
 }
 
 // Reset resets the batch.
 func (b *Batch) Reset() {
-	b.buf = nil
+	b.data = b.data[:0]
 	b.seq = 0
 	b.rLen = 0
 	b.bLen = 0
@@ -97,24 +149,10 @@ func (b *Batch) init(sync bool) {
 	b.sync = sync
 }
 
-func (b *Batch) put(key, value []byte, seq uint64) {
-	if b.rLen == 0 {
-		b.seq = seq
-	}
-	b.Put(key, value)
-}
-
-func (b *Batch) delete(key []byte, seq uint64) {
-	if b.rLen == 0 {
-		b.seq = seq
-	}
-	b.Delete(key)
-}
-
 func (b *Batch) append(p *Batch) {
 	if p.rLen > 0 {
-		b.grow(len(p.buf) - kBatchHdrLen)
-		b.buf = append(b.buf, p.buf[kBatchHdrLen:]...)
+		b.grow(len(p.data) - batchHdrLen)
+		b.data = append(b.data, p.data[batchHdrLen:]...)
 		b.rLen += p.rLen
 	}
 	if p.sync {
@@ -122,95 +160,93 @@ func (b *Batch) append(p *Batch) {
 	}
 }
 
-func (b *Batch) len() int {
-	return b.rLen
-}
-
+// size returns sums of key/value pair length plus 8-bytes ikey.
 func (b *Batch) size() int {
 	return b.bLen
 }
 
 func (b *Batch) encode() []byte {
 	b.grow(0)
-	binary.LittleEndian.PutUint64(b.buf, b.seq)
-	binary.LittleEndian.PutUint32(b.buf[8:], uint32(b.rLen))
+	binary.LittleEndian.PutUint64(b.data, b.seq)
+	binary.LittleEndian.PutUint32(b.data[8:], uint32(b.rLen))
 
-	return b.buf
+	return b.data
 }
 
-func (b *Batch) decode(buf []byte) error {
-	if len(buf) < kBatchHdrLen {
-		return errBatchTooShort
+func (b *Batch) decode(prevSeq uint64, data []byte) error {
+	if len(data) < batchHdrLen {
+		return newErrBatchCorrupted("too short")
 	}
 
-	b.seq = binary.LittleEndian.Uint64(buf)
-	b.rLen = int(binary.LittleEndian.Uint32(buf[8:]))
+	b.seq = binary.LittleEndian.Uint64(data)
+	if b.seq < prevSeq {
+		return newErrBatchCorrupted("invalid sequence number")
+	}
+	b.rLen = int(binary.LittleEndian.Uint32(data[8:]))
+	if b.rLen < 0 {
+		return newErrBatchCorrupted("invalid records length")
+	}
 	// No need to be precise at this point, it won't be used anyway
-	b.bLen = len(buf) - kBatchHdrLen
-	b.buf = buf
+	b.bLen = len(data) - batchHdrLen
+	b.data = data
 
 	return nil
 }
 
-func (b *Batch) decodeRec(f func(i int, t vType, key, value []byte)) error {
-	off := kBatchHdrLen
+func (b *Batch) decodeRec(f func(i int, kt kType, key, value []byte)) (err error) {
+	off := batchHdrLen
 	for i := 0; i < b.rLen; i++ {
-		if off >= len(b.buf) {
-			return errors.New("leveldb: invalid batch record length")
+		if off >= len(b.data) {
+			return newErrBatchCorrupted("invalid records length")
 		}
 
-		t := vType(b.buf[off])
-		if t > tVal {
-			return errors.New("leveldb: invalid batch record type in batch")
+		kt := kType(b.data[off])
+		if kt > ktVal {
+			return newErrBatchCorrupted("bad record: invalid type")
 		}
 		off += 1
 
-		x, n := binary.Uvarint(b.buf[off:])
+		x, n := binary.Uvarint(b.data[off:])
 		off += n
-		if n <= 0 || off+int(x) > len(b.buf) {
-			return errBatchBadRecord
+		if n <= 0 || off+int(x) > len(b.data) {
+			return newErrBatchCorrupted("bad record: invalid key length")
 		}
-		key := b.buf[off : off+int(x)]
+		key := b.data[off : off+int(x)]
 		off += int(x)
-
 		var value []byte
-		if t == tVal {
-			x, n := binary.Uvarint(b.buf[off:])
+		if kt == ktVal {
+			x, n := binary.Uvarint(b.data[off:])
 			off += n
-			if n <= 0 || off+int(x) > len(b.buf) {
-				return errBatchBadRecord
+			if n <= 0 || off+int(x) > len(b.data) {
+				return newErrBatchCorrupted("bad record: invalid value length")
 			}
-			value = b.buf[off : off+int(x)]
+			value = b.data[off : off+int(x)]
 			off += int(x)
 		}
 
-		f(i, t, key, value)
+		f(i, kt, key, value)
 	}
 
 	return nil
 }
 
-func (b *Batch) replay(to batchReplay) error {
-	return b.decodeRec(func(i int, t vType, key, value []byte) {
-		switch t {
-		case tVal:
-			to.put(key, value, b.seq+uint64(i))
-		case tDel:
-			to.delete(key, b.seq+uint64(i))
-		}
-	})
-}
-
 func (b *Batch) memReplay(to *memdb.DB) error {
-	return b.decodeRec(func(i int, t vType, key, value []byte) {
-		ikey := newIKey(key, b.seq+uint64(i), t)
+	return b.decodeRec(func(i int, kt kType, key, value []byte) {
+		ikey := newIkey(key, b.seq+uint64(i), kt)
 		to.Put(ikey, value)
 	})
 }
 
+func (b *Batch) memDecodeAndReplay(prevSeq uint64, data []byte, to *memdb.DB) error {
+	if err := b.decode(prevSeq, data); err != nil {
+		return err
+	}
+	return b.memReplay(to)
+}
+
 func (b *Batch) revertMemReplay(to *memdb.DB) error {
-	return b.decodeRec(func(i int, t vType, key, value []byte) {
-		ikey := newIKey(key, b.seq+uint64(i), t)
+	return b.decodeRec(func(i int, kt kType, key, value []byte) {
+		ikey := newIkey(key, b.seq+uint64(i), kt)
 		to.Delete(ikey)
 	})
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch_test.go

@@ -15,7 +15,7 @@ import (
 )
 
 type tbRec struct {
-	t          vType
+	kt         kType
 	key, value []byte
 }
 
@@ -23,39 +23,39 @@ type testBatch struct {
 	rec []*tbRec
 }
 
-func (p *testBatch) put(key, value []byte, seq uint64) {
-	p.rec = append(p.rec, &tbRec{tVal, key, value})
+func (p *testBatch) Put(key, value []byte) {
+	p.rec = append(p.rec, &tbRec{ktVal, key, value})
 }
 
-func (p *testBatch) delete(key []byte, seq uint64) {
-	p.rec = append(p.rec, &tbRec{tDel, key, nil})
+func (p *testBatch) Delete(key []byte) {
+	p.rec = append(p.rec, &tbRec{ktDel, key, nil})
 }
 
 func compareBatch(t *testing.T, b1, b2 *Batch) {
 	if b1.seq != b2.seq {
 		t.Errorf("invalid seq number want %d, got %d", b1.seq, b2.seq)
 	}
-	if b1.len() != b2.len() {
-		t.Fatalf("invalid record length want %d, got %d", b1.len(), b2.len())
+	if b1.Len() != b2.Len() {
+		t.Fatalf("invalid record length want %d, got %d", b1.Len(), b2.Len())
 	}
 	p1, p2 := new(testBatch), new(testBatch)
-	err := b1.replay(p1)
+	err := b1.Replay(p1)
 	if err != nil {
 		t.Fatal("error when replaying batch 1: ", err)
 	}
-	err = b2.replay(p2)
+	err = b2.Replay(p2)
 	if err != nil {
 		t.Fatal("error when replaying batch 2: ", err)
 	}
 	for i := range p1.rec {
 		r1, r2 := p1.rec[i], p2.rec[i]
-		if r1.t != r2.t {
-			t.Errorf("invalid type on record '%d' want %d, got %d", i, r1.t, r2.t)
+		if r1.kt != r2.kt {
+			t.Errorf("invalid type on record '%d' want %d, got %d", i, r1.kt, r2.kt)
 		}
 		if !bytes.Equal(r1.key, r2.key) {
 			t.Errorf("invalid key on record '%d' want %s, got %s", i, string(r1.key), string(r2.key))
 		}
-		if r1.t == tVal {
+		if r1.kt == ktVal {
 			if !bytes.Equal(r1.value, r2.value) {
 				t.Errorf("invalid value on record '%d' want %s, got %s", i, string(r1.value), string(r2.value))
 			}
@@ -75,7 +75,7 @@ func TestBatch_EncodeDecode(t *testing.T) {
 	b1.Delete([]byte("k"))
 	buf := b1.encode()
 	b2 := new(Batch)
-	err := b2.decode(buf)
+	err := b2.decode(0, buf)
 	if err != nil {
 		t.Error("error when decoding batch: ", err)
 	}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench_test.go

@@ -170,7 +170,7 @@ func (p *dbBench) writes(perBatch int) {
 	b.SetBytes(116)
 }
 
-func (p *dbBench) drop() {
+func (p *dbBench) gc() {
 	p.keys, p.values = nil, nil
 	runtime.GC()
 }
@@ -249,6 +249,9 @@ func (p *dbBench) newIter() iterator.Iterator {
 }
 
 func (p *dbBench) close() {
+	if bp, err := p.db.GetProperty("leveldb.blockpool"); err == nil {
+		p.b.Log("Block pool stats: ", bp)
+	}
 	p.db.Close()
 	p.stor.Close()
 	os.RemoveAll(benchDB)
@@ -331,7 +334,7 @@ func BenchmarkDBRead(b *testing.B) {
 	p := openDBBench(b, false)
 	p.populate(b.N)
 	p.fill()
-	p.drop()
+	p.gc()
 
 	iter := p.newIter()
 	b.ResetTimer()
@@ -362,7 +365,7 @@ func BenchmarkDBReadUncompressed(b *testing.B) {
 	p := openDBBench(b, true)
 	p.populate(b.N)
 	p.fill()
-	p.drop()
+	p.gc()
 
 	iter := p.newIter()
 	b.ResetTimer()
@@ -379,7 +382,7 @@ func BenchmarkDBReadTable(b *testing.B) {
 	p.populate(b.N)
 	p.fill()
 	p.reopen()
-	p.drop()
+	p.gc()
 
 	iter := p.newIter()
 	b.ResetTimer()
@@ -395,7 +398,7 @@ func BenchmarkDBReadReverse(b *testing.B) {
 	p := openDBBench(b, false)
 	p.populate(b.N)
 	p.fill()
-	p.drop()
+	p.gc()
 
 	iter := p.newIter()
 	b.ResetTimer()
@@ -413,7 +416,7 @@ func BenchmarkDBReadReverseTable(b *testing.B) {
 	p.populate(b.N)
 	p.fill()
 	p.reopen()
-	p.drop()
+	p.gc()
 
 	iter := p.newIter()
 	b.ResetTimer()

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache.go

@@ -8,118 +8,669 @@
 package cache
 
 import (
+	"sync"
 	"sync/atomic"
+	"unsafe"
+
+	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
-// SetFunc used by Namespace.Get method to create a cache object. SetFunc
-// may return ok false, in that case the cache object will not be created.
-type SetFunc func() (ok bool, value interface{}, charge int, fin SetFin)
+// Cacher provides interface to implements a caching functionality.
+// An implementation must be goroutine-safe.
+type Cacher interface {
+	// Capacity returns cache capacity.
+	Capacity() int
 
-// SetFin will be called when corresponding cache object are released.
-type SetFin func()
-
-// DelFin will be called when corresponding cache object are released.
-// DelFin will be called after SetFin. The exist is true if the corresponding
-// cache object is actually exist in the cache tree.
-type DelFin func(exist bool)
-
-// PurgeFin will be called when corresponding cache object are released.
-// PurgeFin will be called after SetFin. If PurgeFin present DelFin will
-// not be executed but passed to the PurgeFin, it is up to the caller
-// to call it or not.
-type PurgeFin func(ns, key uint64, delfin DelFin)
-
-// Cache is a cache tree. A cache instance must be goroutine-safe.
-type Cache interface {
 	// SetCapacity sets cache capacity.
 	SetCapacity(capacity int)
 
-	// GetNamespace gets or creates a cache namespace for the given id.
-	GetNamespace(id uint64) Namespace
+	// Promote promotes the 'cache node'.
+	Promote(n *Node)
 
-	// Purge purges all cache namespaces, read Namespace.Purge method documentation.
-	Purge(fin PurgeFin)
+	// Ban evicts the 'cache node' and prevent subsequent 'promote'.
+	Ban(n *Node)
 
-	// Zap zaps all cache namespaces, read Namespace.Zap method documentation.
-	Zap(closed bool)
+	// Evict evicts the 'cache node'.
+	Evict(n *Node)
+
+	// EvictNS evicts 'cache node' with the given namespace.
+	EvictNS(ns uint64)
+
+	// EvictAll evicts all 'cache node'.
+	EvictAll()
+
+	// Close closes the 'cache tree'
+	Close() error
 }
 
-// Namespace is a cache namespace. A namespace instance must be goroutine-safe.
-type Namespace interface {
-	// Get gets cache object for the given key. The given SetFunc (if not nil) will
-	// be called if the given key does not exist.
-	// If the given key does not exist, SetFunc is nil or SetFunc return ok false, Get
-	// will return ok false.
-	Get(key uint64, setf SetFunc) (obj Object, ok bool)
+// Value is a 'cacheable object'. It may implements util.Releaser, if
+// so the the Release method will be called once object is released.
+type Value interface{}
 
-	// Get deletes cache object for the given key. If exist the cache object will
-	// be deleted later when all of its handles have been released (i.e. no one use
-	// it anymore) and the given DelFin (if not nil) will finally be  executed. If
-	// such cache object does not exist the given DelFin will be executed anyway.
-	//
-	// Delete returns true if such cache object exist.
-	Delete(key uint64, fin DelFin) bool
-
-	// Purge deletes all cache objects, read Delete method documentation.
-	Purge(fin PurgeFin)
-
-	// Zap detaches the namespace from the cache tree and delete all its cache
-	// objects. The cache objects deletion and finalizers execution are happen
-	// immediately, even if its existing handles haven't yet been released.
-	// A zapped namespace can't never be filled again.
-	// If closed is false then the Get function will always call the given SetFunc
-	// if it is not nil, but resultant of the SetFunc will not be cached.
-	Zap(closed bool)
+type CacheGetter struct {
+	Cache *Cache
+	NS    uint64
 }
 
-// Object is a cache object.
-type Object interface {
-	// Release releases the cache object. Other methods should not be called
-	// after the cache object has been released.
-	Release()
-
-	// Value returns value of the cache object.
-	Value() interface{}
+func (g *CacheGetter) Get(key uint64, setFunc func() (size int, value Value)) *Handle {
+	return g.Cache.Get(g.NS, key, setFunc)
 }
 
-// Namespace state.
-type nsState int
+// The hash tables implementation is based on:
+// "Dynamic-Sized Nonblocking Hash Tables", by Yujie Liu, Kunlong Zhang, and Michael Spear. ACM Symposium on Principles of Distributed Computing, Jul 2014.
 
 const (
-	nsEffective nsState = iota
-	nsZapped
-	nsClosed
+	mInitialSize           = 1 << 4
+	mOverflowThreshold     = 1 << 5
+	mOverflowGrowThreshold = 1 << 7
 )
 
-// Node state.
-type nodeState int
-
-const (
-	nodeEffective nodeState = iota
-	nodeEvicted
-	nodeRemoved
-)
-
-// Fake object.
-type fakeObject struct {
-	value interface{}
-	fin   func()
-	once  uint32
+type mBucket struct {
+	mu     sync.Mutex
+	node   []*Node
+	frozen bool
 }
 
-func (o *fakeObject) Value() interface{} {
-	if atomic.LoadUint32(&o.once) == 0 {
-		return o.value
+func (b *mBucket) freeze() []*Node {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	if !b.frozen {
+		b.frozen = true
+	}
+	return b.node
+}
+
+func (b *mBucket) get(r *Cache, h *mNode, hash uint32, ns, key uint64, noset bool) (done, added bool, n *Node) {
+	b.mu.Lock()
+
+	if b.frozen {
+		b.mu.Unlock()
+		return
+	}
+
+	// Scan the node.
+	for _, n := range b.node {
+		if n.hash == hash && n.ns == ns && n.key == key {
+			atomic.AddInt32(&n.ref, 1)
+			b.mu.Unlock()
+			return true, false, n
+		}
+	}
+
+	// Get only.
+	if noset {
+		b.mu.Unlock()
+		return true, false, nil
+	}
+
+	// Create node.
+	n = &Node{
+		r:    r,
+		hash: hash,
+		ns:   ns,
+		key:  key,
+		ref:  1,
+	}
+	// Add node to bucket.
+	b.node = append(b.node, n)
+	bLen := len(b.node)
+	b.mu.Unlock()
+
+	// Update counter.
+	grow := atomic.AddInt32(&r.nodes, 1) >= h.growThreshold
+	if bLen > mOverflowThreshold {
+		grow = grow || atomic.AddInt32(&h.overflow, 1) >= mOverflowGrowThreshold
+	}
+
+	// Grow.
+	if grow && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) {
+		nhLen := len(h.buckets) << 1
+		nh := &mNode{
+			buckets:         make([]unsafe.Pointer, nhLen),
+			mask:            uint32(nhLen) - 1,
+			pred:            unsafe.Pointer(h),
+			growThreshold:   int32(nhLen * mOverflowThreshold),
+			shrinkThreshold: int32(nhLen >> 1),
+		}
+		ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh))
+		if !ok {
+			panic("BUG: failed swapping head")
+		}
+		go nh.initBuckets()
+	}
+
+	return true, true, n
+}
+
+func (b *mBucket) delete(r *Cache, h *mNode, hash uint32, ns, key uint64) (done, deleted bool) {
+	b.mu.Lock()
+
+	if b.frozen {
+		b.mu.Unlock()
+		return
+	}
+
+	// Scan the node.
+	var (
+		n    *Node
+		bLen int
+	)
+	for i := range b.node {
+		n = b.node[i]
+		if n.ns == ns && n.key == key {
+			if atomic.LoadInt32(&n.ref) == 0 {
+				deleted = true
+
+				// Call releaser.
+				if n.value != nil {
+					if r, ok := n.value.(util.Releaser); ok {
+						r.Release()
+					}
+					n.value = nil
+				}
+
+				// Remove node from bucket.
+				b.node = append(b.node[:i], b.node[i+1:]...)
+				bLen = len(b.node)
+			}
+			break
+		}
+	}
+	b.mu.Unlock()
+
+	if deleted {
+		// Call OnDel.
+		for _, f := range n.onDel {
+			f()
+		}
+
+		// Update counter.
+		atomic.AddInt32(&r.size, int32(n.size)*-1)
+		shrink := atomic.AddInt32(&r.nodes, -1) < h.shrinkThreshold
+		if bLen >= mOverflowThreshold {
+			atomic.AddInt32(&h.overflow, -1)
+		}
+
+		// Shrink.
+		if shrink && len(h.buckets) > mInitialSize && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) {
+			nhLen := len(h.buckets) >> 1
+			nh := &mNode{
+				buckets:         make([]unsafe.Pointer, nhLen),
+				mask:            uint32(nhLen) - 1,
+				pred:            unsafe.Pointer(h),
+				growThreshold:   int32(nhLen * mOverflowThreshold),
+				shrinkThreshold: int32(nhLen >> 1),
+			}
+			ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh))
+			if !ok {
+				panic("BUG: failed swapping head")
+			}
+			go nh.initBuckets()
+		}
+	}
+
+	return true, deleted
+}
+
+type mNode struct {
+	buckets         []unsafe.Pointer // []*mBucket
+	mask            uint32
+	pred            unsafe.Pointer // *mNode
+	resizeInProgess int32
+
+	overflow        int32
+	growThreshold   int32
+	shrinkThreshold int32
+}
+
+func (n *mNode) initBucket(i uint32) *mBucket {
+	if b := (*mBucket)(atomic.LoadPointer(&n.buckets[i])); b != nil {
+		return b
+	}
+
+	p := (*mNode)(atomic.LoadPointer(&n.pred))
+	if p != nil {
+		var node []*Node
+		if n.mask > p.mask {
+			// Grow.
+			pb := (*mBucket)(atomic.LoadPointer(&p.buckets[i&p.mask]))
+			if pb == nil {
+				pb = p.initBucket(i & p.mask)
+			}
+			m := pb.freeze()
+			// Split nodes.
+			for _, x := range m {
+				if x.hash&n.mask == i {
+					node = append(node, x)
+				}
+			}
+		} else {
+			// Shrink.
+			pb0 := (*mBucket)(atomic.LoadPointer(&p.buckets[i]))
+			if pb0 == nil {
+				pb0 = p.initBucket(i)
+			}
+			pb1 := (*mBucket)(atomic.LoadPointer(&p.buckets[i+uint32(len(n.buckets))]))
+			if pb1 == nil {
+				pb1 = p.initBucket(i + uint32(len(n.buckets)))
+			}
+			m0 := pb0.freeze()
+			m1 := pb1.freeze()
+			// Merge nodes.
+			node = make([]*Node, 0, len(m0)+len(m1))
+			node = append(node, m0...)
+			node = append(node, m1...)
+		}
+		b := &mBucket{node: node}
+		if atomic.CompareAndSwapPointer(&n.buckets[i], nil, unsafe.Pointer(b)) {
+			if len(node) > mOverflowThreshold {
+				atomic.AddInt32(&n.overflow, int32(len(node)-mOverflowThreshold))
+			}
+			return b
+		}
+	}
+
+	return (*mBucket)(atomic.LoadPointer(&n.buckets[i]))
+}
+
+func (n *mNode) initBuckets() {
+	for i := range n.buckets {
+		n.initBucket(uint32(i))
+	}
+	atomic.StorePointer(&n.pred, nil)
+}
+
+// Cache is a 'cache map'.
+type Cache struct {
+	mu     sync.RWMutex
+	mHead  unsafe.Pointer // *mNode
+	nodes  int32
+	size   int32
+	cacher Cacher
+	closed bool
+}
+
+// NewCache creates a new 'cache map'. The cacher is optional and
+// may be nil.
+func NewCache(cacher Cacher) *Cache {
+	h := &mNode{
+		buckets:         make([]unsafe.Pointer, mInitialSize),
+		mask:            mInitialSize - 1,
+		growThreshold:   int32(mInitialSize * mOverflowThreshold),
+		shrinkThreshold: 0,
+	}
+	for i := range h.buckets {
+		h.buckets[i] = unsafe.Pointer(&mBucket{})
+	}
+	r := &Cache{
+		mHead:  unsafe.Pointer(h),
+		cacher: cacher,
+	}
+	return r
+}
+
+func (r *Cache) getBucket(hash uint32) (*mNode, *mBucket) {
+	h := (*mNode)(atomic.LoadPointer(&r.mHead))
+	i := hash & h.mask
+	b := (*mBucket)(atomic.LoadPointer(&h.buckets[i]))
+	if b == nil {
+		b = h.initBucket(i)
+	}
+	return h, b
+}
+
+func (r *Cache) delete(n *Node) bool {
+	for {
+		h, b := r.getBucket(n.hash)
+		done, deleted := b.delete(r, h, n.hash, n.ns, n.key)
+		if done {
+			return deleted
+		}
+	}
+	return false
+}
+
+// Nodes returns number of 'cache node' in the map.
+func (r *Cache) Nodes() int {
+	return int(atomic.LoadInt32(&r.nodes))
+}
+
+// Size returns sums of 'cache node' size in the map.
+func (r *Cache) Size() int {
+	return int(atomic.LoadInt32(&r.size))
+}
+
+// Capacity returns cache capacity.
+func (r *Cache) Capacity() int {
+	if r.cacher == nil {
+		return 0
+	}
+	return r.cacher.Capacity()
+}
+
+// SetCapacity sets cache capacity.
+func (r *Cache) SetCapacity(capacity int) {
+	if r.cacher != nil {
+		r.cacher.SetCapacity(capacity)
+	}
+}
+
+// Get gets 'cache node' with the given namespace and key.
+// If cache node is not found and setFunc is not nil, Get will atomically creates
+// the 'cache node' by calling setFunc. Otherwise Get will returns nil.
+//
+// The returned 'cache handle' should be released after use by calling Release
+// method.
+func (r *Cache) Get(ns, key uint64, setFunc func() (size int, value Value)) *Handle {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	if r.closed {
+		return nil
+	}
+
+	hash := murmur32(ns, key, 0xf00)
+	for {
+		h, b := r.getBucket(hash)
+		done, _, n := b.get(r, h, hash, ns, key, setFunc == nil)
+		if done {
+			if n != nil {
+				n.mu.Lock()
+				if n.value == nil {
+					if setFunc == nil {
+						n.mu.Unlock()
+						n.unref()
+						return nil
+					}
+
+					n.size, n.value = setFunc()
+					if n.value == nil {
+						n.size = 0
+						n.mu.Unlock()
+						n.unref()
+						return nil
+					}
+					atomic.AddInt32(&r.size, int32(n.size))
+				}
+				n.mu.Unlock()
+				if r.cacher != nil {
+					r.cacher.Promote(n)
+				}
+				return &Handle{unsafe.Pointer(n)}
+			}
+
+			break
+		}
 	}
 	return nil
 }
 
-func (o *fakeObject) Release() {
-	if !atomic.CompareAndSwapUint32(&o.once, 0, 1) {
+// Delete removes and ban 'cache node' with the given namespace and key.
+// A banned 'cache node' will never inserted into the 'cache tree'. Ban
+// only attributed to the particular 'cache node', so when a 'cache node'
+// is recreated it will not be banned.
+//
+// If onDel is not nil, then it will be executed if such 'cache node'
+// doesn't exist or once the 'cache node' is released.
+//
+// Delete return true is such 'cache node' exist.
+func (r *Cache) Delete(ns, key uint64, onDel func()) bool {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	if r.closed {
+		return false
+	}
+
+	hash := murmur32(ns, key, 0xf00)
+	for {
+		h, b := r.getBucket(hash)
+		done, _, n := b.get(r, h, hash, ns, key, true)
+		if done {
+			if n != nil {
+				if onDel != nil {
+					n.mu.Lock()
+					n.onDel = append(n.onDel, onDel)
+					n.mu.Unlock()
+				}
+				if r.cacher != nil {
+					r.cacher.Ban(n)
+				}
+				n.unref()
+				return true
+			}
+
+			break
+		}
+	}
+
+	if onDel != nil {
+		onDel()
+	}
+
+	return false
+}
+
+// Evict evicts 'cache node' with the given namespace and key. This will
+// simply call Cacher.Evict.
+//
+// Evict return true is such 'cache node' exist.
+func (r *Cache) Evict(ns, key uint64) bool {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	if r.closed {
+		return false
+	}
+
+	hash := murmur32(ns, key, 0xf00)
+	for {
+		h, b := r.getBucket(hash)
+		done, _, n := b.get(r, h, hash, ns, key, true)
+		if done {
+			if n != nil {
+				if r.cacher != nil {
+					r.cacher.Evict(n)
+				}
+				n.unref()
+				return true
+			}
+
+			break
+		}
+	}
+
+	return false
+}
+
+// EvictNS evicts 'cache node' with the given namespace. This will
+// simply call Cacher.EvictNS.
+func (r *Cache) EvictNS(ns uint64) {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	if r.closed {
 		return
 	}
-	if o.fin != nil {
-		o.fin()
-		o.fin = nil
+
+	if r.cacher != nil {
+		r.cacher.EvictNS(ns)
 	}
 }
+
+// EvictAll evicts all 'cache node'. This will simply call Cacher.EvictAll.
+func (r *Cache) EvictAll() {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+	if r.closed {
+		return
+	}
+
+	if r.cacher != nil {
+		r.cacher.EvictAll()
+	}
+}
+
+// Close closes the 'cache map' and releases all 'cache node'.
+func (r *Cache) Close() error {
+	r.mu.Lock()
+	if !r.closed {
+		r.closed = true
+
+		if r.cacher != nil {
+			if err := r.cacher.Close(); err != nil {
+				return err
+			}
+		}
+
+		h := (*mNode)(r.mHead)
+		h.initBuckets()
+
+		for i := range h.buckets {
+			b := (*mBucket)(h.buckets[i])
+			for _, n := range b.node {
+				// Call releaser.
+				if n.value != nil {
+					if r, ok := n.value.(util.Releaser); ok {
+						r.Release()
+					}
+					n.value = nil
+				}
+
+				// Call OnDel.
+				for _, f := range n.onDel {
+					f()
+				}
+			}
+		}
+	}
+	r.mu.Unlock()
+	return nil
+}
+
+// Node is a 'cache node'.
+type Node struct {
+	r *Cache
+
+	hash    uint32
+	ns, key uint64
+
+	mu    sync.Mutex
+	size  int
+	value Value
+
+	ref   int32
+	onDel []func()
+
+	CacheData unsafe.Pointer
+}
+
+// NS returns this 'cache node' namespace.
+func (n *Node) NS() uint64 {
+	return n.ns
+}
+
+// Key returns this 'cache node' key.
+func (n *Node) Key() uint64 {
+	return n.key
+}
+
+// Size returns this 'cache node' size.
+func (n *Node) Size() int {
+	return n.size
+}
+
+// Value returns this 'cache node' value.
+func (n *Node) Value() Value {
+	return n.value
+}
+
+// Ref returns this 'cache node' ref counter.
+func (n *Node) Ref() int32 {
+	return atomic.LoadInt32(&n.ref)
+}
+
+// GetHandle returns an handle for this 'cache node'.
+func (n *Node) GetHandle() *Handle {
+	if atomic.AddInt32(&n.ref, 1) <= 1 {
+		panic("BUG: Node.GetHandle on zero ref")
+	}
+	return &Handle{unsafe.Pointer(n)}
+}
+
+func (n *Node) unref() {
+	if atomic.AddInt32(&n.ref, -1) == 0 {
+		n.r.delete(n)
+	}
+}
+
+func (n *Node) unrefLocked() {
+	if atomic.AddInt32(&n.ref, -1) == 0 {
+		n.r.mu.RLock()
+		if !n.r.closed {
+			n.r.delete(n)
+		}
+		n.r.mu.RUnlock()
+	}
+}
+
+type Handle struct {
+	n unsafe.Pointer // *Node
+}
+
+func (h *Handle) Value() Value {
+	n := (*Node)(atomic.LoadPointer(&h.n))
+	if n != nil {
+		return n.value
+	}
+	return nil
+}
+
+func (h *Handle) Release() {
+	nPtr := atomic.LoadPointer(&h.n)
+	if nPtr != nil && atomic.CompareAndSwapPointer(&h.n, nPtr, nil) {
+		n := (*Node)(nPtr)
+		n.unrefLocked()
+	}
+}
+
+func murmur32(ns, key uint64, seed uint32) uint32 {
+	const (
+		m = uint32(0x5bd1e995)
+		r = 24
+	)
+
+	k1 := uint32(ns >> 32)
+	k2 := uint32(ns)
+	k3 := uint32(key >> 32)
+	k4 := uint32(key)
+
+	k1 *= m
+	k1 ^= k1 >> r
+	k1 *= m
+
+	k2 *= m
+	k2 ^= k2 >> r
+	k2 *= m
+
+	k3 *= m
+	k3 ^= k3 >> r
+	k3 *= m
+
+	k4 *= m
+	k4 ^= k4 >> r
+	k4 *= m
+
+	h := seed
+
+	h *= m
+	h ^= k1
+	h *= m
+	h ^= k2
+	h *= m
+	h ^= k3
+	h *= m
+	h ^= k4
+
+	h ^= h >> 13
+	h *= m
+	h ^= h >> 15
+
+	return h
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go

@@ -8,17 +8,289 @@ package cache
 
 import (
 	"math/rand"
+	"runtime"
+	"sync"
+	"sync/atomic"
 	"testing"
+	"time"
+	"unsafe"
 )
 
-func set(ns Namespace, key uint64, value interface{}, charge int, fin func()) Object {
-	obj, _ := ns.Get(key, func() (bool, interface{}, int, SetFin) {
-		return true, value, charge, fin
-	})
-	return obj
+type int32o int32
+
+func (o *int32o) acquire() {
+	if atomic.AddInt32((*int32)(o), 1) != 1 {
+		panic("BUG: invalid ref")
+	}
 }
 
-func TestCache_HitMiss(t *testing.T) {
+func (o *int32o) Release() {
+	if atomic.AddInt32((*int32)(o), -1) != 0 {
+		panic("BUG: invalid ref")
+	}
+}
+
+type releaserFunc struct {
+	fn    func()
+	value Value
+}
+
+func (r releaserFunc) Release() {
+	if r.fn != nil {
+		r.fn()
+	}
+}
+
+func set(c *Cache, ns, key uint64, value Value, charge int, relf func()) *Handle {
+	return c.Get(ns, key, func() (int, Value) {
+		if relf != nil {
+			return charge, releaserFunc{relf, value}
+		} else {
+			return charge, value
+		}
+	})
+}
+
+func TestCacheMap(t *testing.T) {
+	runtime.GOMAXPROCS(runtime.NumCPU())
+
+	nsx := []struct {
+		nobjects, nhandles, concurrent, repeat int
+	}{
+		{10000, 400, 50, 3},
+		{100000, 1000, 100, 10},
+	}
+
+	var (
+		objects [][]int32o
+		handles [][]unsafe.Pointer
+	)
+
+	for _, x := range nsx {
+		objects = append(objects, make([]int32o, x.nobjects))
+		handles = append(handles, make([]unsafe.Pointer, x.nhandles))
+	}
+
+	c := NewCache(nil)
+
+	wg := new(sync.WaitGroup)
+	var done int32
+
+	for ns, x := range nsx {
+		for i := 0; i < x.concurrent; i++ {
+			wg.Add(1)
+			go func(ns, i, repeat int, objects []int32o, handles []unsafe.Pointer) {
+				defer wg.Done()
+				r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+				for j := len(objects) * repeat; j >= 0; j-- {
+					key := uint64(r.Intn(len(objects)))
+					h := c.Get(uint64(ns), key, func() (int, Value) {
+						o := &objects[key]
+						o.acquire()
+						return 1, o
+					})
+					if v := h.Value().(*int32o); v != &objects[key] {
+						t.Fatalf("#%d invalid value: want=%p got=%p", ns, &objects[key], v)
+					}
+					if objects[key] != 1 {
+						t.Fatalf("#%d invalid object %d: %d", ns, key, objects[key])
+					}
+					if !atomic.CompareAndSwapPointer(&handles[r.Intn(len(handles))], nil, unsafe.Pointer(h)) {
+						h.Release()
+					}
+				}
+			}(ns, i, x.repeat, objects[ns], handles[ns])
+		}
+
+		go func(handles []unsafe.Pointer) {
+			r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+			for atomic.LoadInt32(&done) == 0 {
+				i := r.Intn(len(handles))
+				h := (*Handle)(atomic.LoadPointer(&handles[i]))
+				if h != nil && atomic.CompareAndSwapPointer(&handles[i], unsafe.Pointer(h), nil) {
+					h.Release()
+				}
+				time.Sleep(time.Millisecond)
+			}
+		}(handles[ns])
+	}
+
+	go func() {
+		handles := make([]*Handle, 100000)
+		for atomic.LoadInt32(&done) == 0 {
+			for i := range handles {
+				handles[i] = c.Get(999999999, uint64(i), func() (int, Value) {
+					return 1, 1
+				})
+			}
+			for _, h := range handles {
+				h.Release()
+			}
+		}
+	}()
+
+	wg.Wait()
+
+	atomic.StoreInt32(&done, 1)
+
+	for _, handles0 := range handles {
+		for i := range handles0 {
+			h := (*Handle)(atomic.LoadPointer(&handles0[i]))
+			if h != nil && atomic.CompareAndSwapPointer(&handles0[i], unsafe.Pointer(h), nil) {
+				h.Release()
+			}
+		}
+	}
+
+	for ns, objects0 := range objects {
+		for i, o := range objects0 {
+			if o != 0 {
+				t.Fatalf("invalid object #%d.%d: ref=%d", ns, i, o)
+			}
+		}
+	}
+}
+
+func TestCacheMap_NodesAndSize(t *testing.T) {
+	c := NewCache(nil)
+	if c.Nodes() != 0 {
+		t.Errorf("invalid nodes counter: want=%d got=%d", 0, c.Nodes())
+	}
+	if c.Size() != 0 {
+		t.Errorf("invalid size counter: want=%d got=%d", 0, c.Size())
+	}
+	set(c, 0, 1, 1, 1, nil)
+	set(c, 0, 2, 2, 2, nil)
+	set(c, 1, 1, 3, 3, nil)
+	set(c, 2, 1, 4, 1, nil)
+	if c.Nodes() != 4 {
+		t.Errorf("invalid nodes counter: want=%d got=%d", 4, c.Nodes())
+	}
+	if c.Size() != 7 {
+		t.Errorf("invalid size counter: want=%d got=%d", 4, c.Size())
+	}
+}
+
+func TestLRUCache_Capacity(t *testing.T) {
+	c := NewCache(NewLRU(10))
+	if c.Capacity() != 10 {
+		t.Errorf("invalid capacity: want=%d got=%d", 10, c.Capacity())
+	}
+	set(c, 0, 1, 1, 1, nil).Release()
+	set(c, 0, 2, 2, 2, nil).Release()
+	set(c, 1, 1, 3, 3, nil).Release()
+	set(c, 2, 1, 4, 1, nil).Release()
+	set(c, 2, 2, 5, 1, nil).Release()
+	set(c, 2, 3, 6, 1, nil).Release()
+	set(c, 2, 4, 7, 1, nil).Release()
+	set(c, 2, 5, 8, 1, nil).Release()
+	if c.Nodes() != 7 {
+		t.Errorf("invalid nodes counter: want=%d got=%d", 7, c.Nodes())
+	}
+	if c.Size() != 10 {
+		t.Errorf("invalid size counter: want=%d got=%d", 10, c.Size())
+	}
+	c.SetCapacity(9)
+	if c.Capacity() != 9 {
+		t.Errorf("invalid capacity: want=%d got=%d", 9, c.Capacity())
+	}
+	if c.Nodes() != 6 {
+		t.Errorf("invalid nodes counter: want=%d got=%d", 6, c.Nodes())
+	}
+	if c.Size() != 8 {
+		t.Errorf("invalid size counter: want=%d got=%d", 8, c.Size())
+	}
+}
+
+func TestCacheMap_NilValue(t *testing.T) {
+	c := NewCache(NewLRU(10))
+	h := c.Get(0, 0, func() (size int, value Value) {
+		return 1, nil
+	})
+	if h != nil {
+		t.Error("cache handle is non-nil")
+	}
+	if c.Nodes() != 0 {
+		t.Errorf("invalid nodes counter: want=%d got=%d", 0, c.Nodes())
+	}
+	if c.Size() != 0 {
+		t.Errorf("invalid size counter: want=%d got=%d", 0, c.Size())
+	}
+}
+
+func TestLRUCache_GetLatency(t *testing.T) {
+	runtime.GOMAXPROCS(runtime.NumCPU())
+
+	const (
+		concurrentSet = 30
+		concurrentGet = 3
+		duration      = 3 * time.Second
+		delay         = 3 * time.Millisecond
+		maxkey        = 100000
+	)
+
+	var (
+		set, getHit, getAll        int32
+		getMaxLatency, getDuration int64
+	)
+
+	c := NewCache(NewLRU(5000))
+	wg := &sync.WaitGroup{}
+	until := time.Now().Add(duration)
+	for i := 0; i < concurrentSet; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			r := rand.New(rand.NewSource(time.Now().UnixNano()))
+			for time.Now().Before(until) {
+				c.Get(0, uint64(r.Intn(maxkey)), func() (int, Value) {
+					time.Sleep(delay)
+					atomic.AddInt32(&set, 1)
+					return 1, 1
+				}).Release()
+			}
+		}(i)
+	}
+	for i := 0; i < concurrentGet; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			r := rand.New(rand.NewSource(time.Now().UnixNano()))
+			for {
+				mark := time.Now()
+				if mark.Before(until) {
+					h := c.Get(0, uint64(r.Intn(maxkey)), nil)
+					latency := int64(time.Now().Sub(mark))
+					m := atomic.LoadInt64(&getMaxLatency)
+					if latency > m {
+						atomic.CompareAndSwapInt64(&getMaxLatency, m, latency)
+					}
+					atomic.AddInt64(&getDuration, latency)
+					if h != nil {
+						atomic.AddInt32(&getHit, 1)
+						h.Release()
+					}
+					atomic.AddInt32(&getAll, 1)
+				} else {
+					break
+				}
+			}
+		}(i)
+	}
+
+	wg.Wait()
+	getAvglatency := time.Duration(getDuration) / time.Duration(getAll)
+	t.Logf("set=%d getHit=%d getAll=%d getMaxLatency=%v getAvgLatency=%v",
+		set, getHit, getAll, time.Duration(getMaxLatency), getAvglatency)
+
+	if getAvglatency > delay/3 {
+		t.Errorf("get avg latency > %v: got=%v", delay/3, getAvglatency)
+	}
+}
+
+func TestLRUCache_HitMiss(t *testing.T) {
 	cases := []struct {
 		key   uint64
 		value string
@@ -36,36 +308,37 @@ func TestCache_HitMiss(t *testing.T) {
 	}
 
 	setfin := 0
-	c := NewLRUCache(1000)
-	ns := c.GetNamespace(0)
+	c := NewCache(NewLRU(1000))
 	for i, x := range cases {
-		set(ns, x.key, x.value, len(x.value), func() {
+		set(c, 0, x.key, x.value, len(x.value), func() {
 			setfin++
 		}).Release()
 		for j, y := range cases {
-			r, ok := ns.Get(y.key, nil)
+			h := c.Get(0, y.key, nil)
 			if j <= i {
 				// should hit
-				if !ok {
+				if h == nil {
 					t.Errorf("case '%d' iteration '%d' is miss", i, j)
-				} else if r.Value().(string) != y.value {
-					t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, r.Value().(string), y.value)
+				} else {
+					if x := h.Value().(releaserFunc).value.(string); x != y.value {
+						t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, x, y.value)
+					}
 				}
 			} else {
 				// should miss
-				if ok {
-					t.Errorf("case '%d' iteration '%d' is hit , value '%s'", i, j, r.Value().(string))
+				if h != nil {
+					t.Errorf("case '%d' iteration '%d' is hit , value '%s'", i, j, h.Value().(releaserFunc).value.(string))
 				}
 			}
-			if ok {
-				r.Release()
+			if h != nil {
+				h.Release()
 			}
 		}
 	}
 
 	for i, x := range cases {
 		finalizerOk := false
-		ns.Delete(x.key, func(exist bool) {
+		c.Delete(0, x.key, func() {
 			finalizerOk = true
 		})
 
@@ -74,22 +347,24 @@ func TestCache_HitMiss(t *testing.T) {
 		}
 
 		for j, y := range cases {
-			r, ok := ns.Get(y.key, nil)
+			h := c.Get(0, y.key, nil)
 			if j > i {
 				// should hit
-				if !ok {
+				if h == nil {
 					t.Errorf("case '%d' iteration '%d' is miss", i, j)
-				} else if r.Value().(string) != y.value {
-					t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, r.Value().(string), y.value)
+				} else {
+					if x := h.Value().(releaserFunc).value.(string); x != y.value {
+						t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, x, y.value)
+					}
 				}
 			} else {
 				// should miss
-				if ok {
-					t.Errorf("case '%d' iteration '%d' is hit, value '%s'", i, j, r.Value().(string))
+				if h != nil {
+					t.Errorf("case '%d' iteration '%d' is hit, value '%s'", i, j, h.Value().(releaserFunc).value.(string))
 				}
 			}
-			if ok {
-				r.Release()
+			if h != nil {
+				h.Release()
 			}
 		}
 	}
@@ -100,137 +375,196 @@ func TestCache_HitMiss(t *testing.T) {
 }
 
 func TestLRUCache_Eviction(t *testing.T) {
-	c := NewLRUCache(12)
-	ns := c.GetNamespace(0)
-	o1 := set(ns, 1, 1, 1, nil)
-	set(ns, 2, 2, 1, nil).Release()
-	set(ns, 3, 3, 1, nil).Release()
-	set(ns, 4, 4, 1, nil).Release()
-	set(ns, 5, 5, 1, nil).Release()
-	if r, ok := ns.Get(2, nil); ok { // 1,3,4,5,2
-		r.Release()
+	c := NewCache(NewLRU(12))
+	o1 := set(c, 0, 1, 1, 1, nil)
+	set(c, 0, 2, 2, 1, nil).Release()
+	set(c, 0, 3, 3, 1, nil).Release()
+	set(c, 0, 4, 4, 1, nil).Release()
+	set(c, 0, 5, 5, 1, nil).Release()
+	if h := c.Get(0, 2, nil); h != nil { // 1,3,4,5,2
+		h.Release()
 	}
-	set(ns, 9, 9, 10, nil).Release() // 5,2,9
+	set(c, 0, 9, 9, 10, nil).Release() // 5,2,9
 
-	for _, x := range []uint64{9, 2, 5, 1} {
-		r, ok := ns.Get(x, nil)
-		if !ok {
-			t.Errorf("miss for key '%d'", x)
+	for _, key := range []uint64{9, 2, 5, 1} {
+		h := c.Get(0, key, nil)
+		if h == nil {
+			t.Errorf("miss for key '%d'", key)
 		} else {
-			if r.Value().(int) != int(x) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
+			if x := h.Value().(int); x != int(key) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
 			}
-			r.Release()
+			h.Release()
 		}
 	}
 	o1.Release()
-	for _, x := range []uint64{1, 2, 5} {
-		r, ok := ns.Get(x, nil)
-		if !ok {
-			t.Errorf("miss for key '%d'", x)
+	for _, key := range []uint64{1, 2, 5} {
+		h := c.Get(0, key, nil)
+		if h == nil {
+			t.Errorf("miss for key '%d'", key)
 		} else {
-			if r.Value().(int) != int(x) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
+			if x := h.Value().(int); x != int(key) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
 			}
-			r.Release()
+			h.Release()
 		}
 	}
-	for _, x := range []uint64{3, 4, 9} {
-		r, ok := ns.Get(x, nil)
-		if ok {
-			t.Errorf("hit for key '%d'", x)
-			if r.Value().(int) != int(x) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
+	for _, key := range []uint64{3, 4, 9} {
+		h := c.Get(0, key, nil)
+		if h != nil {
+			t.Errorf("hit for key '%d'", key)
+			if x := h.Value().(int); x != int(key) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
 			}
-			r.Release()
+			h.Release()
 		}
 	}
 }
 
-func TestLRUCache_SetGet(t *testing.T) {
-	c := NewLRUCache(13)
-	ns := c.GetNamespace(0)
-	for i := 0; i < 200; i++ {
-		n := uint64(rand.Intn(99999) % 20)
-		set(ns, n, n, 1, nil).Release()
-		if p, ok := ns.Get(n, nil); ok {
-			if p.Value() == nil {
-				t.Errorf("key '%d' contains nil value", n)
+func TestLRUCache_Evict(t *testing.T) {
+	c := NewCache(NewLRU(6))
+	set(c, 0, 1, 1, 1, nil).Release()
+	set(c, 0, 2, 2, 1, nil).Release()
+	set(c, 1, 1, 4, 1, nil).Release()
+	set(c, 1, 2, 5, 1, nil).Release()
+	set(c, 2, 1, 6, 1, nil).Release()
+	set(c, 2, 2, 7, 1, nil).Release()
+
+	for ns := 0; ns < 3; ns++ {
+		for key := 1; key < 3; key++ {
+			if h := c.Get(uint64(ns), uint64(key), nil); h != nil {
+				h.Release()
 			} else {
-				got := p.Value().(uint64)
-				if got != n {
-					t.Errorf("invalid value for key '%d' want '%d', got '%d'", n, n, got)
-				}
+				t.Errorf("Cache.Get on #%d.%d return nil", ns, key)
 			}
-			p.Release()
+		}
+	}
+
+	if ok := c.Evict(0, 1); !ok {
+		t.Error("first Cache.Evict on #0.1 return false")
+	}
+	if ok := c.Evict(0, 1); ok {
+		t.Error("second Cache.Evict on #0.1 return true")
+	}
+	if h := c.Get(0, 1, nil); h != nil {
+		t.Errorf("Cache.Get on #0.1 return non-nil: %v", h.Value())
+	}
+
+	c.EvictNS(1)
+	if h := c.Get(1, 1, nil); h != nil {
+		t.Errorf("Cache.Get on #1.1 return non-nil: %v", h.Value())
+	}
+	if h := c.Get(1, 2, nil); h != nil {
+		t.Errorf("Cache.Get on #1.2 return non-nil: %v", h.Value())
+	}
+
+	c.EvictAll()
+	for ns := 0; ns < 3; ns++ {
+		for key := 1; key < 3; key++ {
+			if h := c.Get(uint64(ns), uint64(key), nil); h != nil {
+				t.Errorf("Cache.Get on #%d.%d return non-nil: %v", ns, key, h.Value())
+			}
+		}
+	}
+}
+
+func TestLRUCache_Delete(t *testing.T) {
+	delFuncCalled := 0
+	delFunc := func() {
+		delFuncCalled++
+	}
+
+	c := NewCache(NewLRU(2))
+	set(c, 0, 1, 1, 1, nil).Release()
+	set(c, 0, 2, 2, 1, nil).Release()
+
+	if ok := c.Delete(0, 1, delFunc); !ok {
+		t.Error("Cache.Delete on #1 return false")
+	}
+	if h := c.Get(0, 1, nil); h != nil {
+		t.Errorf("Cache.Get on #1 return non-nil: %v", h.Value())
+	}
+	if ok := c.Delete(0, 1, delFunc); ok {
+		t.Error("Cache.Delete on #1 return true")
+	}
+
+	h2 := c.Get(0, 2, nil)
+	if h2 == nil {
+		t.Error("Cache.Get on #2 return nil")
+	}
+	if ok := c.Delete(0, 2, delFunc); !ok {
+		t.Error("(1) Cache.Delete on #2 return false")
+	}
+	if ok := c.Delete(0, 2, delFunc); !ok {
+		t.Error("(2) Cache.Delete on #2 return false")
+	}
+
+	set(c, 0, 3, 3, 1, nil).Release()
+	set(c, 0, 4, 4, 1, nil).Release()
+	c.Get(0, 2, nil).Release()
+
+	for key := 2; key <= 4; key++ {
+		if h := c.Get(0, uint64(key), nil); h != nil {
+			h.Release()
 		} else {
-			t.Errorf("key '%d' doesn't exist", n)
+			t.Errorf("Cache.Get on #%d return nil", key)
 		}
 	}
+
+	h2.Release()
+	if h := c.Get(0, 2, nil); h != nil {
+		t.Errorf("Cache.Get on #2 return non-nil: %v", h.Value())
+	}
+
+	if delFuncCalled != 4 {
+		t.Errorf("delFunc isn't called 4 times: got=%d", delFuncCalled)
+	}
 }
 
-func TestLRUCache_Purge(t *testing.T) {
-	c := NewLRUCache(3)
-	ns1 := c.GetNamespace(0)
-	o1 := set(ns1, 1, 1, 1, nil)
-	o2 := set(ns1, 2, 2, 1, nil)
-	ns1.Purge(nil)
-	set(ns1, 3, 3, 1, nil).Release()
-	for _, x := range []uint64{1, 2, 3} {
-		r, ok := ns1.Get(x, nil)
-		if !ok {
-			t.Errorf("miss for key '%d'", x)
-		} else {
-			if r.Value().(int) != int(x) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
-			}
-			r.Release()
+func TestLRUCache_Close(t *testing.T) {
+	relFuncCalled := 0
+	relFunc := func() {
+		relFuncCalled++
+	}
+	delFuncCalled := 0
+	delFunc := func() {
+		delFuncCalled++
+	}
+
+	c := NewCache(NewLRU(2))
+	set(c, 0, 1, 1, 1, relFunc).Release()
+	set(c, 0, 2, 2, 1, relFunc).Release()
+
+	h3 := set(c, 0, 3, 3, 1, relFunc)
+	if h3 == nil {
+		t.Error("Cache.Get on #3 return nil")
+	}
+	if ok := c.Delete(0, 3, delFunc); !ok {
+		t.Error("Cache.Delete on #3 return false")
+	}
+
+	c.Close()
+
+	if relFuncCalled != 3 {
+		t.Errorf("relFunc isn't called 3 times: got=%d", relFuncCalled)
+	}
+	if delFuncCalled != 1 {
+		t.Errorf("delFunc isn't called 1 times: got=%d", delFuncCalled)
+	}
+}
+
+func BenchmarkLRUCache(b *testing.B) {
+	c := NewCache(NewLRU(10000))
+
+	b.SetParallelism(10)
+	b.RunParallel(func(pb *testing.PB) {
+		r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+		for pb.Next() {
+			key := uint64(r.Intn(1000000))
+			c.Get(0, key, func() (int, Value) {
+				return 1, key
+			}).Release()
 		}
-	}
-	o1.Release()
-	o2.Release()
-	for _, x := range []uint64{1, 2} {
-		r, ok := ns1.Get(x, nil)
-		if ok {
-			t.Errorf("hit for key '%d'", x)
-			if r.Value().(int) != int(x) {
-				t.Errorf("invalid value for key '%d' want '%d', got '%d'", x, x, r.Value().(int))
-			}
-			r.Release()
-		}
-	}
-}
-
-func BenchmarkLRUCache_SetRelease(b *testing.B) {
-	capacity := b.N / 100
-	if capacity <= 0 {
-		capacity = 10
-	}
-	c := NewLRUCache(capacity)
-	ns := c.GetNamespace(0)
-	b.ResetTimer()
-	for i := uint64(0); i < uint64(b.N); i++ {
-		set(ns, i, nil, 1, nil).Release()
-	}
-}
-
-func BenchmarkLRUCache_SetReleaseTwice(b *testing.B) {
-	capacity := b.N / 100
-	if capacity <= 0 {
-		capacity = 10
-	}
-	c := NewLRUCache(capacity)
-	ns := c.GetNamespace(0)
-	b.ResetTimer()
-
-	na := b.N / 2
-	nb := b.N - na
-
-	for i := uint64(0); i < uint64(na); i++ {
-		set(ns, i, nil, 1, nil).Release()
-	}
-
-	for i := uint64(0); i < uint64(nb); i++ {
-		set(ns, i, nil, 1, nil).Release()
-	}
+	})
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/empty_cache.go

@@ -1,246 +0,0 @@
-// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package cache
-
-import (
-	"sync"
-	"sync/atomic"
-)
-
-type emptyCache struct {
-	sync.Mutex
-	table map[uint64]*emptyNS
-}
-
-// NewEmptyCache creates a new initialized empty cache.
-func NewEmptyCache() Cache {
-	return &emptyCache{
-		table: make(map[uint64]*emptyNS),
-	}
-}
-
-func (c *emptyCache) GetNamespace(id uint64) Namespace {
-	c.Lock()
-	defer c.Unlock()
-
-	if ns, ok := c.table[id]; ok {
-		return ns
-	}
-
-	ns := &emptyNS{
-		cache: c,
-		id:    id,
-		table: make(map[uint64]*emptyNode),
-	}
-	c.table[id] = ns
-	return ns
-}
-
-func (c *emptyCache) Purge(fin PurgeFin) {
-	c.Lock()
-	for _, ns := range c.table {
-		ns.purgeNB(fin)
-	}
-	c.Unlock()
-}
-
-func (c *emptyCache) Zap(closed bool) {
-	c.Lock()
-	for _, ns := range c.table {
-		ns.zapNB(closed)
-	}
-	c.table = make(map[uint64]*emptyNS)
-	c.Unlock()
-}
-
-func (*emptyCache) SetCapacity(capacity int) {}
-
-type emptyNS struct {
-	cache *emptyCache
-	id    uint64
-	table map[uint64]*emptyNode
-	state nsState
-}
-
-func (ns *emptyNS) Get(key uint64, setf SetFunc) (o Object, ok bool) {
-	ns.cache.Lock()
-
-	switch ns.state {
-	case nsZapped:
-		ns.cache.Unlock()
-		if setf == nil {
-			return
-		}
-
-		var value interface{}
-		var fin func()
-		ok, value, _, fin = setf()
-		if ok {
-			o = &fakeObject{
-				value: value,
-				fin:   fin,
-			}
-		}
-		return
-	case nsClosed:
-		ns.cache.Unlock()
-		return
-	}
-
-	n, ok := ns.table[key]
-	if ok {
-		n.ref++
-	} else {
-		if setf == nil {
-			ns.cache.Unlock()
-			return
-		}
-
-		var value interface{}
-		var fin func()
-		ok, value, _, fin = setf()
-		if !ok {
-			ns.cache.Unlock()
-			return
-		}
-
-		n = &emptyNode{
-			ns:     ns,
-			key:    key,
-			value:  value,
-			setfin: fin,
-			ref:    1,
-		}
-		ns.table[key] = n
-	}
-
-	ns.cache.Unlock()
-	o = &emptyObject{node: n}
-	return
-}
-
-func (ns *emptyNS) Delete(key uint64, fin DelFin) bool {
-	ns.cache.Lock()
-
-	if ns.state != nsEffective {
-		ns.cache.Unlock()
-		if fin != nil {
-			fin(false)
-		}
-		return false
-	}
-
-	n, ok := ns.table[key]
-	if !ok {
-		ns.cache.Unlock()
-		if fin != nil {
-			fin(false)
-		}
-		return false
-	}
-	n.delfin = fin
-	ns.cache.Unlock()
-	return true
-}
-
-func (ns *emptyNS) purgeNB(fin PurgeFin) {
-	if ns.state != nsEffective {
-		return
-	}
-	for _, n := range ns.table {
-		n.purgefin = fin
-	}
-}
-
-func (ns *emptyNS) Purge(fin PurgeFin) {
-	ns.cache.Lock()
-	ns.purgeNB(fin)
-	ns.cache.Unlock()
-}
-
-func (ns *emptyNS) zapNB(closed bool) {
-	if ns.state != nsEffective {
-		return
-	}
-	for _, n := range ns.table {
-		n.execFin()
-	}
-	if closed {
-		ns.state = nsClosed
-	} else {
-		ns.state = nsZapped
-	}
-	ns.table = nil
-}
-
-func (ns *emptyNS) Zap(closed bool) {
-	ns.cache.Lock()
-	ns.zapNB(closed)
-	delete(ns.cache.table, ns.id)
-	ns.cache.Unlock()
-}
-
-type emptyNode struct {
-	ns       *emptyNS
-	key      uint64
-	value    interface{}
-	ref      int
-	setfin   SetFin
-	delfin   DelFin
-	purgefin PurgeFin
-}
-
-func (n *emptyNode) execFin() {
-	if n.setfin != nil {
-		n.setfin()
-		n.setfin = nil
-	}
-	if n.purgefin != nil {
-		n.purgefin(n.ns.id, n.key, n.delfin)
-		n.delfin = nil
-		n.purgefin = nil
-	} else if n.delfin != nil {
-		n.delfin(true)
-		n.delfin = nil
-	}
-}
-
-func (n *emptyNode) evict() {
-	n.ns.cache.Lock()
-	n.ref--
-	if n.ref == 0 {
-		if n.ns.state == nsEffective {
-			// Remove elem.
-			delete(n.ns.table, n.key)
-			// Execute finalizer.
-			n.execFin()
-		}
-	} else if n.ref < 0 {
-		panic("leveldb/cache: emptyNode: negative node reference")
-	}
-	n.ns.cache.Unlock()
-}
-
-type emptyObject struct {
-	node *emptyNode
-	once uint32
-}
-
-func (o *emptyObject) Value() interface{} {
-	if atomic.LoadUint32(&o.once) == 0 {
-		return o.node.value
-	}
-	return nil
-}
-
-func (o *emptyObject) Release() {
-	if !atomic.CompareAndSwapUint32(&o.once, 0, 1) {
-		return
-	}
-	o.node.evict()
-	o.node = nil
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru.go

@@ -0,0 +1,195 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package cache
+
+import (
+	"sync"
+	"unsafe"
+)
+
+type lruNode struct {
+	n   *Node
+	h   *Handle
+	ban bool
+
+	next, prev *lruNode
+}
+
+func (n *lruNode) insert(at *lruNode) {
+	x := at.next
+	at.next = n
+	n.prev = at
+	n.next = x
+	x.prev = n
+}
+
+func (n *lruNode) remove() {
+	if n.prev != nil {
+		n.prev.next = n.next
+		n.next.prev = n.prev
+		n.prev = nil
+		n.next = nil
+	} else {
+		panic("BUG: removing removed node")
+	}
+}
+
+type lru struct {
+	mu       sync.Mutex
+	capacity int
+	used     int
+	recent   lruNode
+}
+
+func (r *lru) reset() {
+	r.recent.next = &r.recent
+	r.recent.prev = &r.recent
+	r.used = 0
+}
+
+func (r *lru) Capacity() int {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	return r.capacity
+}
+
+func (r *lru) SetCapacity(capacity int) {
+	var evicted []*lruNode
+
+	r.mu.Lock()
+	r.capacity = capacity
+	for r.used > r.capacity {
+		rn := r.recent.prev
+		if rn == nil {
+			panic("BUG: invalid LRU used or capacity counter")
+		}
+		rn.remove()
+		rn.n.CacheData = nil
+		r.used -= rn.n.Size()
+		evicted = append(evicted, rn)
+	}
+	r.mu.Unlock()
+
+	for _, rn := range evicted {
+		rn.h.Release()
+	}
+}
+
+func (r *lru) Promote(n *Node) {
+	var evicted []*lruNode
+
+	r.mu.Lock()
+	if n.CacheData == nil {
+		if n.Size() <= r.capacity {
+			rn := &lruNode{n: n, h: n.GetHandle()}
+			rn.insert(&r.recent)
+			n.CacheData = unsafe.Pointer(rn)
+			r.used += n.Size()
+
+			for r.used > r.capacity {
+				rn := r.recent.prev
+				if rn == nil {
+					panic("BUG: invalid LRU used or capacity counter")
+				}
+				rn.remove()
+				rn.n.CacheData = nil
+				r.used -= rn.n.Size()
+				evicted = append(evicted, rn)
+			}
+		}
+	} else {
+		rn := (*lruNode)(n.CacheData)
+		if !rn.ban {
+			rn.remove()
+			rn.insert(&r.recent)
+		}
+	}
+	r.mu.Unlock()
+
+	for _, rn := range evicted {
+		rn.h.Release()
+	}
+}
+
+func (r *lru) Ban(n *Node) {
+	r.mu.Lock()
+	if n.CacheData == nil {
+		n.CacheData = unsafe.Pointer(&lruNode{n: n, ban: true})
+	} else {
+		rn := (*lruNode)(n.CacheData)
+		if !rn.ban {
+			rn.remove()
+			rn.ban = true
+			r.used -= rn.n.Size()
+			r.mu.Unlock()
+
+			rn.h.Release()
+			rn.h = nil
+			return
+		}
+	}
+	r.mu.Unlock()
+}
+
+func (r *lru) Evict(n *Node) {
+	r.mu.Lock()
+	rn := (*lruNode)(n.CacheData)
+	if rn == nil || rn.ban {
+		r.mu.Unlock()
+		return
+	}
+	n.CacheData = nil
+	r.mu.Unlock()
+
+	rn.h.Release()
+}
+
+func (r *lru) EvictNS(ns uint64) {
+	var evicted []*lruNode
+
+	r.mu.Lock()
+	for e := r.recent.prev; e != &r.recent; {
+		rn := e
+		e = e.prev
+		if rn.n.NS() == ns {
+			rn.remove()
+			rn.n.CacheData = nil
+			r.used -= rn.n.Size()
+			evicted = append(evicted, rn)
+		}
+	}
+	r.mu.Unlock()
+
+	for _, rn := range evicted {
+		rn.h.Release()
+	}
+}
+
+func (r *lru) EvictAll() {
+	r.mu.Lock()
+	back := r.recent.prev
+	for rn := back; rn != &r.recent; rn = rn.prev {
+		rn.n.CacheData = nil
+	}
+	r.reset()
+	r.mu.Unlock()
+
+	for rn := back; rn != &r.recent; rn = rn.prev {
+		rn.h.Release()
+	}
+}
+
+func (r *lru) Close() error {
+	return nil
+}
+
+// NewLRU create a new LRU-cache.
+func NewLRU(capacity int) Cacher {
+	r := &lru{capacity: capacity}
+	r.reset()
+	return r
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru_cache.go

@@ -1,354 +0,0 @@
-// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package cache
-
-import (
-	"sync"
-	"sync/atomic"
-)
-
-// lruCache represent a LRU cache state.
-type lruCache struct {
-	sync.Mutex
-
-	recent   lruNode
-	table    map[uint64]*lruNs
-	capacity int
-	size     int
-}
-
-// NewLRUCache creates a new initialized LRU cache with the given capacity.
-func NewLRUCache(capacity int) Cache {
-	c := &lruCache{
-		table:    make(map[uint64]*lruNs),
-		capacity: capacity,
-	}
-	c.recent.rNext = &c.recent
-	c.recent.rPrev = &c.recent
-	return c
-}
-
-// SetCapacity set cache capacity.
-func (c *lruCache) SetCapacity(capacity int) {
-	c.Lock()
-	c.capacity = capacity
-	c.evict()
-	c.Unlock()
-}
-
-// GetNamespace return namespace object for given id.
-func (c *lruCache) GetNamespace(id uint64) Namespace {
-	c.Lock()
-	defer c.Unlock()
-
-	if p, ok := c.table[id]; ok {
-		return p
-	}
-
-	p := &lruNs{
-		lru:   c,
-		id:    id,
-		table: make(map[uint64]*lruNode),
-	}
-	c.table[id] = p
-	return p
-}
-
-// Purge purge entire cache.
-func (c *lruCache) Purge(fin PurgeFin) {
-	c.Lock()
-	for _, ns := range c.table {
-		ns.purgeNB(fin)
-	}
-	c.Unlock()
-}
-
-func (c *lruCache) Zap(closed bool) {
-	c.Lock()
-	for _, ns := range c.table {
-		ns.zapNB(closed)
-	}
-	c.table = make(map[uint64]*lruNs)
-	c.Unlock()
-}
-
-func (c *lruCache) evict() {
-	top := &c.recent
-	for n := c.recent.rPrev; c.size > c.capacity && n != top; {
-		n.state = nodeEvicted
-		n.rRemove()
-		n.evictNB()
-		c.size -= n.charge
-		n = c.recent.rPrev
-	}
-}
-
-type lruNs struct {
-	lru   *lruCache
-	id    uint64
-	table map[uint64]*lruNode
-	state nsState
-}
-
-func (ns *lruNs) Get(key uint64, setf SetFunc) (o Object, ok bool) {
-	lru := ns.lru
-	lru.Lock()
-
-	switch ns.state {
-	case nsZapped:
-		lru.Unlock()
-		if setf == nil {
-			return
-		}
-
-		var value interface{}
-		var fin func()
-		ok, value, _, fin = setf()
-		if ok {
-			o = &fakeObject{
-				value: value,
-				fin:   fin,
-			}
-		}
-		return
-	case nsClosed:
-		lru.Unlock()
-		return
-	}
-
-	n, ok := ns.table[key]
-	if ok {
-		switch n.state {
-		case nodeEvicted:
-			// Insert to recent list.
-			n.state = nodeEffective
-			n.ref++
-			lru.size += n.charge
-			lru.evict()
-			fallthrough
-		case nodeEffective:
-			// Bump to front
-			n.rRemove()
-			n.rInsert(&lru.recent)
-		}
-		n.ref++
-	} else {
-		if setf == nil {
-			lru.Unlock()
-			return
-		}
-
-		var value interface{}
-		var charge int
-		var fin func()
-		ok, value, charge, fin = setf()
-		if !ok {
-			lru.Unlock()
-			return
-		}
-
-		n = &lruNode{
-			ns:     ns,
-			key:    key,
-			value:  value,
-			charge: charge,
-			setfin: fin,
-			ref:    2,
-		}
-		ns.table[key] = n
-		n.rInsert(&lru.recent)
-
-		lru.size += charge
-		lru.evict()
-	}
-
-	lru.Unlock()
-	o = &lruObject{node: n}
-	return
-}
-
-func (ns *lruNs) Delete(key uint64, fin DelFin) bool {
-	lru := ns.lru
-	lru.Lock()
-
-	if ns.state != nsEffective {
-		lru.Unlock()
-		if fin != nil {
-			fin(false)
-		}
-		return false
-	}
-
-	n, ok := ns.table[key]
-	if !ok {
-		lru.Unlock()
-		if fin != nil {
-			fin(false)
-		}
-		return false
-	}
-
-	n.delfin = fin
-	switch n.state {
-	case nodeRemoved:
-		lru.Unlock()
-		return false
-	case nodeEffective:
-		lru.size -= n.charge
-		n.rRemove()
-		n.evictNB()
-	}
-	n.state = nodeRemoved
-
-	lru.Unlock()
-	return true
-}
-
-func (ns *lruNs) purgeNB(fin PurgeFin) {
-	lru := ns.lru
-	if ns.state != nsEffective {
-		return
-	}
-
-	for _, n := range ns.table {
-		n.purgefin = fin
-		if n.state == nodeEffective {
-			lru.size -= n.charge
-			n.rRemove()
-			n.evictNB()
-		}
-		n.state = nodeRemoved
-	}
-}
-
-func (ns *lruNs) Purge(fin PurgeFin) {
-	ns.lru.Lock()
-	ns.purgeNB(fin)
-	ns.lru.Unlock()
-}
-
-func (ns *lruNs) zapNB(closed bool) {
-	lru := ns.lru
-	if ns.state != nsEffective {
-		return
-	}
-
-	if closed {
-		ns.state = nsClosed
-	} else {
-		ns.state = nsZapped
-	}
-	for _, n := range ns.table {
-		if n.state == nodeEffective {
-			lru.size -= n.charge
-			n.rRemove()
-		}
-		n.state = nodeRemoved
-		n.execFin()
-	}
-	ns.table = nil
-}
-
-func (ns *lruNs) Zap(closed bool) {
-	ns.lru.Lock()
-	ns.zapNB(closed)
-	delete(ns.lru.table, ns.id)
-	ns.lru.Unlock()
-}
-
-type lruNode struct {
-	ns *lruNs
-
-	rNext, rPrev *lruNode
-
-	key      uint64
-	value    interface{}
-	charge   int
-	ref      int
-	state    nodeState
-	setfin   SetFin
-	delfin   DelFin
-	purgefin PurgeFin
-}
-
-func (n *lruNode) rInsert(at *lruNode) {
-	x := at.rNext
-	at.rNext = n
-	n.rPrev = at
-	n.rNext = x
-	x.rPrev = n
-}
-
-func (n *lruNode) rRemove() bool {
-	// only remove if not already removed
-	if n.rPrev == nil {
-		return false
-	}
-
-	n.rPrev.rNext = n.rNext
-	n.rNext.rPrev = n.rPrev
-	n.rPrev = nil
-	n.rNext = nil
-
-	return true
-}
-
-func (n *lruNode) execFin() {
-	if n.setfin != nil {
-		n.setfin()
-		n.setfin = nil
-	}
-	if n.purgefin != nil {
-		n.purgefin(n.ns.id, n.key, n.delfin)
-		n.delfin = nil
-		n.purgefin = nil
-	} else if n.delfin != nil {
-		n.delfin(true)
-		n.delfin = nil
-	}
-}
-
-func (n *lruNode) evictNB() {
-	n.ref--
-	if n.ref == 0 {
-		if n.ns.state == nsEffective {
-			// remove elem
-			delete(n.ns.table, n.key)
-			// execute finalizer
-			n.execFin()
-		}
-	} else if n.ref < 0 {
-		panic("leveldb/cache: lruCache: negative node reference")
-	}
-}
-
-func (n *lruNode) evict() {
-	n.ns.lru.Lock()
-	n.evictNB()
-	n.ns.lru.Unlock()
-}
-
-type lruObject struct {
-	node *lruNode
-	once uint32
-}
-
-func (o *lruObject) Value() interface{} {
-	if atomic.LoadUint32(&o.once) == 0 {
-		return o.node.value
-	}
-	return nil
-}
-
-func (o *lruObject) Release() {
-	if !atomic.CompareAndSwapUint32(&o.once, 0, 1) {
-		return
-	}
-
-	o.node.evict()
-	o.node = nil
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/config.go

@@ -1,40 +0,0 @@
-// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package leveldb
-
-const (
-	kNumLevels = 7
-
-	// Level-0 compaction is started when we hit this many files.
-	kL0_CompactionTrigger float64 = 4
-
-	// Soft limit on number of level-0 files.  We slow down writes at this point.
-	kL0_SlowdownWritesTrigger = 8
-
-	// Maximum number of level-0 files.  We stop writes at this point.
-	kL0_StopWritesTrigger = 12
-
-	// Maximum level to which a new compacted memdb is pushed if it
-	// does not create overlap.  We try to push to level 2 to avoid the
-	// relatively expensive level 0=>1 compactions and to avoid some
-	// expensive manifest file operations.  We do not push all the way to
-	// the largest level since that can generate a lot of wasted disk
-	// space if the same key space is being repeatedly overwritten.
-	kMaxMemCompactLevel = 2
-
-	// Maximum size of a table.
-	kMaxTableSize = 2 * 1048576
-
-	// Maximum bytes of overlaps in grandparent (i.e., level+2) before we
-	// stop building a single file in a level->level+1 compaction.
-	kMaxGrandParentOverlapBytes = 10 * kMaxTableSize
-
-	// Maximum number of bytes in all compacted files.  We avoid expanding
-	// the lower level file set of a compaction if it would make the
-	// total compaction cover more than this many bytes.
-	kExpCompactionMaxBytes = 25 * kMaxTableSize
-)

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/corrupt_test.go

@@ -9,13 +9,12 @@ package leveldb
 import (
 	"bytes"
 	"fmt"
+	"github.com/syndtr/goleveldb/leveldb/filter"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/storage"
 	"io"
 	"math/rand"
 	"testing"
-
-	"github.com/syndtr/goleveldb/leveldb/cache"
-	"github.com/syndtr/goleveldb/leveldb/opt"
-	"github.com/syndtr/goleveldb/leveldb/storage"
 )
 
 const ctValSize = 1000
@@ -32,8 +31,8 @@ func newDbCorruptHarnessWopt(t *testing.T, o *opt.Options) *dbCorruptHarness {
 
 func newDbCorruptHarness(t *testing.T) *dbCorruptHarness {
 	return newDbCorruptHarnessWopt(t, &opt.Options{
-		BlockCache: cache.NewLRUCache(100),
-		Strict:     opt.StrictJournalChecksum,
+		BlockCacheCapacity: 100,
+		Strict:             opt.StrictJournalChecksum,
 	})
 }
 
@@ -96,21 +95,22 @@ func (h *dbCorruptHarness) deleteRand(n, max int, rnd *rand.Rand) {
 	}
 }
 
-func (h *dbCorruptHarness) corrupt(ft storage.FileType, offset, n int) {
+func (h *dbCorruptHarness) corrupt(ft storage.FileType, fi, offset, n int) {
 	p := &h.dbHarness
 	t := p.t
 
-	var file storage.File
 	ff, _ := p.stor.GetFiles(ft)
-	for _, f := range ff {
-		if file == nil || f.Num() > file.Num() {
-			file = f
-		}
+	sff := files(ff)
+	sff.sort()
+	if fi < 0 {
+		fi = len(sff) - 1
 	}
-	if file == nil {
-		t.Fatalf("no such file with type %q", ft)
+	if fi >= len(sff) {
+		t.Fatalf("no such file with type %q with index %d", ft, fi)
 	}
 
+	file := sff[fi]
+
 	r, err := file.Open()
 	if err != nil {
 		t.Fatal("cannot open file: ", err)
@@ -225,8 +225,8 @@ func TestCorruptDB_Journal(t *testing.T) {
 	h.build(100)
 	h.check(100, 100)
 	h.closeDB()
-	h.corrupt(storage.TypeJournal, 19, 1)
-	h.corrupt(storage.TypeJournal, 32*1024+1000, 1)
+	h.corrupt(storage.TypeJournal, -1, 19, 1)
+	h.corrupt(storage.TypeJournal, -1, 32*1024+1000, 1)
 
 	h.openDB()
 	h.check(36, 36)
@@ -242,7 +242,7 @@ func TestCorruptDB_Table(t *testing.T) {
 	h.compactRangeAt(0, "", "")
 	h.compactRangeAt(1, "", "")
 	h.closeDB()
-	h.corrupt(storage.TypeTable, 100, 1)
+	h.corrupt(storage.TypeTable, -1, 100, 1)
 
 	h.openDB()
 	h.check(99, 99)
@@ -256,7 +256,7 @@ func TestCorruptDB_TableIndex(t *testing.T) {
 	h.build(10000)
 	h.compactMem()
 	h.closeDB()
-	h.corrupt(storage.TypeTable, -2000, 500)
+	h.corrupt(storage.TypeTable, -1, -2000, 500)
 
 	h.openDB()
 	h.check(5000, 9999)
@@ -267,9 +267,9 @@ func TestCorruptDB_TableIndex(t *testing.T) {
 func TestCorruptDB_MissingManifest(t *testing.T) {
 	rnd := rand.New(rand.NewSource(0x0badda7a))
 	h := newDbCorruptHarnessWopt(t, &opt.Options{
-		BlockCache:  cache.NewLRUCache(100),
-		Strict:      opt.StrictJournalChecksum,
-		WriteBuffer: 1000 * 60,
+		BlockCacheCapacity: 100,
+		Strict:             opt.StrictJournalChecksum,
+		WriteBuffer:        1000 * 60,
 	})
 
 	h.build(1000)
@@ -355,7 +355,7 @@ func TestCorruptDB_CorruptedManifest(t *testing.T) {
 	h.compactMem()
 	h.compactRange("", "")
 	h.closeDB()
-	h.corrupt(storage.TypeManifest, 0, 1000)
+	h.corrupt(storage.TypeManifest, -1, 0, 1000)
 	h.openAssert(false)
 
 	h.recover()
@@ -370,7 +370,7 @@ func TestCorruptDB_CompactionInputError(t *testing.T) {
 	h.build(10)
 	h.compactMem()
 	h.closeDB()
-	h.corrupt(storage.TypeTable, 100, 1)
+	h.corrupt(storage.TypeTable, -1, 100, 1)
 
 	h.openDB()
 	h.check(9, 9)
@@ -387,7 +387,7 @@ func TestCorruptDB_UnrelatedKeys(t *testing.T) {
 	h.build(10)
 	h.compactMem()
 	h.closeDB()
-	h.corrupt(storage.TypeTable, 100, 1)
+	h.corrupt(storage.TypeTable, -1, 100, 1)
 
 	h.openDB()
 	h.put(string(tkey(1000)), string(tval(1000, ctValSize)))
@@ -470,3 +470,31 @@ func TestCorruptDB_MissingTableFiles(t *testing.T) {
 
 	h.close()
 }
+
+func TestCorruptDB_RecoverTable(t *testing.T) {
+	h := newDbCorruptHarnessWopt(t, &opt.Options{
+		WriteBuffer:         112 * opt.KiB,
+		CompactionTableSize: 90 * opt.KiB,
+		Filter:              filter.NewBloomFilter(10),
+	})
+
+	h.build(1000)
+	h.compactMem()
+	h.compactRangeAt(0, "", "")
+	h.compactRangeAt(1, "", "")
+	seq := h.db.seq
+	h.closeDB()
+	h.corrupt(storage.TypeTable, 0, 1000, 1)
+	h.corrupt(storage.TypeTable, 3, 10000, 1)
+	// Corrupted filter shouldn't affect recovery.
+	h.corrupt(storage.TypeTable, 3, 113888, 10)
+	h.corrupt(storage.TypeTable, -1, 20000, 1)
+
+	h.recover()
+	if h.db.seq != seq {
+		t.Errorf("invalid seq, want=%d got=%d", seq, h.db.seq)
+	}
+	h.check(985, 985)
+
+	h.close()
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go

@@ -7,15 +7,17 @@
 package leveldb
 
 import (
-	"errors"
+	"container/list"
 	"fmt"
 	"io"
 	"os"
 	"runtime"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
+	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/journal"
 	"github.com/syndtr/goleveldb/leveldb/memdb"
@@ -35,8 +37,8 @@ type DB struct {
 
 	// MemDB.
 	memMu             sync.RWMutex
-	mem               *memdb.DB
-	frozenMem         *memdb.DB
+	memPool           chan *memdb.DB
+	mem, frozenMem    *memDB
 	journal           *journal.Writer
 	journalWriter     storage.Writer
 	journalFile       storage.File
@@ -45,25 +47,29 @@ type DB struct {
 
 	// Snapshot.
 	snapsMu   sync.Mutex
-	snapsRoot snapshotElement
+	snapsList *list.List
+
+	// Stats.
+	aliveSnaps, aliveIters int32
 
 	// Write.
 	writeC       chan *Batch
 	writeMergedC chan bool
 	writeLockC   chan struct{}
 	writeAckC    chan error
+	writeDelay   time.Duration
+	writeDelayN  int
 	journalC     chan *Batch
 	journalAckC  chan error
 
 	// Compaction.
-	tcompCmdC     chan cCmd
-	tcompPauseC   chan chan<- struct{}
-	tcompTriggerC chan struct{}
-	mcompCmdC     chan cCmd
-	mcompTriggerC chan struct{}
-	compErrC      chan error
-	compErrSetC   chan error
-	compStats     [kNumLevels]cStats
+	tcompCmdC   chan cCmd
+	tcompPauseC chan chan<- struct{}
+	mcompCmdC   chan cCmd
+	compErrC    chan error
+	compPerErrC chan error
+	compErrSetC chan error
+	compStats   []cStats
 
 	// Close.
 	closeW sync.WaitGroup
@@ -78,7 +84,11 @@ func openDB(s *session) (*DB, error) {
 	db := &DB{
 		s: s,
 		// Initial sequence
-		seq: s.stSeq,
+		seq: s.stSeqNum,
+		// MemDB
+		memPool: make(chan *memdb.DB, 1),
+		// Snapshot
+		snapsList: list.New(),
 		// Write
 		writeC:       make(chan *Batch),
 		writeMergedC: make(chan bool),
@@ -87,17 +97,16 @@ func openDB(s *session) (*DB, error) {
 		journalC:     make(chan *Batch),
 		journalAckC:  make(chan error),
 		// Compaction
-		tcompCmdC:     make(chan cCmd),
-		tcompPauseC:   make(chan chan<- struct{}),
-		tcompTriggerC: make(chan struct{}, 1),
-		mcompCmdC:     make(chan cCmd),
-		mcompTriggerC: make(chan struct{}, 1),
-		compErrC:      make(chan error),
-		compErrSetC:   make(chan error),
+		tcompCmdC:   make(chan cCmd),
+		tcompPauseC: make(chan chan<- struct{}),
+		mcompCmdC:   make(chan cCmd),
+		compErrC:    make(chan error),
+		compPerErrC: make(chan error),
+		compErrSetC: make(chan error),
+		compStats:   make([]cStats, s.o.GetNumLevel()),
 		// Close
 		closeC: make(chan struct{}),
 	}
-	db.initSnapshot()
 
 	if err := db.recoverJournal(); err != nil {
 		return nil, err
@@ -113,8 +122,9 @@ func openDB(s *session) (*DB, error) {
 		return nil, err
 	}
 
-	// Don't include compaction error goroutine into wait group.
+	// Doesn't need to be included in the wait group.
 	go db.compactionError()
+	go db.mpoolDrain()
 
 	db.closeW.Add(3)
 	go db.tCompaction()
@@ -246,6 +256,10 @@ func RecoverFile(path string, o *opt.Options) (db *DB, err error) {
 }
 
 func recoverTable(s *session, o *opt.Options) error {
+	o = dupOptions(o)
+	// Mask StrictReader, lets StrictRecovery doing its job.
+	o.Strict &= ^opt.StrictReader
+
 	// Get all tables and sort it by file number.
 	tableFiles_, err := s.getFiles(storage.TypeTable)
 	if err != nil {
@@ -254,9 +268,16 @@ func recoverTable(s *session, o *opt.Options) error {
 	tableFiles := files(tableFiles_)
 	tableFiles.sort()
 
-	var mSeq uint64
-	var good, corrupted int
-	rec := new(sessionRecord)
+	var (
+		maxSeq                                                            uint64
+		recoveredKey, goodKey, corruptedKey, corruptedBlock, droppedTable int
+
+		// We will drop corrupted table.
+		strict = o.GetStrict(opt.StrictRecovery)
+
+		rec   = &sessionRecord{numLevel: o.GetNumLevel()}
+		bpool = util.NewBufferPool(o.GetBlockSize() + 5)
+	)
 	buildTable := func(iter iterator.Iterator) (tmp storage.File, size int64, err error) {
 		tmp = s.newTemp()
 		writer, err := tmp.Create()
@@ -303,7 +324,12 @@ func recoverTable(s *session, o *opt.Options) error {
 		if err != nil {
 			return err
 		}
-		defer reader.Close()
+		var closed bool
+		defer func() {
+			if !closed {
+				reader.Close()
+			}
+		}()
 
 		// Get file size.
 		size, err := reader.Seek(0, 2)
@@ -311,25 +337,32 @@ func recoverTable(s *session, o *opt.Options) error {
 			return err
 		}
 
-		var tSeq uint64
-		var tgood, tcorrupted, blockerr int
-		var imin, imax []byte
-		tr := table.NewReader(reader, size, nil, o)
+		var (
+			tSeq                                     uint64
+			tgoodKey, tcorruptedKey, tcorruptedBlock int
+			imin, imax                               []byte
+		)
+		tr, err := table.NewReader(reader, size, storage.NewFileInfo(file), nil, bpool, o)
+		if err != nil {
+			return err
+		}
 		iter := tr.NewIterator(nil, nil)
 		iter.(iterator.ErrorCallbackSetter).SetErrorCallback(func(err error) {
-			s.logf("table@recovery found error @%d %q", file.Num(), err)
-			blockerr++
+			if errors.IsCorrupted(err) {
+				s.logf("table@recovery block corruption @%d %q", file.Num(), err)
+				tcorruptedBlock++
+			}
 		})
 
 		// Scan the table.
 		for iter.Next() {
 			key := iter.Key()
-			_, seq, _, ok := parseIkey(key)
-			if !ok {
-				tcorrupted++
+			_, seq, _, kerr := parseIkey(key)
+			if kerr != nil {
+				tcorruptedKey++
 				continue
 			}
-			tgood++
+			tgoodKey++
 			if seq > tSeq {
 				tSeq = seq
 			}
@@ -344,8 +377,18 @@ func recoverTable(s *session, o *opt.Options) error {
 		}
 		iter.Release()
 
-		if tgood > 0 {
-			if tcorrupted > 0 || blockerr > 0 {
+		goodKey += tgoodKey
+		corruptedKey += tcorruptedKey
+		corruptedBlock += tcorruptedBlock
+
+		if strict && (tcorruptedKey > 0 || tcorruptedBlock > 0) {
+			droppedTable++
+			s.logf("table@recovery dropped @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", file.Num(), tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq)
+			return nil
+		}
+
+		if tgoodKey > 0 {
+			if tcorruptedKey > 0 || tcorruptedBlock > 0 {
 				// Rebuild the table.
 				s.logf("table@recovery rebuilding @%d", file.Num())
 				iter := tr.NewIterator(nil, nil)
@@ -354,25 +397,25 @@ func recoverTable(s *session, o *opt.Options) error {
 				if err != nil {
 					return err
 				}
+				closed = true
 				reader.Close()
 				if err := file.Replace(tmp); err != nil {
 					return err
 				}
 				size = newSize
 			}
-			if tSeq > mSeq {
-				mSeq = tSeq
+			if tSeq > maxSeq {
+				maxSeq = tSeq
 			}
+			recoveredKey += tgoodKey
 			// Add table to level 0.
 			rec.addTable(0, file.Num(), uint64(size), imin, imax)
-			s.logf("table@recovery recovered @%d N·%d C·%d B·%d S·%d Q·%d", file.Num(), tgood, tcorrupted, blockerr, size, tSeq)
+			s.logf("table@recovery recovered @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", file.Num(), tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq)
 		} else {
-			s.logf("table@recovery unrecoverable @%d C·%d B·%d S·%d", file.Num(), tcorrupted, blockerr, size)
+			droppedTable++
+			s.logf("table@recovery unrecoverable @%d Ck·%d Cb·%d S·%d", file.Num(), tcorruptedKey, tcorruptedBlock, size)
 		}
 
-		good += tgood
-		corrupted += tcorrupted
-
 		return nil
 	}
 
@@ -389,11 +432,11 @@ func recoverTable(s *session, o *opt.Options) error {
 			}
 		}
 
-		s.logf("table@recovery recovered F·%d N·%d C·%d Q·%d", len(tableFiles), good, corrupted, mSeq)
+		s.logf("table@recovery recovered F·%d N·%d Gk·%d Ck·%d Q·%d", len(tableFiles), recoveredKey, goodKey, corruptedKey, maxSeq)
 	}
 
 	// Set sequence number.
-	rec.setSeq(mSeq + 1)
+	rec.setSeqNum(maxSeq)
 
 	// Create new manifest.
 	if err := s.create(); err != nil {
@@ -476,25 +519,30 @@ func (db *DB) recoverJournal() error {
 				if err == io.EOF {
 					break
 				}
-				return err
+				return errors.SetFile(err, file)
 			}
 
 			buf.Reset()
 			if _, err := buf.ReadFrom(r); err != nil {
-				if strict {
-					return err
+				if err == io.ErrUnexpectedEOF {
+					// This is error returned due to corruption, with strict == false.
+					continue
+				} else {
+					return errors.SetFile(err, file)
 				}
-				continue
 			}
-			if err := batch.decode(buf.Bytes()); err != nil {
-				return err
-			}
-			if err := batch.memReplay(mem); err != nil {
-				return err
+			if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mem); err != nil {
+				if strict || !errors.IsCorrupted(err) {
+					return errors.SetFile(err, file)
+				} else {
+					db.s.logf("journal error: %v (skipped)", err)
+					// We won't apply sequence number as it might be corrupted.
+					continue
+				}
 			}
 
 			// Save sequence number.
-			db.seq = batch.seq + uint64(batch.len())
+			db.seq = batch.seq + uint64(batch.Len())
 
 			// Flush it if large enough.
 			if mem.Size() >= writeBuffer {
@@ -555,22 +603,27 @@ func (db *DB) recoverJournal() error {
 }
 
 func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, err error) {
-	ikey := newIKey(key, seq, tSeek)
+	ikey := newIkey(key, seq, ktSeek)
 
 	em, fm := db.getMems()
-	for _, m := range [...]*memdb.DB{em, fm} {
+	for _, m := range [...]*memDB{em, fm} {
 		if m == nil {
 			continue
 		}
+		defer m.decref()
 
-		mk, mv, me := m.Find(ikey)
+		mk, mv, me := m.mdb.Find(ikey)
 		if me == nil {
-			ukey, _, t, ok := parseIkey(mk)
-			if ok && db.s.icmp.uCompare(ukey, key) == 0 {
-				if t == tDel {
+			ukey, _, kt, kerr := parseIkey(mk)
+			if kerr != nil {
+				// Shouldn't have had happen.
+				panic(kerr)
+			}
+			if db.s.icmp.uCompare(ukey, key) == 0 {
+				if kt == ktDel {
 					return nil, ErrNotFound
 				}
-				return mv, nil
+				return append([]byte{}, mv...), nil
 			}
 		} else if me != ErrNotFound {
 			return nil, me
@@ -578,27 +631,87 @@ func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, er
 	}
 
 	v := db.s.version()
-	value, cSched, err := v.get(ikey, ro)
+	value, cSched, err := v.get(ikey, ro, false)
 	v.release()
 	if cSched {
 		// Trigger table compaction.
-		db.compTrigger(db.tcompTriggerC)
+		db.compSendTrigger(db.tcompCmdC)
+	}
+	return
+}
+
+func (db *DB) has(key []byte, seq uint64, ro *opt.ReadOptions) (ret bool, err error) {
+	ikey := newIkey(key, seq, ktSeek)
+
+	em, fm := db.getMems()
+	for _, m := range [...]*memDB{em, fm} {
+		if m == nil {
+			continue
+		}
+		defer m.decref()
+
+		mk, _, me := m.mdb.Find(ikey)
+		if me == nil {
+			ukey, _, kt, kerr := parseIkey(mk)
+			if kerr != nil {
+				// Shouldn't have had happen.
+				panic(kerr)
+			}
+			if db.s.icmp.uCompare(ukey, key) == 0 {
+				if kt == ktDel {
+					return false, nil
+				}
+				return true, nil
+			}
+		} else if me != ErrNotFound {
+			return false, me
+		}
+	}
+
+	v := db.s.version()
+	_, cSched, err := v.get(ikey, ro, true)
+	v.release()
+	if cSched {
+		// Trigger table compaction.
+		db.compSendTrigger(db.tcompCmdC)
+	}
+	if err == nil {
+		ret = true
+	} else if err == ErrNotFound {
+		err = nil
 	}
 	return
 }
 
 // Get gets the value for the given key. It returns ErrNotFound if the
-// DB does not contain the key.
+// DB does not contains the key.
 //
-// The caller should not modify the contents of the returned slice, but
-// it is safe to modify the contents of the argument after Get returns.
+// The returned slice is its own copy, it is safe to modify the contents
+// of the returned slice.
+// It is safe to modify the contents of the argument after Get returns.
 func (db *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
 	err = db.ok()
 	if err != nil {
 		return
 	}
 
-	return db.get(key, db.getSeq(), ro)
+	se := db.acquireSnapshot()
+	defer db.releaseSnapshot(se)
+	return db.get(key, se.seq, ro)
+}
+
+// Has returns true if the DB does contains the given key.
+//
+// It is safe to modify the contents of the argument after Get returns.
+func (db *DB) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) {
+	err = db.ok()
+	if err != nil {
+		return
+	}
+
+	se := db.acquireSnapshot()
+	defer db.releaseSnapshot(se)
+	return db.has(key, se.seq, ro)
 }
 
 // NewIterator returns an iterator for the latest snapshot of the
@@ -622,9 +735,11 @@ func (db *DB) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Itera
 		return iterator.NewEmptyIterator(err)
 	}
 
-	snap := db.newSnapshot()
-	defer snap.Release()
-	return snap.NewIterator(slice, ro)
+	se := db.acquireSnapshot()
+	defer db.releaseSnapshot(se)
+	// Iterator holds 'version' lock, 'version' is immutable so snapshot
+	// can be released after iterator created.
+	return db.newIterator(se.seq, slice, ro)
 }
 
 // GetSnapshot returns a latest snapshot of the underlying DB. A snapshot
@@ -644,11 +759,21 @@ func (db *DB) GetSnapshot() (*Snapshot, error) {
 //
 // Property names:
 //	leveldb.num-files-at-level{n}
-//		Returns the number of filer at level 'n'.
+//		Returns the number of files at level 'n'.
 //	leveldb.stats
 //		Returns statistics of the underlying DB.
 //	leveldb.sstables
 //		Returns sstables list for each level.
+//	leveldb.blockpool
+//		Returns block pool stats.
+//	leveldb.cachedblock
+//		Returns size of cached block.
+//	leveldb.openedtables
+//		Returns number of opened tables.
+//	leveldb.alivesnaps
+//		Returns number of alive snapshots.
+//	leveldb.aliveiters
+//		Returns number of alive iterators.
 func (db *DB) GetProperty(name string) (value string, err error) {
 	err = db.ok()
 	if err != nil {
@@ -664,12 +789,13 @@ func (db *DB) GetProperty(name string) (value string, err error) {
 	v := db.s.version()
 	defer v.release()
 
+	numFilesPrefix := "num-files-at-level"
 	switch {
-	case strings.HasPrefix(p, "num-files-at-level"):
+	case strings.HasPrefix(p, numFilesPrefix):
 		var level uint
 		var rest string
-		n, _ := fmt.Scanf("%d%s", &level, &rest)
-		if n != 1 || level >= kNumLevels {
+		n, _ := fmt.Sscanf(p[len(numFilesPrefix):], "%d%s", &level, &rest)
+		if n != 1 || int(level) >= db.s.o.GetNumLevel() {
 			err = errors.New("leveldb: GetProperty: invalid property: " + name)
 		} else {
 			value = fmt.Sprint(v.tLen(int(level)))
@@ -694,6 +820,20 @@ func (db *DB) GetProperty(name string) (value string, err error) {
 				value += fmt.Sprintf("%d:%d[%q .. %q]\n", t.file.Num(), t.size, t.imin, t.imax)
 			}
 		}
+	case p == "blockpool":
+		value = fmt.Sprintf("%v", db.s.tops.bpool)
+	case p == "cachedblock":
+		if db.s.tops.bcache != nil {
+			value = fmt.Sprintf("%d", db.s.tops.bcache.Size())
+		} else {
+			value = "<nil>"
+		}
+	case p == "openedtables":
+		value = fmt.Sprintf("%d", db.s.tops.cache.Size())
+	case p == "alivesnaps":
+		value = fmt.Sprintf("%d", atomic.LoadInt32(&db.aliveSnaps))
+	case p == "aliveiters":
+		value = fmt.Sprintf("%d", atomic.LoadInt32(&db.aliveIters))
 	default:
 		err = errors.New("leveldb: GetProperty: unknown property: " + name)
 	}
@@ -717,8 +857,8 @@ func (db *DB) SizeOf(ranges []util.Range) (Sizes, error) {
 
 	sizes := make(Sizes, 0, len(ranges))
 	for _, r := range ranges {
-		imin := newIKey(r.Start, kMaxSeq, tSeek)
-		imax := newIKey(r.Limit, kMaxSeq, tSeek)
+		imin := newIkey(r.Start, kMaxSeq, ktSeek)
+		imax := newIkey(r.Limit, kMaxSeq, ktSeek)
 		start, err := v.offsetOf(imin)
 		if err != nil {
 			return nil, err
@@ -761,18 +901,23 @@ func (db *DB) Close() error {
 	default:
 	}
 
+	// Signal all goroutines.
 	close(db.closeC)
 
-	// Wait for the close WaitGroup.
+	// Wait for all gorotines to exit.
 	db.closeW.Wait()
 
-	// Close journal.
+	// Lock writer and closes journal.
 	db.writeLockC <- struct{}{}
 	if db.journal != nil {
 		db.journal.Close()
 		db.journalWriter.Close()
 	}
 
+	if db.writeDelayN > 0 {
+		db.logf("db@write was delayed N·%d T·%v", db.writeDelayN, db.writeDelay)
+	}
+
 	// Close session.
 	db.s.close()
 	db.logf("db@close done T·%v", time.Since(start))
@@ -792,7 +937,6 @@ func (db *DB) Close() error {
 	db.journalWriter = nil
 	db.journalFile = nil
 	db.frozenJournalFile = nil
-	db.snapsRoot = snapshotElement{}
 	db.closer = nil
 
 	return err

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go

@@ -7,11 +7,12 @@
 package leveldb
 
 import (
-	"errors"
 	"sync"
 	"time"
 
+	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/memdb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
 )
 
 var (
@@ -68,7 +69,7 @@ type cMem struct {
 }
 
 func newCMem(s *session) *cMem {
-	return &cMem{s: s, rec: new(sessionRecord)}
+	return &cMem{s: s, rec: &sessionRecord{numLevel: s.o.GetNumLevel()}}
 }
 
 func (c *cMem) flush(mem *memdb.DB, level int) error {
@@ -84,7 +85,9 @@ func (c *cMem) flush(mem *memdb.DB, level int) error {
 
 	// Pick level.
 	if level < 0 {
-		level = s.version_NB().pickLevel(t.imin.ukey(), t.imax.ukey())
+		v := s.version()
+		level = v.pickLevel(t.imin.ukey(), t.imax.ukey())
+		v.release()
 	}
 	c.rec.addTableFile(level, t)
 
@@ -95,24 +98,32 @@ func (c *cMem) flush(mem *memdb.DB, level int) error {
 }
 
 func (c *cMem) reset() {
-	c.rec = new(sessionRecord)
+	c.rec = &sessionRecord{numLevel: c.s.o.GetNumLevel()}
 }
 
 func (c *cMem) commit(journal, seq uint64) error {
 	c.rec.setJournalNum(journal)
-	c.rec.setSeq(seq)
+	c.rec.setSeqNum(seq)
 
 	// Commit changes.
 	return c.s.commit(c.rec)
 }
 
 func (db *DB) compactionError() {
-	var err error
+	var (
+		err     error
+		wlocked bool
+	)
 noerr:
+	// No error.
 	for {
 		select {
 		case err = <-db.compErrSetC:
-			if err != nil {
+			switch {
+			case err == nil:
+			case errors.IsCorrupted(err):
+				goto hasperr
+			default:
 				goto haserr
 			}
 		case _, _ = <-db.closeC:
@@ -120,17 +131,39 @@ noerr:
 		}
 	}
 haserr:
+	// Transient error.
 	for {
 		select {
 		case db.compErrC <- err:
 		case err = <-db.compErrSetC:
-			if err == nil {
+			switch {
+			case err == nil:
 				goto noerr
+			case errors.IsCorrupted(err):
+				goto hasperr
+			default:
 			}
 		case _, _ = <-db.closeC:
 			return
 		}
 	}
+hasperr:
+	// Persistent error.
+	for {
+		select {
+		case db.compErrC <- err:
+		case db.compPerErrC <- err:
+		case db.writeLockC <- struct{}{}:
+			// Hold write lock, so that write won't pass-through.
+			wlocked = true
+		case _, _ = <-db.closeC:
+			if wlocked {
+				// We should release the lock or Close will hang.
+				<-db.writeLockC
+			}
+			return
+		}
+	}
 }
 
 type compactionTransactCounter int
@@ -139,12 +172,17 @@ func (cnt *compactionTransactCounter) incr() {
 	*cnt++
 }
 
-func (db *DB) compactionTransact(name string, exec func(cnt *compactionTransactCounter) error, rollback func() error) {
+type compactionTransactInterface interface {
+	run(cnt *compactionTransactCounter) error
+	revert() error
+}
+
+func (db *DB) compactionTransact(name string, t compactionTransactInterface) {
 	defer func() {
 		if x := recover(); x != nil {
-			if x == errCompactionTransactExiting && rollback != nil {
-				if err := rollback(); err != nil {
-					db.logf("%s rollback error %q", name, err)
+			if x == errCompactionTransactExiting {
+				if err := t.revert(); err != nil {
+					db.logf("%s revert error %q", name, err)
 				}
 			}
 			panic(x)
@@ -156,9 +194,13 @@ func (db *DB) compactionTransact(name string, exec func(cnt *compactionTransactC
 		backoffMax = 8 * time.Second
 		backoffMul = 2 * time.Second
 	)
-	backoff := backoffMin
-	backoffT := time.NewTimer(backoff)
-	lastCnt := compactionTransactCounter(0)
+	var (
+		backoff  = backoffMin
+		backoffT = time.NewTimer(backoff)
+		lastCnt  = compactionTransactCounter(0)
+
+		disableBackoff = db.s.o.GetDisableCompactionBackoff()
+	)
 	for n := 0; ; n++ {
 		// Check wether the DB is closed.
 		if db.isClosed() {
@@ -170,11 +212,19 @@ func (db *DB) compactionTransact(name string, exec func(cnt *compactionTransactC
 
 		// Execute.
 		cnt := compactionTransactCounter(0)
-		err := exec(&cnt)
+		err := t.run(&cnt)
+		if err != nil {
+			db.logf("%s error I·%d %q", name, cnt, err)
+		}
 
 		// Set compaction error status.
 		select {
 		case db.compErrSetC <- err:
+		case perr := <-db.compPerErrC:
+			if err != nil {
+				db.logf("%s exiting (persistent error %q)", name, perr)
+				db.compactionExitTransact()
+			}
 		case _, _ = <-db.closeC:
 			db.logf("%s exiting", name)
 			db.compactionExitTransact()
@@ -182,31 +232,56 @@ func (db *DB) compactionTransact(name string, exec func(cnt *compactionTransactC
 		if err == nil {
 			return
 		}
-		db.logf("%s error I·%d %q", name, cnt, err)
-
-		// Reset backoff duration if counter is advancing.
-		if cnt > lastCnt {
-			backoff = backoffMin
-			lastCnt = cnt
-		}
-
-		// Backoff.
-		backoffT.Reset(backoff)
-		if backoff < backoffMax {
-			backoff *= backoffMul
-			if backoff > backoffMax {
-				backoff = backoffMax
-			}
-		}
-		select {
-		case <-backoffT.C:
-		case _, _ = <-db.closeC:
-			db.logf("%s exiting", name)
+		if errors.IsCorrupted(err) {
+			db.logf("%s exiting (corruption detected)", name)
 			db.compactionExitTransact()
 		}
+
+		if !disableBackoff {
+			// Reset backoff duration if counter is advancing.
+			if cnt > lastCnt {
+				backoff = backoffMin
+				lastCnt = cnt
+			}
+
+			// Backoff.
+			backoffT.Reset(backoff)
+			if backoff < backoffMax {
+				backoff *= backoffMul
+				if backoff > backoffMax {
+					backoff = backoffMax
+				}
+			}
+			select {
+			case <-backoffT.C:
+			case _, _ = <-db.closeC:
+				db.logf("%s exiting", name)
+				db.compactionExitTransact()
+			}
+		}
 	}
 }
 
+type compactionTransactFunc struct {
+	runFunc    func(cnt *compactionTransactCounter) error
+	revertFunc func() error
+}
+
+func (t *compactionTransactFunc) run(cnt *compactionTransactCounter) error {
+	return t.runFunc(cnt)
+}
+
+func (t *compactionTransactFunc) revert() error {
+	if t.revertFunc != nil {
+		return t.revertFunc()
+	}
+	return nil
+}
+
+func (db *DB) compactionTransactFunc(name string, run func(cnt *compactionTransactCounter) error, revert func() error) {
+	db.compactionTransact(name, &compactionTransactFunc{run, revert})
+}
+
 func (db *DB) compactionExitTransact() {
 	panic(errCompactionTransactExiting)
 }
@@ -216,14 +291,15 @@ func (db *DB) memCompaction() {
 	if mem == nil {
 		return
 	}
+	defer mem.decref()
 
 	c := newCMem(db.s)
 	stats := new(cStatsStaging)
 
-	db.logf("mem@flush N·%d S·%s", mem.Len(), shortenb(mem.Size()))
+	db.logf("mem@flush N·%d S·%s", mem.mdb.Len(), shortenb(mem.mdb.Size()))
 
 	// Don't compact empty memdb.
-	if mem.Len() == 0 {
+	if mem.mdb.Len() == 0 {
 		db.logf("mem@flush skipping")
 		// drop frozen mem
 		db.dropFrozenMem()
@@ -231,20 +307,23 @@ func (db *DB) memCompaction() {
 	}
 
 	// Pause table compaction.
-	ch := make(chan struct{})
+	resumeC := make(chan struct{})
 	select {
-	case db.tcompPauseC <- (chan<- struct{})(ch):
+	case db.tcompPauseC <- (chan<- struct{})(resumeC):
+	case <-db.compPerErrC:
+		close(resumeC)
+		resumeC = nil
 	case _, _ = <-db.closeC:
 		return
 	}
 
-	db.compactionTransact("mem@flush", func(cnt *compactionTransactCounter) (err error) {
+	db.compactionTransactFunc("mem@flush", func(cnt *compactionTransactCounter) (err error) {
 		stats.startTimer()
 		defer stats.stopTimer()
-		return c.flush(mem, -1)
+		return c.flush(mem.mdb, -1)
 	}, func() error {
 		for _, r := range c.rec.addedTables {
-			db.logf("mem@flush rollback @%d", r.num)
+			db.logf("mem@flush revert @%d", r.num)
 			f := db.s.getTableFile(r.num)
 			if err := f.Remove(); err != nil {
 				return err
@@ -253,13 +332,13 @@ func (db *DB) memCompaction() {
 		return nil
 	})
 
-	db.compactionTransact("mem@commit", func(cnt *compactionTransactCounter) (err error) {
+	db.compactionTransactFunc("mem@commit", func(cnt *compactionTransactCounter) (err error) {
 		stats.startTimer()
 		defer stats.stopTimer()
 		return c.commit(db.journalFile.Num(), db.frozenSeq)
 	}, nil)
 
-	db.logf("mem@flush commited F·%d T·%v", len(c.rec.addedTables), stats.duration)
+	db.logf("mem@flush committed F·%d T·%v", len(c.rec.addedTables), stats.duration)
 
 	for _, r := range c.rec.addedTables {
 		stats.write += r.size
@@ -270,26 +349,223 @@ func (db *DB) memCompaction() {
 	db.dropFrozenMem()
 
 	// Resume table compaction.
-	select {
-	case <-ch:
-	case _, _ = <-db.closeC:
-		return
+	if resumeC != nil {
+		select {
+		case <-resumeC:
+			close(resumeC)
+		case _, _ = <-db.closeC:
+			return
+		}
 	}
 
 	// Trigger table compaction.
-	db.compTrigger(db.mcompTriggerC)
+	db.compSendTrigger(db.tcompCmdC)
+}
+
+type tableCompactionBuilder struct {
+	db           *DB
+	s            *session
+	c            *compaction
+	rec          *sessionRecord
+	stat0, stat1 *cStatsStaging
+
+	snapHasLastUkey bool
+	snapLastUkey    []byte
+	snapLastSeq     uint64
+	snapIter        int
+	snapKerrCnt     int
+	snapDropCnt     int
+
+	kerrCnt int
+	dropCnt int
+
+	minSeq    uint64
+	strict    bool
+	tableSize int
+
+	tw *tWriter
+}
+
+func (b *tableCompactionBuilder) appendKV(key, value []byte) error {
+	// Create new table if not already.
+	if b.tw == nil {
+		// Check for pause event.
+		if b.db != nil {
+			select {
+			case ch := <-b.db.tcompPauseC:
+				b.db.pauseCompaction(ch)
+			case _, _ = <-b.db.closeC:
+				b.db.compactionExitTransact()
+			default:
+			}
+		}
+
+		// Create new table.
+		var err error
+		b.tw, err = b.s.tops.create()
+		if err != nil {
+			return err
+		}
+	}
+
+	// Write key/value into table.
+	return b.tw.append(key, value)
+}
+
+func (b *tableCompactionBuilder) needFlush() bool {
+	return b.tw.tw.BytesLen() >= b.tableSize
+}
+
+func (b *tableCompactionBuilder) flush() error {
+	t, err := b.tw.finish()
+	if err != nil {
+		return err
+	}
+	b.rec.addTableFile(b.c.level+1, t)
+	b.stat1.write += t.size
+	b.s.logf("table@build created L%d@%d N·%d S·%s %q:%q", b.c.level+1, t.file.Num(), b.tw.tw.EntriesLen(), shortenb(int(t.size)), t.imin, t.imax)
+	b.tw = nil
+	return nil
+}
+
+func (b *tableCompactionBuilder) cleanup() {
+	if b.tw != nil {
+		b.tw.drop()
+		b.tw = nil
+	}
+}
+
+func (b *tableCompactionBuilder) run(cnt *compactionTransactCounter) error {
+	snapResumed := b.snapIter > 0
+	hasLastUkey := b.snapHasLastUkey // The key might has zero length, so this is necessary.
+	lastUkey := append([]byte{}, b.snapLastUkey...)
+	lastSeq := b.snapLastSeq
+	b.kerrCnt = b.snapKerrCnt
+	b.dropCnt = b.snapDropCnt
+	// Restore compaction state.
+	b.c.restore()
+
+	defer b.cleanup()
+
+	b.stat1.startTimer()
+	defer b.stat1.stopTimer()
+
+	iter := b.c.newIterator()
+	defer iter.Release()
+	for i := 0; iter.Next(); i++ {
+		// Incr transact counter.
+		cnt.incr()
+
+		// Skip until last state.
+		if i < b.snapIter {
+			continue
+		}
+
+		resumed := false
+		if snapResumed {
+			resumed = true
+			snapResumed = false
+		}
+
+		ikey := iter.Key()
+		ukey, seq, kt, kerr := parseIkey(ikey)
+
+		if kerr == nil {
+			shouldStop := !resumed && b.c.shouldStopBefore(ikey)
+
+			if !hasLastUkey || b.s.icmp.uCompare(lastUkey, ukey) != 0 {
+				// First occurrence of this user key.
+
+				// Only rotate tables if ukey doesn't hop across.
+				if b.tw != nil && (shouldStop || b.needFlush()) {
+					if err := b.flush(); err != nil {
+						return err
+					}
+
+					// Creates snapshot of the state.
+					b.c.save()
+					b.snapHasLastUkey = hasLastUkey
+					b.snapLastUkey = append(b.snapLastUkey[:0], lastUkey...)
+					b.snapLastSeq = lastSeq
+					b.snapIter = i
+					b.snapKerrCnt = b.kerrCnt
+					b.snapDropCnt = b.dropCnt
+				}
+
+				hasLastUkey = true
+				lastUkey = append(lastUkey[:0], ukey...)
+				lastSeq = kMaxSeq
+			}
+
+			switch {
+			case lastSeq <= b.minSeq:
+				// Dropped because newer entry for same user key exist
+				fallthrough // (A)
+			case kt == ktDel && seq <= b.minSeq && b.c.baseLevelForKey(lastUkey):
+				// For this user key:
+				// (1) there is no data in higher levels
+				// (2) data in lower levels will have larger seq numbers
+				// (3) data in layers that are being compacted here and have
+				//     smaller seq numbers will be dropped in the next
+				//     few iterations of this loop (by rule (A) above).
+				// Therefore this deletion marker is obsolete and can be dropped.
+				lastSeq = seq
+				b.dropCnt++
+				continue
+			default:
+				lastSeq = seq
+			}
+		} else {
+			if b.strict {
+				return kerr
+			}
+
+			// Don't drop corrupted keys.
+			hasLastUkey = false
+			lastUkey = lastUkey[:0]
+			lastSeq = kMaxSeq
+			b.kerrCnt++
+		}
+
+		if err := b.appendKV(ikey, iter.Value()); err != nil {
+			return err
+		}
+	}
+
+	if err := iter.Error(); err != nil {
+		return err
+	}
+
+	// Finish last table.
+	if b.tw != nil && !b.tw.empty() {
+		return b.flush()
+	}
+	return nil
+}
+
+func (b *tableCompactionBuilder) revert() error {
+	for _, at := range b.rec.addedTables {
+		b.s.logf("table@build revert @%d", at.num)
+		f := b.s.getTableFile(at.num)
+		if err := f.Remove(); err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 func (db *DB) tableCompaction(c *compaction, noTrivial bool) {
-	rec := new(sessionRecord)
-	rec.addCompactionPointer(c.level, c.imax)
+	defer c.release()
+
+	rec := &sessionRecord{numLevel: db.s.o.GetNumLevel()}
+	rec.addCompPtr(c.level, c.imax)
 
 	if !noTrivial && c.trivial() {
 		t := c.tables[0][0]
 		db.logf("table@move L%d@%d -> L%d", c.level, t.file.Num(), c.level+1)
-		rec.deleteTable(c.level, t.file.Num())
+		rec.delTable(c.level, t.file.Num())
 		rec.addTableFile(c.level+1, t)
-		db.compactionTransact("table@move", func(cnt *compactionTransactCounter) (err error) {
+		db.compactionTransactFunc("table@move", func(cnt *compactionTransactCounter) (err error) {
 			return db.s.commit(rec)
 		}, nil)
 		return
@@ -300,184 +576,34 @@ func (db *DB) tableCompaction(c *compaction, noTrivial bool) {
 		for _, t := range tables {
 			stats[i].read += t.size
 			// Insert deleted tables into record
-			rec.deleteTable(c.level+i, t.file.Num())
+			rec.delTable(c.level+i, t.file.Num())
 		}
 	}
 	sourceSize := int(stats[0].read + stats[1].read)
 	minSeq := db.minSeq()
 	db.logf("table@compaction L%d·%d -> L%d·%d S·%s Q·%d", c.level, len(c.tables[0]), c.level+1, len(c.tables[1]), shortenb(sourceSize), minSeq)
 
-	var snapUkey []byte
-	var snapHasUkey bool
-	var snapSeq uint64
-	var snapIter int
-	var snapDropCnt int
-	var dropCnt int
-	db.compactionTransact("table@build", func(cnt *compactionTransactCounter) (err error) {
-		ukey := append([]byte{}, snapUkey...)
-		hasUkey := snapHasUkey
-		lseq := snapSeq
-		dropCnt = snapDropCnt
-		snapSched := snapIter == 0
-
-		var tw *tWriter
-		finish := func() error {
-			t, err := tw.finish()
-			if err != nil {
-				return err
-			}
-			rec.addTableFile(c.level+1, t)
-			stats[1].write += t.size
-			db.logf("table@build created L%d@%d N·%d S·%s %q:%q", c.level+1, t.file.Num(), tw.tw.EntriesLen(), shortenb(int(t.size)), t.imin, t.imax)
-			return nil
-		}
-
-		defer func() {
-			stats[1].stopTimer()
-			if tw != nil {
-				tw.drop()
-				tw = nil
-			}
-		}()
-
-		stats[1].startTimer()
-		iter := c.newIterator()
-		defer iter.Release()
-		for i := 0; iter.Next(); i++ {
-			// Incr transact counter.
-			cnt.incr()
-
-			// Skip until last state.
-			if i < snapIter {
-				continue
-			}
-
-			ikey := iKey(iter.Key())
-
-			if c.shouldStopBefore(ikey) && tw != nil {
-				err = finish()
-				if err != nil {
-					return
-				}
-				snapSched = true
-				tw = nil
-			}
-
-			// Scheduled for snapshot, snapshot will used to retry compaction
-			// if error occured.
-			if snapSched {
-				snapUkey = append(snapUkey[:0], ukey...)
-				snapHasUkey = hasUkey
-				snapSeq = lseq
-				snapIter = i
-				snapDropCnt = dropCnt
-				snapSched = false
-			}
-
-			if seq, vt, ok := ikey.parseNum(); !ok {
-				// Don't drop error keys
-				ukey = ukey[:0]
-				hasUkey = false
-				lseq = kMaxSeq
-			} else {
-				if !hasUkey || db.s.icmp.uCompare(ikey.ukey(), ukey) != 0 {
-					// First occurrence of this user key
-					ukey = append(ukey[:0], ikey.ukey()...)
-					hasUkey = true
-					lseq = kMaxSeq
-				}
-
-				drop := false
-				if lseq <= minSeq {
-					// Dropped because newer entry for same user key exist
-					drop = true // (A)
-				} else if vt == tDel && seq <= minSeq && c.baseLevelForKey(ukey) {
-					// For this user key:
-					// (1) there is no data in higher levels
-					// (2) data in lower levels will have larger seq numbers
-					// (3) data in layers that are being compacted here and have
-					//     smaller seq numbers will be dropped in the next
-					//     few iterations of this loop (by rule (A) above).
-					// Therefore this deletion marker is obsolete and can be dropped.
-					drop = true
-				}
-
-				lseq = seq
-				if drop {
-					dropCnt++
-					continue
-				}
-			}
-
-			// Create new table if not already
-			if tw == nil {
-				// Check for pause event.
-				select {
-				case ch := <-db.tcompPauseC:
-					db.pauseCompaction(ch)
-				case _, _ = <-db.closeC:
-					db.compactionExitTransact()
-				default:
-				}
-
-				// Create new table.
-				tw, err = db.s.tops.create()
-				if err != nil {
-					return
-				}
-			}
-
-			// Write key/value into table
-			err = tw.append(ikey, iter.Value())
-			if err != nil {
-				return
-			}
-
-			// Finish table if it is big enough
-			if tw.tw.BytesLen() >= kMaxTableSize {
-				err = finish()
-				if err != nil {
-					return
-				}
-				snapSched = true
-				tw = nil
-			}
-		}
-
-		err = iter.Error()
-		if err != nil {
-			return
-		}
-
-		// Finish last table
-		if tw != nil && !tw.empty() {
-			err = finish()
-			if err != nil {
-				return
-			}
-			tw = nil
-		}
-		return
-	}, func() error {
-		for _, r := range rec.addedTables {
-			db.logf("table@build rollback @%d", r.num)
-			f := db.s.getTableFile(r.num)
-			if err := f.Remove(); err != nil {
-				return err
-			}
-		}
-		return nil
-	})
+	b := &tableCompactionBuilder{
+		db:        db,
+		s:         db.s,
+		c:         c,
+		rec:       rec,
+		stat1:     &stats[1],
+		minSeq:    minSeq,
+		strict:    db.s.o.GetStrict(opt.StrictCompaction),
+		tableSize: db.s.o.GetCompactionTableSize(c.level + 1),
+	}
+	db.compactionTransact("table@build", b)
 
 	// Commit changes
-	db.compactionTransact("table@commit", func(cnt *compactionTransactCounter) (err error) {
+	db.compactionTransactFunc("table@commit", func(cnt *compactionTransactCounter) (err error) {
 		stats[1].startTimer()
 		defer stats[1].stopTimer()
 		return db.s.commit(rec)
 	}, nil)
 
 	resultSize := int(stats[1].write)
-	db.logf("table@compaction commited F%s S%s D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), dropCnt, stats[1].duration)
+	db.logf("table@compaction committed F%s S%s Ke·%d D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), b.kerrCnt, b.dropCnt, stats[1].duration)
 
 	// Save compaction stats
 	for i := range stats {
@@ -493,14 +619,14 @@ func (db *DB) tableRangeCompaction(level int, umin, umax []byte) {
 			db.tableCompaction(c, true)
 		}
 	} else {
-		v := db.s.version_NB()
-
+		v := db.s.version()
 		m := 1
 		for i, t := range v.tables[1:] {
 			if t.overlaps(db.s.icmp, umin, umax, false) {
 				m = i + 1
 			}
 		}
+		v.release()
 
 		for level := 0; level < m; level++ {
 			if c := db.s.getCompactionRange(level, umin, umax); c != nil {
@@ -517,7 +643,9 @@ func (db *DB) tableAutoCompaction() {
 }
 
 func (db *DB) tableNeedCompaction() bool {
-	return db.s.version_NB().needCompaction()
+	v := db.s.version()
+	defer v.release()
+	return v.needCompaction()
 }
 
 func (db *DB) pauseCompaction(ch chan<- struct{}) {
@@ -537,7 +665,12 @@ type cIdle struct {
 }
 
 func (r cIdle) ack(err error) {
-	r.ackC <- err
+	if r.ackC != nil {
+		defer func() {
+			recover()
+		}()
+		r.ackC <- err
+	}
 }
 
 type cRange struct {
@@ -547,29 +680,45 @@ type cRange struct {
 }
 
 func (r cRange) ack(err error) {
-	defer func() {
-		recover()
-	}()
 	if r.ackC != nil {
+		defer func() {
+			recover()
+		}()
 		r.ackC <- err
 	}
 }
 
-func (db *DB) compSendIdle(compC chan<- cCmd) error {
+// This will trigger auto compation and/or wait for all compaction to be done.
+func (db *DB) compSendIdle(compC chan<- cCmd) (err error) {
 	ch := make(chan error)
 	defer close(ch)
 	// Send cmd.
 	select {
 	case compC <- cIdle{ch}:
-	case err := <-db.compErrC:
-		return err
+	case err = <-db.compErrC:
+		return
 	case _, _ = <-db.closeC:
 		return ErrClosed
 	}
 	// Wait cmd.
-	return <-ch
+	select {
+	case err = <-ch:
+	case err = <-db.compErrC:
+	case _, _ = <-db.closeC:
+		return ErrClosed
+	}
+	return err
 }
 
+// This will trigger auto compaction but will not wait for it.
+func (db *DB) compSendTrigger(compC chan<- cCmd) {
+	select {
+	case compC <- cIdle{}:
+	default:
+	}
+}
+
+// Send range compaction request.
 func (db *DB) compSendRange(compC chan<- cCmd, level int, min, max []byte) (err error) {
 	ch := make(chan error)
 	defer close(ch)
@@ -583,19 +732,14 @@ func (db *DB) compSendRange(compC chan<- cCmd, level int, min, max []byte) (err
 	}
 	// Wait cmd.
 	select {
-	case err = <-db.compErrC:
 	case err = <-ch:
+	case err = <-db.compErrC:
+	case _, _ = <-db.closeC:
+		return ErrClosed
 	}
 	return err
 }
 
-func (db *DB) compTrigger(compTriggerC chan struct{}) {
-	select {
-	case compTriggerC <- struct{}{}:
-	default:
-	}
-}
-
 func (db *DB) mCompaction() {
 	var x cCmd
 
@@ -614,11 +758,14 @@ func (db *DB) mCompaction() {
 	for {
 		select {
 		case x = <-db.mcompCmdC:
-			db.memCompaction()
-			x.ack(nil)
-			x = nil
-		case <-db.mcompTriggerC:
-			db.memCompaction()
+			switch x.(type) {
+			case cIdle:
+				db.memCompaction()
+				x.ack(nil)
+				x = nil
+			default:
+				panic("leveldb: unknown command")
+			}
 		case _, _ = <-db.closeC:
 			return
 		}
@@ -649,7 +796,6 @@ func (db *DB) tCompaction() {
 		if db.tableNeedCompaction() {
 			select {
 			case x = <-db.tcompCmdC:
-			case <-db.tcompTriggerC:
 			case ch := <-db.tcompPauseC:
 				db.pauseCompaction(ch)
 				continue
@@ -665,7 +811,6 @@ func (db *DB) tCompaction() {
 			ackQ = ackQ[:0]
 			select {
 			case x = <-db.tcompCmdC:
-			case <-db.tcompTriggerC:
 			case ch := <-db.tcompPauseC:
 				db.pauseCompaction(ch)
 				continue
@@ -680,6 +825,8 @@ func (db *DB) tCompaction() {
 			case cRange:
 				db.tableRangeCompaction(cmd.level, cmd.min, cmd.max)
 				x.ack(nil)
+			default:
+				panic("leveldb: unknown command")
 			}
 			x = nil
 		}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_iter.go

@@ -9,6 +9,8 @@ package leveldb
 import (
 	"errors"
 	"runtime"
+	"sync"
+	"sync/atomic"
 
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -19,6 +21,17 @@ var (
 	errInvalidIkey = errors.New("leveldb: Iterator: invalid internal key")
 )
 
+type memdbReleaser struct {
+	once sync.Once
+	m    *memDB
+}
+
+func (mr *memdbReleaser) Release() {
+	mr.once.Do(func() {
+		mr.m.decref()
+	})
+}
+
 func (db *DB) newRawIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
 	em, fm := db.getMems()
 	v := db.s.version()
@@ -26,12 +39,16 @@ func (db *DB) newRawIterator(slice *util.Range, ro *opt.ReadOptions) iterator.It
 	ti := v.getIterators(slice, ro)
 	n := len(ti) + 2
 	i := make([]iterator.Iterator, 0, n)
-	i = append(i, em.NewIterator(slice))
+	emi := em.mdb.NewIterator(slice)
+	emi.SetReleaser(&memdbReleaser{m: em})
+	i = append(i, emi)
 	if fm != nil {
-		i = append(i, fm.NewIterator(slice))
+		fmi := fm.mdb.NewIterator(slice)
+		fmi.SetReleaser(&memdbReleaser{m: fm})
+		i = append(i, fmi)
 	}
 	i = append(i, ti...)
-	strict := db.s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator)
+	strict := opt.GetStrict(db.s.o.Options, ro, opt.StrictReader)
 	mi := iterator.NewMergedIterator(i, db.s.icmp, strict)
 	mi.SetReleaser(&versionReleaser{v: v})
 	return mi
@@ -42,21 +59,23 @@ func (db *DB) newIterator(seq uint64, slice *util.Range, ro *opt.ReadOptions) *d
 	if slice != nil {
 		islice = &util.Range{}
 		if slice.Start != nil {
-			islice.Start = newIKey(slice.Start, kMaxSeq, tSeek)
+			islice.Start = newIkey(slice.Start, kMaxSeq, ktSeek)
 		}
 		if slice.Limit != nil {
-			islice.Limit = newIKey(slice.Limit, kMaxSeq, tSeek)
+			islice.Limit = newIkey(slice.Limit, kMaxSeq, ktSeek)
 		}
 	}
 	rawIter := db.newRawIterator(islice, ro)
 	iter := &dbIter{
+		db:     db,
 		icmp:   db.s.icmp,
 		iter:   rawIter,
 		seq:    seq,
-		strict: db.s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator),
+		strict: opt.GetStrict(db.s.o.Options, ro, opt.StrictReader),
 		key:    make([]byte, 0),
 		value:  make([]byte, 0),
 	}
+	atomic.AddInt32(&db.aliveIters, 1)
 	runtime.SetFinalizer(iter, (*dbIter).Release)
 	return iter
 }
@@ -73,6 +92,7 @@ const (
 
 // dbIter represent an interator states over a database session.
 type dbIter struct {
+	db     *DB
 	icmp   *iComparer
 	iter   iterator.Iterator
 	seq    uint64
@@ -142,7 +162,7 @@ func (i *dbIter) Seek(key []byte) bool {
 		return false
 	}
 
-	ikey := newIKey(key, i.seq, tSeek)
+	ikey := newIkey(key, i.seq, ktSeek)
 	if i.iter.Seek(ikey) {
 		i.dir = dirSOI
 		return i.next()
@@ -154,15 +174,14 @@ func (i *dbIter) Seek(key []byte) bool {
 
 func (i *dbIter) next() bool {
 	for {
-		ukey, seq, t, ok := parseIkey(i.iter.Key())
-		if ok {
+		if ukey, seq, kt, kerr := parseIkey(i.iter.Key()); kerr == nil {
 			if seq <= i.seq {
-				switch t {
-				case tDel:
+				switch kt {
+				case ktDel:
 					// Skip deleted key.
 					i.key = append(i.key[:0], ukey...)
 					i.dir = dirForward
-				case tVal:
+				case ktVal:
 					if i.dir == dirSOI || i.icmp.uCompare(ukey, i.key) > 0 {
 						i.key = append(i.key[:0], ukey...)
 						i.value = append(i.value[:0], i.iter.Value()...)
@@ -172,7 +191,7 @@ func (i *dbIter) next() bool {
 				}
 			}
 		} else if i.strict {
-			i.setErr(errInvalidIkey)
+			i.setErr(kerr)
 			break
 		}
 		if !i.iter.Next() {
@@ -205,20 +224,19 @@ func (i *dbIter) prev() bool {
 	del := true
 	if i.iter.Valid() {
 		for {
-			ukey, seq, t, ok := parseIkey(i.iter.Key())
-			if ok {
+			if ukey, seq, kt, kerr := parseIkey(i.iter.Key()); kerr == nil {
 				if seq <= i.seq {
 					if !del && i.icmp.uCompare(ukey, i.key) < 0 {
 						return true
 					}
-					del = (t == tDel)
+					del = (kt == ktDel)
 					if !del {
 						i.key = append(i.key[:0], ukey...)
 						i.value = append(i.value[:0], i.iter.Value()...)
 					}
 				}
 			} else if i.strict {
-				i.setErr(errInvalidIkey)
+				i.setErr(kerr)
 				return false
 			}
 			if !i.iter.Prev() {
@@ -247,13 +265,12 @@ func (i *dbIter) Prev() bool {
 		return i.Last()
 	case dirForward:
 		for i.iter.Prev() {
-			ukey, _, _, ok := parseIkey(i.iter.Key())
-			if ok {
+			if ukey, _, _, kerr := parseIkey(i.iter.Key()); kerr == nil {
 				if i.icmp.uCompare(ukey, i.key) < 0 {
 					goto cont
 				}
 			} else if i.strict {
-				i.setErr(errInvalidIkey)
+				i.setErr(kerr)
 				return false
 			}
 		}
@@ -287,6 +304,7 @@ func (i *dbIter) Release() {
 
 		if i.releaser != nil {
 			i.releaser.Release()
+			i.releaser = nil
 		}
 
 		i.dir = dirReleased
@@ -294,13 +312,19 @@ func (i *dbIter) Release() {
 		i.value = nil
 		i.iter.Release()
 		i.iter = nil
+		atomic.AddInt32(&i.db.aliveIters, -1)
+		i.db = nil
 	}
 }
 
 func (i *dbIter) SetReleaser(releaser util.Releaser) {
-	if i.dir != dirReleased {
-		i.releaser = releaser
+	if i.dir == dirReleased {
+		panic(util.ErrReleased)
 	}
+	if i.releaser != nil && releaser != nil {
+		panic(util.ErrHasReleaser)
+	}
+	i.releaser = releaser
 }
 
 func (i *dbIter) Error() error {

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_snapshot.go

@@ -7,8 +7,11 @@
 package leveldb
 
 import (
+	"container/list"
+	"fmt"
 	"runtime"
 	"sync"
+	"sync/atomic"
 
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -18,51 +21,41 @@ import (
 type snapshotElement struct {
 	seq uint64
 	ref int
-	// Next and previous pointers in the doubly-linked list of elements.
-	next, prev *snapshotElement
-}
-
-// Initialize the snapshot.
-func (db *DB) initSnapshot() {
-	db.snapsRoot.next = &db.snapsRoot
-	db.snapsRoot.prev = &db.snapsRoot
+	e   *list.Element
 }
 
 // Acquires a snapshot, based on latest sequence.
 func (db *DB) acquireSnapshot() *snapshotElement {
 	db.snapsMu.Lock()
+	defer db.snapsMu.Unlock()
+
 	seq := db.getSeq()
-	elem := db.snapsRoot.prev
-	if elem == &db.snapsRoot || elem.seq != seq {
-		at := db.snapsRoot.prev
-		next := at.next
-		elem = &snapshotElement{
-			seq:  seq,
-			prev: at,
-			next: next,
+
+	if e := db.snapsList.Back(); e != nil {
+		se := e.Value.(*snapshotElement)
+		if se.seq == seq {
+			se.ref++
+			return se
+		} else if seq < se.seq {
+			panic("leveldb: sequence number is not increasing")
 		}
-		at.next = elem
-		next.prev = elem
 	}
-	elem.ref++
-	db.snapsMu.Unlock()
-	return elem
+	se := &snapshotElement{seq: seq, ref: 1}
+	se.e = db.snapsList.PushBack(se)
+	return se
 }
 
 // Releases given snapshot element.
-func (db *DB) releaseSnapshot(elem *snapshotElement) {
-	if !db.isClosed() {
-		db.snapsMu.Lock()
-		elem.ref--
-		if elem.ref == 0 {
-			elem.prev.next = elem.next
-			elem.next.prev = elem.prev
-			elem.next = nil
-			elem.prev = nil
-		} else if elem.ref < 0 {
-			panic("leveldb: Snapshot: negative element reference")
-		}
-		db.snapsMu.Unlock()
+func (db *DB) releaseSnapshot(se *snapshotElement) {
+	db.snapsMu.Lock()
+	defer db.snapsMu.Unlock()
+
+	se.ref--
+	if se.ref == 0 {
+		db.snapsList.Remove(se.e)
+		se.e = nil
+	} else if se.ref < 0 {
+		panic("leveldb: Snapshot: negative element reference")
 	}
 }
 
@@ -70,10 +63,11 @@ func (db *DB) releaseSnapshot(elem *snapshotElement) {
 func (db *DB) minSeq() uint64 {
 	db.snapsMu.Lock()
 	defer db.snapsMu.Unlock()
-	elem := db.snapsRoot.prev
-	if elem != &db.snapsRoot {
-		return elem.seq
+
+	if e := db.snapsList.Front(); e != nil {
+		return e.Value.(*snapshotElement).seq
 	}
+
 	return db.getSeq()
 }
 
@@ -81,7 +75,7 @@ func (db *DB) minSeq() uint64 {
 type Snapshot struct {
 	db       *DB
 	elem     *snapshotElement
-	mu       sync.Mutex
+	mu       sync.RWMutex
 	released bool
 }
 
@@ -91,12 +85,17 @@ func (db *DB) newSnapshot() *Snapshot {
 		db:   db,
 		elem: db.acquireSnapshot(),
 	}
+	atomic.AddInt32(&db.aliveSnaps, 1)
 	runtime.SetFinalizer(snap, (*Snapshot).Release)
 	return snap
 }
 
+func (snap *Snapshot) String() string {
+	return fmt.Sprintf("leveldb.Snapshot{%d}", snap.elem.seq)
+}
+
 // Get gets the value for the given key. It returns ErrNotFound if
-// the DB does not contain the key.
+// the DB does not contains the key.
 //
 // The caller should not modify the contents of the returned slice, but
 // it is safe to modify the contents of the argument after Get returns.
@@ -105,8 +104,8 @@ func (snap *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err er
 	if err != nil {
 		return
 	}
-	snap.mu.Lock()
-	defer snap.mu.Unlock()
+	snap.mu.RLock()
+	defer snap.mu.RUnlock()
 	if snap.released {
 		err = ErrSnapshotReleased
 		return
@@ -114,6 +113,23 @@ func (snap *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err er
 	return snap.db.get(key, snap.elem.seq, ro)
 }
 
+// Has returns true if the DB does contains the given key.
+//
+// It is safe to modify the contents of the argument after Get returns.
+func (snap *Snapshot) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) {
+	err = snap.db.ok()
+	if err != nil {
+		return
+	}
+	snap.mu.RLock()
+	defer snap.mu.RUnlock()
+	if snap.released {
+		err = ErrSnapshotReleased
+		return
+	}
+	return snap.db.has(key, snap.elem.seq, ro)
+}
+
 // NewIterator returns an iterator for the snapshot of the uderlying DB.
 // The returned iterator is not goroutine-safe, but it is safe to use
 // multiple iterators concurrently, with each in a dedicated goroutine.
@@ -160,6 +176,7 @@ func (snap *Snapshot) Release() {
 
 		snap.released = true
 		snap.db.releaseSnapshot(snap.elem)
+		atomic.AddInt32(&snap.db.aliveSnaps, -1)
 		snap.db = nil
 		snap.elem = nil
 	}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_state.go

@@ -8,11 +8,36 @@ package leveldb
 
 import (
 	"sync/atomic"
+	"time"
 
 	"github.com/syndtr/goleveldb/leveldb/journal"
 	"github.com/syndtr/goleveldb/leveldb/memdb"
 )
 
+type memDB struct {
+	db  *DB
+	mdb *memdb.DB
+	ref int32
+}
+
+func (m *memDB) incref() {
+	atomic.AddInt32(&m.ref, 1)
+}
+
+func (m *memDB) decref() {
+	if ref := atomic.AddInt32(&m.ref, -1); ref == 0 {
+		// Only put back memdb with std capacity.
+		if m.mdb.Capacity() == m.db.s.o.GetWriteBuffer() {
+			m.mdb.Reset()
+			m.db.mpoolPut(m.mdb)
+		}
+		m.db = nil
+		m.mdb = nil
+	} else if ref < 0 {
+		panic("negative memdb ref")
+	}
+}
+
 // Get latest sequence number.
 func (db *DB) getSeq() uint64 {
 	return atomic.LoadUint64(&db.seq)
@@ -23,9 +48,44 @@ func (db *DB) addSeq(delta uint64) {
 	atomic.AddUint64(&db.seq, delta)
 }
 
+func (db *DB) mpoolPut(mem *memdb.DB) {
+	defer func() {
+		recover()
+	}()
+	select {
+	case db.memPool <- mem:
+	default:
+	}
+}
+
+func (db *DB) mpoolGet() *memdb.DB {
+	select {
+	case mem := <-db.memPool:
+		return mem
+	default:
+		return nil
+	}
+}
+
+func (db *DB) mpoolDrain() {
+	ticker := time.NewTicker(30 * time.Second)
+	for {
+		select {
+		case <-ticker.C:
+			select {
+			case <-db.memPool:
+			default:
+			}
+		case _, _ = <-db.closeC:
+			close(db.memPool)
+			return
+		}
+	}
+}
+
 // Create new memdb and froze the old one; need external synchronization.
 // newMem only called synchronously by the writer.
-func (db *DB) newMem(n int) (mem *memdb.DB, err error) {
+func (db *DB) newMem(n int) (mem *memDB, err error) {
 	num := db.s.allocFileNum()
 	file := db.s.getJournalFile(num)
 	w, err := file.Create()
@@ -37,6 +97,10 @@ func (db *DB) newMem(n int) (mem *memdb.DB, err error) {
 	db.memMu.Lock()
 	defer db.memMu.Unlock()
 
+	if db.frozenMem != nil {
+		panic("still has frozen mem")
+	}
+
 	if db.journal == nil {
 		db.journal = journal.NewWriter(w)
 	} else {
@@ -47,8 +111,16 @@ func (db *DB) newMem(n int) (mem *memdb.DB, err error) {
 	db.journalWriter = w
 	db.journalFile = file
 	db.frozenMem = db.mem
-	db.mem = memdb.New(db.s.icmp, maxInt(db.s.o.GetWriteBuffer(), n))
-	mem = db.mem
+	mdb := db.mpoolGet()
+	if mdb == nil || mdb.Capacity() < n {
+		mdb = memdb.New(db.s.icmp, maxInt(db.s.o.GetWriteBuffer(), n))
+	}
+	mem = &memDB{
+		db:  db,
+		mdb: mdb,
+		ref: 2,
+	}
+	db.mem = mem
 	// The seq only incremented by the writer. And whoever called newMem
 	// should hold write lock, so no need additional synchronization here.
 	db.frozenSeq = db.seq
@@ -56,16 +128,27 @@ func (db *DB) newMem(n int) (mem *memdb.DB, err error) {
 }
 
 // Get all memdbs.
-func (db *DB) getMems() (e *memdb.DB, f *memdb.DB) {
+func (db *DB) getMems() (e, f *memDB) {
 	db.memMu.RLock()
 	defer db.memMu.RUnlock()
+	if db.mem == nil {
+		panic("nil effective mem")
+	}
+	db.mem.incref()
+	if db.frozenMem != nil {
+		db.frozenMem.incref()
+	}
 	return db.mem, db.frozenMem
 }
 
 // Get frozen memdb.
-func (db *DB) getEffectiveMem() *memdb.DB {
+func (db *DB) getEffectiveMem() *memDB {
 	db.memMu.RLock()
 	defer db.memMu.RUnlock()
+	if db.mem == nil {
+		panic("nil effective mem")
+	}
+	db.mem.incref()
 	return db.mem
 }
 
@@ -77,9 +160,12 @@ func (db *DB) hasFrozenMem() bool {
 }
 
 // Get frozen memdb.
-func (db *DB) getFrozenMem() *memdb.DB {
+func (db *DB) getFrozenMem() *memDB {
 	db.memMu.RLock()
 	defer db.memMu.RUnlock()
+	if db.frozenMem != nil {
+		db.frozenMem.incref()
+	}
 	return db.frozenMem
 }
 
@@ -92,6 +178,7 @@ func (db *DB) dropFrozenMem() {
 		db.logf("journal@remove removed @%d", db.frozenJournalFile.Num())
 	}
 	db.frozenJournalFile = nil
+	db.frozenMem.decref()
 	db.frozenMem = nil
 	db.memMu.Unlock()
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_util.go

@@ -7,8 +7,7 @@
 package leveldb
 
 import (
-	"errors"
-
+	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	"github.com/syndtr/goleveldb/leveldb/storage"
@@ -38,7 +37,9 @@ func (db *DB) logf(format string, v ...interface{}) { db.s.logf(format, v...) }
 
 // Check and clean files.
 func (db *DB) checkAndCleanFiles() error {
-	v := db.s.version_NB()
+	v := db.s.version()
+	defer v.release()
+
 	tablesMap := make(map[uint64]bool)
 	for _, tables := range v.tables {
 		for _, t := range tables {
@@ -78,12 +79,14 @@ func (db *DB) checkAndCleanFiles() error {
 	}
 
 	if nTables != len(tablesMap) {
+		var missing []*storage.FileInfo
 		for num, present := range tablesMap {
 			if !present {
+				missing = append(missing, &storage.FileInfo{Type: storage.TypeTable, Num: num})
 				db.logf("db@janitor table missing @%d", num)
 			}
 		}
-		return ErrCorrupted{Type: MissingFiles, Err: errors.New("leveldb: table files missing")}
+		return errors.NewErrCorrupted(nil, &errors.ErrMissingFiles{Files: missing})
 	}
 
 	db.logf("db@janitor F·%d G·%d", len(files), len(rem))

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_write.go

@@ -45,7 +45,7 @@ func (db *DB) jWriter() {
 	}
 }
 
-func (db *DB) rotateMem(n int) (mem *memdb.DB, err error) {
+func (db *DB) rotateMem(n int) (mem *memDB, err error) {
 	// Wait for pending memdb compaction.
 	err = db.compSendIdle(db.mcompCmdC)
 	if err != nil {
@@ -59,24 +59,30 @@ func (db *DB) rotateMem(n int) (mem *memdb.DB, err error) {
 	}
 
 	// Schedule memdb compaction.
-	db.compTrigger(db.mcompTriggerC)
+	db.compSendTrigger(db.mcompCmdC)
 	return
 }
 
-func (db *DB) flush(n int) (mem *memdb.DB, nn int, err error) {
+func (db *DB) flush(n int) (mem *memDB, nn int, err error) {
 	delayed := false
-	flush := func() bool {
+	flush := func() (retry bool) {
 		v := db.s.version()
 		defer v.release()
 		mem = db.getEffectiveMem()
-		nn = mem.Free()
+		defer func() {
+			if retry {
+				mem.decref()
+				mem = nil
+			}
+		}()
+		nn = mem.mdb.Free()
 		switch {
-		case v.tLen(0) >= kL0_SlowdownWritesTrigger && !delayed:
+		case v.tLen(0) >= db.s.o.GetWriteL0SlowdownTrigger() && !delayed:
 			delayed = true
 			time.Sleep(time.Millisecond)
 		case nn >= n:
 			return false
-		case v.tLen(0) >= kL0_StopWritesTrigger:
+		case v.tLen(0) >= db.s.o.GetWriteL0PauseTrigger():
 			delayed = true
 			err = db.compSendIdle(db.tcompCmdC)
 			if err != nil {
@@ -84,12 +90,17 @@ func (db *DB) flush(n int) (mem *memdb.DB, nn int, err error) {
 			}
 		default:
 			// Allow memdb to grow if it has no entry.
-			if mem.Len() == 0 {
+			if mem.mdb.Len() == 0 {
 				nn = n
-				return false
+			} else {
+				mem.decref()
+				mem, err = db.rotateMem(n)
+				if err == nil {
+					nn = mem.mdb.Free()
+				} else {
+					nn = 0
+				}
 			}
-			mem, err = db.rotateMem(n)
-			nn = mem.Free()
 			return false
 		}
 		return true
@@ -98,7 +109,12 @@ func (db *DB) flush(n int) (mem *memdb.DB, nn int, err error) {
 	for flush() {
 	}
 	if delayed {
-		db.logf("db@write delayed T·%v", time.Since(start))
+		db.writeDelay += time.Since(start)
+		db.writeDelayN++
+	} else if db.writeDelayN > 0 {
+		db.logf("db@write was delayed N·%d T·%v", db.writeDelayN, db.writeDelay)
+		db.writeDelay = 0
+		db.writeDelayN = 0
 	}
 	return
 }
@@ -109,28 +125,33 @@ func (db *DB) flush(n int) (mem *memdb.DB, nn int, err error) {
 // It is safe to modify the contents of the arguments after Write returns.
 func (db *DB) Write(b *Batch, wo *opt.WriteOptions) (err error) {
 	err = db.ok()
-	if err != nil || b == nil || b.len() == 0 {
+	if err != nil || b == nil || b.Len() == 0 {
 		return
 	}
 
 	b.init(wo.GetSync())
 
 	// The write happen synchronously.
-retry:
 	select {
 	case db.writeC <- b:
 		if <-db.writeMergedC {
 			return <-db.writeAckC
 		}
-		goto retry
 	case db.writeLockC <- struct{}{}:
+	case err = <-db.compPerErrC:
+		return
 	case _, _ = <-db.closeC:
 		return ErrClosed
 	}
 
 	merged := 0
+	danglingMerge := false
 	defer func() {
-		<-db.writeLockC
+		if danglingMerge {
+			db.writeMergedC <- false
+		} else {
+			<-db.writeLockC
+		}
 		for i := 0; i < merged; i++ {
 			db.writeAckC <- err
 		}
@@ -140,6 +161,7 @@ retry:
 	if err != nil {
 		return
 	}
+	defer mem.decref()
 
 	// Calculate maximum size of the batch.
 	m := 1 << 20
@@ -158,7 +180,7 @@ drain:
 				db.writeMergedC <- true
 				merged++
 			} else {
-				db.writeMergedC <- false
+				danglingMerge = true
 				break drain
 			}
 		default:
@@ -173,35 +195,43 @@ drain:
 	if b.size() >= (128 << 10) {
 		// Push the write batch to the journal writer
 		select {
+		case db.journalC <- b:
+			// Write into memdb
+			if berr := b.memReplay(mem.mdb); berr != nil {
+				panic(berr)
+			}
+		case err = <-db.compPerErrC:
+			return
 		case _, _ = <-db.closeC:
 			err = ErrClosed
 			return
-		case db.journalC <- b:
-			// Write into memdb
-			b.memReplay(mem)
 		}
 		// Wait for journal writer
 		select {
-		case _, _ = <-db.closeC:
-			err = ErrClosed
-			return
 		case err = <-db.journalAckC:
 			if err != nil {
 				// Revert memdb if error detected
-				b.revertMemReplay(mem)
+				if berr := b.revertMemReplay(mem.mdb); berr != nil {
+					panic(berr)
+				}
 				return
 			}
+		case _, _ = <-db.closeC:
+			err = ErrClosed
+			return
 		}
 	} else {
 		err = db.writeJournal(b)
 		if err != nil {
 			return
 		}
-		b.memReplay(mem)
+		if berr := b.memReplay(mem.mdb); berr != nil {
+			panic(berr)
+		}
 	}
 
 	// Set last seq number.
-	db.addSeq(uint64(b.len()))
+	db.addSeq(uint64(b.Len()))
 
 	if b.size() >= memFree {
 		db.rotateMem(0)
@@ -250,15 +280,19 @@ func (db *DB) CompactRange(r util.Range) error {
 		return err
 	}
 
+	// Lock writer.
 	select {
 	case db.writeLockC <- struct{}{}:
+	case err := <-db.compPerErrC:
+		return err
 	case _, _ = <-db.closeC:
 		return ErrClosed
 	}
 
 	// Check for overlaps in memdb.
 	mem := db.getEffectiveMem()
-	if isMemOverlaps(db.s.icmp, mem, r.Start, r.Limit) {
+	defer mem.decref()
+	if isMemOverlaps(db.s.icmp, mem.mdb, r.Start, r.Limit) {
 		// Memdb compaction.
 		if _, err := db.rotateMem(0); err != nil {
 			<-db.writeLockC

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/doc.go

@@ -37,6 +37,16 @@
 //	err = iter.Error()
 //	...
 //
+// Iterate over subset of database content with a particular prefix:
+//	iter := db.NewIterator(util.BytesPrefix([]byte("foo-")), nil)
+//	for iter.Next() {
+//		// Use key/value.
+//		...
+//	}
+//	iter.Release()
+//	err = iter.Error()
+//	...
+//
 // Seek-then-Iterate:
 //
 // 	iter := db.NewIterator(nil, nil)

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors.go

@@ -7,32 +7,12 @@
 package leveldb
 
 import (
-	"errors"
-
-	"github.com/syndtr/goleveldb/leveldb/util"
+	"github.com/syndtr/goleveldb/leveldb/errors"
 )
 
 var (
-	ErrNotFound         = util.ErrNotFound
+	ErrNotFound         = errors.ErrNotFound
 	ErrSnapshotReleased = errors.New("leveldb: snapshot released")
 	ErrIterReleased     = errors.New("leveldb: iterator released")
 	ErrClosed           = errors.New("leveldb: closed")
 )
-
-type CorruptionType int
-
-const (
-	CorruptedManifest CorruptionType = iota
-	MissingFiles
-)
-
-// ErrCorrupted is the type that wraps errors that indicate corruption in
-// the database.
-type ErrCorrupted struct {
-	Type CorruptionType
-	Err  error
-}
-
-func (e ErrCorrupted) Error() string {
-	return e.Err.Error()
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors/errors.go

@@ -0,0 +1,76 @@
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package errors provides common error types used throughout leveldb.
+package errors
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/syndtr/goleveldb/leveldb/storage"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+var (
+	ErrNotFound    = New("leveldb: not found")
+	ErrReleased    = util.ErrReleased
+	ErrHasReleaser = util.ErrHasReleaser
+)
+
+// New returns an error that formats as the given text.
+func New(text string) error {
+	return errors.New(text)
+}
+
+// ErrCorrupted is the type that wraps errors that indicate corruption in
+// the database.
+type ErrCorrupted struct {
+	File *storage.FileInfo
+	Err  error
+}
+
+func (e *ErrCorrupted) Error() string {
+	if e.File != nil {
+		return fmt.Sprintf("%v [file=%v]", e.Err, e.File)
+	} else {
+		return e.Err.Error()
+	}
+}
+
+// NewErrCorrupted creates new ErrCorrupted error.
+func NewErrCorrupted(f storage.File, err error) error {
+	return &ErrCorrupted{storage.NewFileInfo(f), err}
+}
+
+// IsCorrupted returns a boolean indicating whether the error is indicating
+// a corruption.
+func IsCorrupted(err error) bool {
+	switch err.(type) {
+	case *ErrCorrupted:
+		return true
+	}
+	return false
+}
+
+// ErrMissingFiles is the type that indicating a corruption due to missing
+// files.
+type ErrMissingFiles struct {
+	Files []*storage.FileInfo
+}
+
+func (e *ErrMissingFiles) Error() string { return "file missing" }
+
+// SetFile sets 'file info' of the given error with the given file.
+// Currently only ErrCorrupted is supported, otherwise will do nothing.
+func SetFile(err error, f storage.File) error {
+	switch x := err.(type) {
+	case *ErrCorrupted:
+		x.File = storage.NewFileInfo(f)
+		return x
+	}
+	return err
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/external_test.go

@@ -17,13 +17,14 @@ import (
 var _ = testutil.Defer(func() {
 	Describe("Leveldb external", func() {
 		o := &opt.Options{
-			BlockCache:           opt.NoCache,
-			BlockRestartInterval: 5,
-			BlockSize:            50,
-			Compression:          opt.NoCompression,
-			MaxOpenFiles:         0,
-			Strict:               opt.StrictAll,
-			WriteBuffer:          1000,
+			DisableBlockCache:      true,
+			BlockRestartInterval:   5,
+			BlockSize:              80,
+			Compression:            opt.NoCompression,
+			OpenFilesCacheCapacity: -1,
+			Strict:                 opt.StrictAll,
+			WriteBuffer:            1000,
+			CompactionTableSize:    2000,
 		}
 
 		Describe("write test", func() {
@@ -40,18 +41,17 @@ var _ = testutil.Defer(func() {
 		})
 
 		Describe("read test", func() {
-			testutil.AllKeyValueTesting(nil, func(kv testutil.KeyValue) testutil.DB {
+			testutil.AllKeyValueTesting(nil, nil, func(kv testutil.KeyValue) testutil.DB {
 				// Building the DB.
 				db := newTestingDB(o, nil, nil)
 				kv.IterateShuffled(nil, func(i int, key, value []byte) {
 					err := db.TestPut(key, value)
 					Expect(err).NotTo(HaveOccurred())
 				})
-				testutil.Defer("teardown", func() {
-					db.TestClose()
-				})
 
 				return db
+			}, func(db testutil.DB) {
+				db.(*testingDB).TestClose()
 			})
 		})
 	})

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/go13_bench_test.go

@@ -0,0 +1,58 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// +build go1.3
+
+package leveldb
+
+import (
+	"sync/atomic"
+	"testing"
+)
+
+func BenchmarkDBReadConcurrent(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.fill()
+	p.gc()
+	defer p.close()
+
+	b.ResetTimer()
+	b.SetBytes(116)
+
+	b.RunParallel(func(pb *testing.PB) {
+		iter := p.newIter()
+		defer iter.Release()
+		for pb.Next() && iter.Next() {
+		}
+	})
+}
+
+func BenchmarkDBReadConcurrent2(b *testing.B) {
+	p := openDBBench(b, false)
+	p.populate(b.N)
+	p.fill()
+	p.gc()
+	defer p.close()
+
+	b.ResetTimer()
+	b.SetBytes(116)
+
+	var dir uint32
+	b.RunParallel(func(pb *testing.PB) {
+		iter := p.newIter()
+		defer iter.Release()
+		if atomic.AddUint32(&dir, 1)%2 == 0 {
+			for pb.Next() && iter.Next() {
+			}
+		} else {
+			if pb.Next() && iter.Last() {
+				for pb.Next() && iter.Prev() {
+				}
+			}
+		}
+	})
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/array_iter.go

@@ -40,13 +40,19 @@ type basicArrayIterator struct {
 	util.BasicReleaser
 	array BasicArray
 	pos   int
+	err   error
 }
 
 func (i *basicArrayIterator) Valid() bool {
-	return i.pos >= 0 && i.pos < i.array.Len()
+	return i.pos >= 0 && i.pos < i.array.Len() && !i.Released()
 }
 
 func (i *basicArrayIterator) First() bool {
+	if i.Released() {
+		i.err = ErrIterReleased
+		return false
+	}
+
 	if i.array.Len() == 0 {
 		i.pos = -1
 		return false
@@ -56,6 +62,11 @@ func (i *basicArrayIterator) First() bool {
 }
 
 func (i *basicArrayIterator) Last() bool {
+	if i.Released() {
+		i.err = ErrIterReleased
+		return false
+	}
+
 	n := i.array.Len()
 	if n == 0 {
 		i.pos = 0
@@ -66,6 +77,11 @@ func (i *basicArrayIterator) Last() bool {
 }
 
 func (i *basicArrayIterator) Seek(key []byte) bool {
+	if i.Released() {
+		i.err = ErrIterReleased
+		return false
+	}
+
 	n := i.array.Len()
 	if n == 0 {
 		i.pos = 0
@@ -79,6 +95,11 @@ func (i *basicArrayIterator) Seek(key []byte) bool {
 }
 
 func (i *basicArrayIterator) Next() bool {
+	if i.Released() {
+		i.err = ErrIterReleased
+		return false
+	}
+
 	i.pos++
 	if n := i.array.Len(); i.pos >= n {
 		i.pos = n
@@ -88,6 +109,11 @@ func (i *basicArrayIterator) Next() bool {
 }
 
 func (i *basicArrayIterator) Prev() bool {
+	if i.Released() {
+		i.err = ErrIterReleased
+		return false
+	}
+
 	i.pos--
 	if i.pos < 0 {
 		i.pos = -1
@@ -96,7 +122,7 @@ func (i *basicArrayIterator) Prev() bool {
 	return true
 }
 
-func (i *basicArrayIterator) Error() error { return nil }
+func (i *basicArrayIterator) Error() error { return i.err }
 
 type arrayIterator struct {
 	basicArrayIterator

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter.go

@@ -7,6 +7,7 @@
 package iterator
 
 import (
+	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
@@ -22,13 +23,13 @@ type IteratorIndexer interface {
 
 type indexedIterator struct {
 	util.BasicReleaser
-	index     IteratorIndexer
-	strict    bool
-	strictGet bool
+	index  IteratorIndexer
+	strict bool
 
-	data Iterator
-	err  error
-	errf func(err error)
+	data   Iterator
+	err    error
+	errf   func(err error)
+	closed bool
 }
 
 func (i *indexedIterator) setData() {
@@ -36,11 +37,6 @@ func (i *indexedIterator) setData() {
 		i.data.Release()
 	}
 	i.data = i.index.Get()
-	if i.strictGet {
-		if err := i.data.Error(); err != nil {
-			i.err = err
-		}
-	}
 }
 
 func (i *indexedIterator) clearData() {
@@ -50,14 +46,21 @@ func (i *indexedIterator) clearData() {
 	i.data = nil
 }
 
-func (i *indexedIterator) dataErr() bool {
-	if i.errf != nil {
-		if err := i.data.Error(); err != nil {
+func (i *indexedIterator) indexErr() {
+	if err := i.index.Error(); err != nil {
+		if i.errf != nil {
 			i.errf(err)
 		}
+		i.err = err
 	}
-	if i.strict {
-		if err := i.data.Error(); err != nil {
+}
+
+func (i *indexedIterator) dataErr() bool {
+	if err := i.data.Error(); err != nil {
+		if i.errf != nil {
+			i.errf(err)
+		}
+		if i.strict || !errors.IsCorrupted(err) {
 			i.err = err
 			return true
 		}
@@ -72,9 +75,13 @@ func (i *indexedIterator) Valid() bool {
 func (i *indexedIterator) First() bool {
 	if i.err != nil {
 		return false
+	} else if i.Released() {
+		i.err = ErrIterReleased
+		return false
 	}
 
 	if !i.index.First() {
+		i.indexErr()
 		i.clearData()
 		return false
 	}
@@ -85,9 +92,13 @@ func (i *indexedIterator) First() bool {
 func (i *indexedIterator) Last() bool {
 	if i.err != nil {
 		return false
+	} else if i.Released() {
+		i.err = ErrIterReleased
+		return false
 	}
 
 	if !i.index.Last() {
+		i.indexErr()
 		i.clearData()
 		return false
 	}
@@ -105,9 +116,13 @@ func (i *indexedIterator) Last() bool {
 func (i *indexedIterator) Seek(key []byte) bool {
 	if i.err != nil {
 		return false
+	} else if i.Released() {
+		i.err = ErrIterReleased
+		return false
 	}
 
 	if !i.index.Seek(key) {
+		i.indexErr()
 		i.clearData()
 		return false
 	}
@@ -125,6 +140,9 @@ func (i *indexedIterator) Seek(key []byte) bool {
 func (i *indexedIterator) Next() bool {
 	if i.err != nil {
 		return false
+	} else if i.Released() {
+		i.err = ErrIterReleased
+		return false
 	}
 
 	switch {
@@ -136,6 +154,7 @@ func (i *indexedIterator) Next() bool {
 		fallthrough
 	case i.data == nil:
 		if !i.index.Next() {
+			i.indexErr()
 			return false
 		}
 		i.setData()
@@ -147,6 +166,9 @@ func (i *indexedIterator) Next() bool {
 func (i *indexedIterator) Prev() bool {
 	if i.err != nil {
 		return false
+	} else if i.Released() {
+		i.err = ErrIterReleased
+		return false
 	}
 
 	switch {
@@ -158,6 +180,7 @@ func (i *indexedIterator) Prev() bool {
 		fallthrough
 	case i.data == nil:
 		if !i.index.Prev() {
+			i.indexErr()
 			return false
 		}
 		i.setData()
@@ -206,16 +229,14 @@ func (i *indexedIterator) SetErrorCallback(f func(err error)) {
 	i.errf = f
 }
 
-// NewIndexedIterator returns an indexed iterator. An index is iterator
-// that returns another iterator, a data iterator. A data iterator is the
+// NewIndexedIterator returns an 'indexed iterator'. An index is iterator
+// that returns another iterator, a 'data iterator'. A 'data iterator' is the
 // iterator that contains actual key/value pairs.
 //
-// If strict is true then error yield by data iterator will halt the indexed
-// iterator, on contrary if strict is false then the indexed iterator will
-// ignore those error and move on to the next index. If strictGet is true and
-// index.Get() yield an 'error iterator' then the indexed iterator will be halted.
-// An 'error iterator' is iterator which its Error() method always return non-nil
-// even before any 'seeks method' is called.
-func NewIndexedIterator(index IteratorIndexer, strict, strictGet bool) Iterator {
-	return &indexedIterator{index: index, strict: strict, strictGet: strictGet}
+// If strict is true the any 'corruption errors' (i.e errors.IsCorrupted(err) == true)
+// won't be ignored and will halt 'indexed iterator', otherwise the iterator will
+// continue to the next 'data iterator'. Corruption on 'index iterator' will not be
+// ignored and will halt the iterator.
+func NewIndexedIterator(index IteratorIndexer, strict bool) Iterator {
+	return &indexedIterator{index: index, strict: strict}
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter_test.go

@@ -65,7 +65,7 @@ var _ = testutil.Defer(func() {
 					// Test the iterator.
 					t := testutil.IteratorTesting{
 						KeyValue: kv.Clone(),
-						Iter:     NewIndexedIterator(NewArrayIndexer(index), true, true),
+						Iter:     NewIndexedIterator(NewArrayIndexer(index), true),
 					}
 					testutil.DoIteratorTesting(&t)
 					done <- true

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/iter.go

@@ -14,6 +14,10 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
+var (
+	ErrIterReleased = errors.New("leveldb/iterator: iterator released")
+)
+
 // IteratorSeeker is the interface that wraps the 'seeks method'.
 type IteratorSeeker interface {
 	// First moves the iterator to the first key/value pair. If the iterator
@@ -100,28 +104,13 @@ type ErrorCallbackSetter interface {
 }
 
 type emptyIterator struct {
-	releaser util.Releaser
-	released bool
-	err      error
+	util.BasicReleaser
+	err error
 }
 
 func (i *emptyIterator) rErr() {
-	if i.err == nil && i.released {
-		i.err = errors.New("leveldb/iterator: iterator released")
-	}
-}
-
-func (i *emptyIterator) Release() {
-	if i.releaser != nil {
-		i.releaser.Release()
-		i.releaser = nil
-	}
-	i.released = true
-}
-
-func (i *emptyIterator) SetReleaser(releaser util.Releaser) {
-	if !i.released {
-		i.releaser = releaser
+	if i.err == nil && i.Released() {
+		i.err = ErrIterReleased
 	}
 }
 

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/iter_suite_test.go

@@ -3,15 +3,9 @@ package iterator_test
 import (
 	"testing"
 
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
 	"github.com/syndtr/goleveldb/leveldb/testutil"
 )
 
 func TestIterator(t *testing.T) {
-	testutil.RunDefer()
-
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Iterator Suite")
+	testutil.RunSuite(t, "Iterator Suite")
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter.go

@@ -7,16 +7,11 @@
 package iterator
 
 import (
-	"errors"
-
 	"github.com/syndtr/goleveldb/leveldb/comparer"
+	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
-var (
-	ErrIterReleased = errors.New("leveldb/iterator: iterator released")
-)
-
 type dir int
 
 const (
@@ -48,13 +43,11 @@ func assertKey(key []byte) []byte {
 }
 
 func (i *mergedIterator) iterErr(iter Iterator) bool {
-	if i.errf != nil {
-		if err := iter.Error(); err != nil {
+	if err := iter.Error(); err != nil {
+		if i.errf != nil {
 			i.errf(err)
 		}
-	}
-	if i.strict {
-		if err := iter.Error(); err != nil {
+		if i.strict || !errors.IsCorrupted(err) {
 			i.err = err
 			return true
 		}
@@ -274,9 +267,13 @@ func (i *mergedIterator) Release() {
 }
 
 func (i *mergedIterator) SetReleaser(releaser util.Releaser) {
-	if i.dir != dirReleased {
-		i.releaser = releaser
+	if i.dir == dirReleased {
+		panic(util.ErrReleased)
 	}
+	if i.releaser != nil && releaser != nil {
+		panic(util.ErrHasReleaser)
+	}
+	i.releaser = releaser
 }
 
 func (i *mergedIterator) Error() error {
@@ -294,9 +291,9 @@ func (i *mergedIterator) SetErrorCallback(f func(err error)) {
 // keys: if iters[i] contains a key k then iters[j] will not contain that key k.
 // None of the iters may be nil.
 //
-// If strict is true then error yield by any iterators will halt the merged
-// iterator, on contrary if strict is false then the merged iterator will
-// ignore those error and move on to the next iterator.
+// If strict is true the any 'corruption errors' (i.e errors.IsCorrupted(err) == true)
+// won't be ignored and will halt 'merged iterator', otherwise the iterator will
+// continue to the next 'input iterator'.
 func NewMergedIterator(iters []Iterator, cmp comparer.Comparer, strict bool) Iterator {
 	return &mergedIterator{
 		iters:  iters,

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal/journal.go

@@ -79,10 +79,10 @@ package journal
 
 import (
 	"encoding/binary"
-	"errors"
 	"fmt"
 	"io"
 
+	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
@@ -103,18 +103,18 @@ type flusher interface {
 	Flush() error
 }
 
-// DroppedError is the error type that passed to Dropper.Drop method.
-type DroppedError struct {
+// ErrCorrupted is the error type that generated by corrupted block or chunk.
+type ErrCorrupted struct {
 	Size   int
 	Reason string
 }
 
-func (e DroppedError) Error() string {
-	return fmt.Sprintf("leveldb/journal: dropped %d bytes: %s", e.Size, e.Reason)
+func (e *ErrCorrupted) Error() string {
+	return fmt.Sprintf("leveldb/journal: block/chunk corrupted: %s (%d bytes)", e.Reason, e.Size)
 }
 
 // Dropper is the interface that wrap simple Drop method. The Drop
-// method will be called when the journal reader dropping a chunk.
+// method will be called when the journal reader dropping a block or chunk.
 type Dropper interface {
 	Drop(err error)
 }
@@ -158,76 +158,78 @@ func NewReader(r io.Reader, dropper Dropper, strict, checksum bool) *Reader {
 	}
 }
 
+var errSkip = errors.New("leveldb/journal: skipped")
+
+func (r *Reader) corrupt(n int, reason string, skip bool) error {
+	if r.dropper != nil {
+		r.dropper.Drop(&ErrCorrupted{n, reason})
+	}
+	if r.strict && !skip {
+		r.err = errors.NewErrCorrupted(nil, &ErrCorrupted{n, reason})
+		return r.err
+	}
+	return errSkip
+}
+
 // nextChunk sets r.buf[r.i:r.j] to hold the next chunk's payload, reading the
 // next block into the buffer if necessary.
-func (r *Reader) nextChunk(wantFirst, skip bool) error {
+func (r *Reader) nextChunk(first bool) error {
 	for {
 		if r.j+headerSize <= r.n {
 			checksum := binary.LittleEndian.Uint32(r.buf[r.j+0 : r.j+4])
 			length := binary.LittleEndian.Uint16(r.buf[r.j+4 : r.j+6])
 			chunkType := r.buf[r.j+6]
 
-			var err error
 			if checksum == 0 && length == 0 && chunkType == 0 {
 				// Drop entire block.
-				err = DroppedError{r.n - r.j, "zero header"}
+				m := r.n - r.j
 				r.i = r.n
 				r.j = r.n
+				return r.corrupt(m, "zero header", false)
 			} else {
 				m := r.n - r.j
 				r.i = r.j + headerSize
 				r.j = r.j + headerSize + int(length)
 				if r.j > r.n {
 					// Drop entire block.
-					err = DroppedError{m, "chunk length overflows block"}
 					r.i = r.n
 					r.j = r.n
+					return r.corrupt(m, "chunk length overflows block", false)
 				} else if r.checksum && checksum != util.NewCRC(r.buf[r.i-1:r.j]).Value() {
 					// Drop entire block.
-					err = DroppedError{m, "checksum mismatch"}
 					r.i = r.n
 					r.j = r.n
+					return r.corrupt(m, "checksum mismatch", false)
 				}
 			}
-			if wantFirst && err == nil && chunkType != fullChunkType && chunkType != firstChunkType {
-				if skip {
-					// The chunk are intentionally skipped.
-					if chunkType == lastChunkType {
-						skip = false
-					}
-					continue
-				} else {
-					// Drop the chunk.
-					err = DroppedError{r.j - r.i + headerSize, "orphan chunk"}
-				}
+			if first && chunkType != fullChunkType && chunkType != firstChunkType {
+				m := r.j - r.i
+				r.i = r.j
+				// Report the error, but skip it.
+				return r.corrupt(m+headerSize, "orphan chunk", true)
 			}
-			if err == nil {
-				r.last = chunkType == fullChunkType || chunkType == lastChunkType
-			} else {
-				if r.dropper != nil {
-					r.dropper.Drop(err)
-				}
-				if r.strict {
-					r.err = err
-				}
+			r.last = chunkType == fullChunkType || chunkType == lastChunkType
+			return nil
+		}
+
+		// The last block.
+		if r.n < blockSize && r.n > 0 {
+			if !first {
+				return r.corrupt(0, "missing chunk part", false)
 			}
+			r.err = io.EOF
+			return r.err
+		}
+
+		// Read block.
+		n, err := io.ReadFull(r.r, r.buf[:])
+		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 			return err
 		}
-		if r.n < blockSize && r.n > 0 {
-			// This is the last block.
-			if r.j != r.n {
-				r.err = io.ErrUnexpectedEOF
-			} else {
-				r.err = io.EOF
-			}
-			return r.err
-		}
-		n, err := io.ReadFull(r.r, r.buf[:])
-		if err != nil && err != io.ErrUnexpectedEOF {
-			r.err = err
-			return r.err
-		}
 		if n == 0 {
+			if !first {
+				return r.corrupt(0, "missing chunk part", false)
+			}
 			r.err = io.EOF
 			return r.err
 		}
@@ -237,29 +239,26 @@ func (r *Reader) nextChunk(wantFirst, skip bool) error {
 
 // Next returns a reader for the next journal. It returns io.EOF if there are no
 // more journals. The reader returned becomes stale after the next Next call,
-// and should no longer be used.
+// and should no longer be used. If strict is false, the reader will returns
+// io.ErrUnexpectedEOF error when found corrupted journal.
 func (r *Reader) Next() (io.Reader, error) {
 	r.seq++
 	if r.err != nil {
 		return nil, r.err
 	}
-	skip := !r.last
+	r.i = r.j
 	for {
-		r.i = r.j
-		if r.nextChunk(true, skip) != nil {
-			// So that 'orphan chunk' drop will be reported.
-			skip = false
-		} else {
+		if err := r.nextChunk(true); err == nil {
 			break
-		}
-		if r.err != nil {
-			return nil, r.err
+		} else if err != errSkip {
+			return nil, err
 		}
 	}
 	return &singleReader{r, r.seq, nil}, nil
 }
 
-// Reset resets the journal reader, allows reuse of the journal reader.
+// Reset resets the journal reader, allows reuse of the journal reader. Reset returns
+// last accumulated error.
 func (r *Reader) Reset(reader io.Reader, dropper Dropper, strict, checksum bool) error {
 	r.seq++
 	err := r.err
@@ -296,7 +295,11 @@ func (x *singleReader) Read(p []byte) (int, error) {
 		if r.last {
 			return 0, io.EOF
 		}
-		if x.err = r.nextChunk(false, false); x.err != nil {
+		x.err = r.nextChunk(false)
+		if x.err != nil {
+			if x.err == errSkip {
+				x.err = io.ErrUnexpectedEOF
+			}
 			return 0, x.err
 		}
 	}
@@ -320,7 +323,11 @@ func (x *singleReader) ReadByte() (byte, error) {
 		if r.last {
 			return 0, io.EOF
 		}
-		if x.err = r.nextChunk(false, false); x.err != nil {
+		x.err = r.nextChunk(false)
+		if x.err != nil {
+			if x.err == errSkip {
+				x.err = io.ErrUnexpectedEOF
+			}
 			return 0, x.err
 		}
 	}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal/journal_test.go

@@ -12,6 +12,7 @@ package journal
 
 import (
 	"bytes"
+	"encoding/binary"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -326,3 +327,492 @@ func TestStaleWriter(t *testing.T) {
 		t.Fatalf("stale write #1: unexpected error: %v", err)
 	}
 }
+
+func TestCorrupt_MissingLastBlock(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+
+	// First record.
+	ww, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-1024)); err != nil {
+		t.Fatalf("write #0: unexpected error: %v", err)
+	}
+
+	// Second record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
+		t.Fatalf("write #1: unexpected error: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	// Cut the last block.
+	b := buf.Bytes()[:blockSize]
+	r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
+
+	// First read.
+	rr, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #0: %v", err)
+	}
+	if n != blockSize-1024 {
+		t.Fatalf("read #0: got %d bytes want %d", n, blockSize-1024)
+	}
+
+	// Second read.
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != io.ErrUnexpectedEOF {
+		t.Fatalf("read #1: unexpected error: %v", err)
+	}
+
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("last next: unexpected error: %v", err)
+	}
+}
+
+func TestCorrupt_CorruptedFirstBlock(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+
+	// First record.
+	ww, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
+		t.Fatalf("write #0: unexpected error: %v", err)
+	}
+
+	// Second record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
+		t.Fatalf("write #1: unexpected error: %v", err)
+	}
+
+	// Third record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
+		t.Fatalf("write #2: unexpected error: %v", err)
+	}
+
+	// Fourth record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+2)); err != nil {
+		t.Fatalf("write #3: unexpected error: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	b := buf.Bytes()
+	// Corrupting block #0.
+	for i := 0; i < 1024; i++ {
+		b[i] = '1'
+	}
+
+	r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
+
+	// First read (third record).
+	rr, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #0: %v", err)
+	}
+	if want := int64(blockSize-headerSize) + 1; n != want {
+		t.Fatalf("read #0: got %d bytes want %d", n, want)
+	}
+
+	// Second read (fourth record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #1: %v", err)
+	}
+	if want := int64(blockSize-headerSize) + 2; n != want {
+		t.Fatalf("read #1: got %d bytes want %d", n, want)
+	}
+
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("last next: unexpected error: %v", err)
+	}
+}
+
+func TestCorrupt_CorruptedMiddleBlock(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+
+	// First record.
+	ww, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
+		t.Fatalf("write #0: unexpected error: %v", err)
+	}
+
+	// Second record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
+		t.Fatalf("write #1: unexpected error: %v", err)
+	}
+
+	// Third record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
+		t.Fatalf("write #2: unexpected error: %v", err)
+	}
+
+	// Fourth record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+2)); err != nil {
+		t.Fatalf("write #3: unexpected error: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	b := buf.Bytes()
+	// Corrupting block #1.
+	for i := 0; i < 1024; i++ {
+		b[blockSize+i] = '1'
+	}
+
+	r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
+
+	// First read (first record).
+	rr, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #0: %v", err)
+	}
+	if want := int64(blockSize / 2); n != want {
+		t.Fatalf("read #0: got %d bytes want %d", n, want)
+	}
+
+	// Second read (second record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != io.ErrUnexpectedEOF {
+		t.Fatalf("read #1: unexpected error: %v", err)
+	}
+
+	// Third read (fourth record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #2: %v", err)
+	}
+	if want := int64(blockSize-headerSize) + 2; n != want {
+		t.Fatalf("read #2: got %d bytes want %d", n, want)
+	}
+
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("last next: unexpected error: %v", err)
+	}
+}
+
+func TestCorrupt_CorruptedLastBlock(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+
+	// First record.
+	ww, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
+		t.Fatalf("write #0: unexpected error: %v", err)
+	}
+
+	// Second record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
+		t.Fatalf("write #1: unexpected error: %v", err)
+	}
+
+	// Third record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
+		t.Fatalf("write #2: unexpected error: %v", err)
+	}
+
+	// Fourth record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+2)); err != nil {
+		t.Fatalf("write #3: unexpected error: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	b := buf.Bytes()
+	// Corrupting block #3.
+	for i := len(b) - 1; i > len(b)-1024; i-- {
+		b[i] = '1'
+	}
+
+	r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
+
+	// First read (first record).
+	rr, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #0: %v", err)
+	}
+	if want := int64(blockSize / 2); n != want {
+		t.Fatalf("read #0: got %d bytes want %d", n, want)
+	}
+
+	// Second read (second record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #1: %v", err)
+	}
+	if want := int64(blockSize - headerSize); n != want {
+		t.Fatalf("read #1: got %d bytes want %d", n, want)
+	}
+
+	// Third read (third record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #2: %v", err)
+	}
+	if want := int64(blockSize-headerSize) + 1; n != want {
+		t.Fatalf("read #2: got %d bytes want %d", n, want)
+	}
+
+	// Fourth read (fourth record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != io.ErrUnexpectedEOF {
+		t.Fatalf("read #3: unexpected error: %v", err)
+	}
+
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("last next: unexpected error: %v", err)
+	}
+}
+
+func TestCorrupt_FirstChuckLengthOverflow(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+
+	// First record.
+	ww, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
+		t.Fatalf("write #0: unexpected error: %v", err)
+	}
+
+	// Second record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
+		t.Fatalf("write #1: unexpected error: %v", err)
+	}
+
+	// Third record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
+		t.Fatalf("write #2: unexpected error: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	b := buf.Bytes()
+	// Corrupting record #1.
+	x := blockSize
+	binary.LittleEndian.PutUint16(b[x+4:], 0xffff)
+
+	r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
+
+	// First read (first record).
+	rr, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #0: %v", err)
+	}
+	if want := int64(blockSize / 2); n != want {
+		t.Fatalf("read #0: got %d bytes want %d", n, want)
+	}
+
+	// Second read (second record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != io.ErrUnexpectedEOF {
+		t.Fatalf("read #1: unexpected error: %v", err)
+	}
+
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("last next: unexpected error: %v", err)
+	}
+}
+
+func TestCorrupt_MiddleChuckLengthOverflow(t *testing.T) {
+	buf := new(bytes.Buffer)
+
+	w := NewWriter(buf)
+
+	// First record.
+	ww, err := w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
+		t.Fatalf("write #0: unexpected error: %v", err)
+	}
+
+	// Second record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
+		t.Fatalf("write #1: unexpected error: %v", err)
+	}
+
+	// Third record.
+	ww, err = w.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
+		t.Fatalf("write #2: unexpected error: %v", err)
+	}
+
+	if err := w.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	b := buf.Bytes()
+	// Corrupting record #1.
+	x := blockSize/2 + headerSize
+	binary.LittleEndian.PutUint16(b[x+4:], 0xffff)
+
+	r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
+
+	// First read (first record).
+	rr, err := r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err := io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #0: %v", err)
+	}
+	if want := int64(blockSize / 2); n != want {
+		t.Fatalf("read #0: got %d bytes want %d", n, want)
+	}
+
+	// Second read (third record).
+	rr, err = r.Next()
+	if err != nil {
+		t.Fatal(err)
+	}
+	n, err = io.Copy(ioutil.Discard, rr)
+	if err != nil {
+		t.Fatalf("read #1: %v", err)
+	}
+	if want := int64(blockSize-headerSize) + 1; n != want {
+		t.Fatalf("read #1: got %d bytes want %d", n, want)
+	}
+
+	if _, err := r.Next(); err != io.EOF {
+		t.Fatalf("last next: unexpected error: %v", err)
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/key.go

@@ -9,15 +9,30 @@ package leveldb
 import (
 	"encoding/binary"
 	"fmt"
+
+	"github.com/syndtr/goleveldb/leveldb/errors"
 )
 
-type vType int
+type ErrIkeyCorrupted struct {
+	Ikey   []byte
+	Reason string
+}
 
-func (t vType) String() string {
-	switch t {
-	case tDel:
+func (e *ErrIkeyCorrupted) Error() string {
+	return fmt.Sprintf("leveldb: iKey %q corrupted: %s", e.Ikey, e.Reason)
+}
+
+func newErrIkeyCorrupted(ikey []byte, reason string) error {
+	return errors.NewErrCorrupted(nil, &ErrIkeyCorrupted{append([]byte{}, ikey...), reason})
+}
+
+type kType int
+
+func (kt kType) String() string {
+	switch kt {
+	case ktDel:
 		return "d"
-	case tVal:
+	case ktVal:
 		return "v"
 	}
 	return "x"
@@ -26,16 +41,16 @@ func (t vType) String() string {
 // Value types encoded as the last component of internal keys.
 // Don't modify; this value are saved to disk.
 const (
-	tDel vType = iota
-	tVal
+	ktDel kType = iota
+	ktVal
 )
 
-// tSeek defines the vType that should be passed when constructing an
+// ktSeek defines the kType that should be passed when constructing an
 // internal key for seeking to a particular sequence number (since we
 // sort sequence numbers in decreasing order and the value type is
 // embedded as the low 8 bits in the sequence number in internal keys,
 // we need to use the highest-numbered ValueType, not the lowest).
-const tSeek = tVal
+const ktSeek = ktVal
 
 const (
 	// Maximum value possible for sequence number; the 8-bits are
@@ -43,7 +58,7 @@ const (
 	// 64-bit integer.
 	kMaxSeq uint64 = (uint64(1) << 56) - 1
 	// Maximum value possible for packed sequence number and type.
-	kMaxNum uint64 = (kMaxSeq << 8) | uint64(tSeek)
+	kMaxNum uint64 = (kMaxSeq << 8) | uint64(ktSeek)
 )
 
 // Maximum number encoded in bytes.
@@ -55,85 +70,73 @@ func init() {
 
 type iKey []byte
 
-func newIKey(ukey []byte, seq uint64, t vType) iKey {
-	if seq > kMaxSeq || t > tVal {
-		panic("invalid seq number or value type")
+func newIkey(ukey []byte, seq uint64, kt kType) iKey {
+	if seq > kMaxSeq {
+		panic("leveldb: invalid sequence number")
+	} else if kt > ktVal {
+		panic("leveldb: invalid type")
 	}
 
-	b := make(iKey, len(ukey)+8)
-	copy(b, ukey)
-	binary.LittleEndian.PutUint64(b[len(ukey):], (seq<<8)|uint64(t))
-	return b
+	ik := make(iKey, len(ukey)+8)
+	copy(ik, ukey)
+	binary.LittleEndian.PutUint64(ik[len(ukey):], (seq<<8)|uint64(kt))
+	return ik
 }
 
-func parseIkey(p []byte) (ukey []byte, seq uint64, t vType, ok bool) {
-	if len(p) < 8 {
-		return
+func parseIkey(ik []byte) (ukey []byte, seq uint64, kt kType, err error) {
+	if len(ik) < 8 {
+		return nil, 0, 0, newErrIkeyCorrupted(ik, "invalid length")
 	}
-	num := binary.LittleEndian.Uint64(p[len(p)-8:])
-	seq, t = uint64(num>>8), vType(num&0xff)
-	if t > tVal {
-		return
+	num := binary.LittleEndian.Uint64(ik[len(ik)-8:])
+	seq, kt = uint64(num>>8), kType(num&0xff)
+	if kt > ktVal {
+		return nil, 0, 0, newErrIkeyCorrupted(ik, "invalid type")
 	}
-	ukey = p[:len(p)-8]
-	ok = true
+	ukey = ik[:len(ik)-8]
 	return
 }
 
-func validIkey(p []byte) bool {
-	_, _, _, ok := parseIkey(p)
-	return ok
+func validIkey(ik []byte) bool {
+	_, _, _, err := parseIkey(ik)
+	return err == nil
 }
 
-func (p iKey) assert() {
-	if p == nil {
-		panic("nil iKey")
+func (ik iKey) assert() {
+	if ik == nil {
+		panic("leveldb: nil iKey")
 	}
-	if len(p) < 8 {
-		panic(fmt.Sprintf("invalid iKey %q, len=%d", []byte(p), len(p)))
+	if len(ik) < 8 {
+		panic(fmt.Sprintf("leveldb: iKey %q, len=%d: invalid length", []byte(ik), len(ik)))
 	}
 }
 
-func (p iKey) ok() bool {
-	if len(p) < 8 {
-		return false
-	}
-	_, _, ok := p.parseNum()
-	return ok
+func (ik iKey) ukey() []byte {
+	ik.assert()
+	return ik[:len(ik)-8]
 }
 
-func (p iKey) ukey() []byte {
-	p.assert()
-	return p[:len(p)-8]
+func (ik iKey) num() uint64 {
+	ik.assert()
+	return binary.LittleEndian.Uint64(ik[len(ik)-8:])
 }
 
-func (p iKey) num() uint64 {
-	p.assert()
-	return binary.LittleEndian.Uint64(p[len(p)-8:])
-}
-
-func (p iKey) parseNum() (seq uint64, t vType, ok bool) {
-	if p == nil {
-		panic("nil iKey")
+func (ik iKey) parseNum() (seq uint64, kt kType) {
+	num := ik.num()
+	seq, kt = uint64(num>>8), kType(num&0xff)
+	if kt > ktVal {
+		panic(fmt.Sprintf("leveldb: iKey %q, len=%d: invalid type %#x", []byte(ik), len(ik), kt))
 	}
-	if len(p) < 8 {
-		return
-	}
-	num := p.num()
-	seq, t = uint64(num>>8), vType(num&0xff)
-	if t > tVal {
-		return 0, 0, false
-	}
-	ok = true
 	return
 }
 
-func (p iKey) String() string {
-	if len(p) == 0 {
+func (ik iKey) String() string {
+	if ik == nil {
 		return "<nil>"
 	}
-	if seq, t, ok := p.parseNum(); ok {
-		return fmt.Sprintf("%s,%s%d", shorten(string(p.ukey())), t, seq)
+
+	if ukey, seq, kt, err := parseIkey(ik); err == nil {
+		return fmt.Sprintf("%s,%s%d", shorten(string(ukey)), kt, seq)
+	} else {
+		return "<invalid>"
 	}
-	return "<invalid>"
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/key_test.go

@@ -15,8 +15,8 @@ import (
 
 var defaultIComparer = &iComparer{comparer.DefaultComparer}
 
-func ikey(key string, seq uint64, t vType) iKey {
-	return newIKey([]byte(key), uint64(seq), t)
+func ikey(key string, seq uint64, kt kType) iKey {
+	return newIkey([]byte(key), uint64(seq), kt)
 }
 
 func shortSep(a, b []byte) []byte {
@@ -37,27 +37,37 @@ func shortSuccessor(b []byte) []byte {
 	return dst
 }
 
-func testSingleKey(t *testing.T, key string, seq uint64, vt vType) {
-	ik := ikey(key, seq, vt)
+func testSingleKey(t *testing.T, key string, seq uint64, kt kType) {
+	ik := ikey(key, seq, kt)
 
 	if !bytes.Equal(ik.ukey(), []byte(key)) {
 		t.Errorf("user key does not equal, got %v, want %v", string(ik.ukey()), key)
 	}
 
-	if rseq, rt, ok := ik.parseNum(); ok {
+	rseq, rt := ik.parseNum()
+	if rseq != seq {
+		t.Errorf("seq number does not equal, got %v, want %v", rseq, seq)
+	}
+	if rt != kt {
+		t.Errorf("type does not equal, got %v, want %v", rt, kt)
+	}
+
+	if rukey, rseq, rt, kerr := parseIkey(ik); kerr == nil {
+		if !bytes.Equal(rukey, []byte(key)) {
+			t.Errorf("user key does not equal, got %v, want %v", string(ik.ukey()), key)
+		}
 		if rseq != seq {
 			t.Errorf("seq number does not equal, got %v, want %v", rseq, seq)
 		}
-
-		if rt != vt {
-			t.Errorf("type does not equal, got %v, want %v", rt, vt)
+		if rt != kt {
+			t.Errorf("type does not equal, got %v, want %v", rt, kt)
 		}
 	} else {
-		t.Error("cannot parse seq and type")
+		t.Errorf("key error: %v", kerr)
 	}
 }
 
-func TestIKey_EncodeDecode(t *testing.T) {
+func TestIkey_EncodeDecode(t *testing.T) {
 	keys := []string{"", "k", "hello", "longggggggggggggggggggggg"}
 	seqs := []uint64{
 		1, 2, 3,
@@ -67,8 +77,8 @@ func TestIKey_EncodeDecode(t *testing.T) {
 	}
 	for _, key := range keys {
 		for _, seq := range seqs {
-			testSingleKey(t, key, seq, tVal)
-			testSingleKey(t, "hello", 1, tDel)
+			testSingleKey(t, key, seq, ktVal)
+			testSingleKey(t, "hello", 1, ktDel)
 		}
 	}
 }
@@ -79,45 +89,45 @@ func assertBytes(t *testing.T, want, got []byte) {
 	}
 }
 
-func TestIKeyShortSeparator(t *testing.T) {
+func TestIkeyShortSeparator(t *testing.T) {
 	// When user keys are same
-	assertBytes(t, ikey("foo", 100, tVal),
-		shortSep(ikey("foo", 100, tVal),
-			ikey("foo", 99, tVal)))
-	assertBytes(t, ikey("foo", 100, tVal),
-		shortSep(ikey("foo", 100, tVal),
-			ikey("foo", 101, tVal)))
-	assertBytes(t, ikey("foo", 100, tVal),
-		shortSep(ikey("foo", 100, tVal),
-			ikey("foo", 100, tVal)))
-	assertBytes(t, ikey("foo", 100, tVal),
-		shortSep(ikey("foo", 100, tVal),
-			ikey("foo", 100, tDel)))
+	assertBytes(t, ikey("foo", 100, ktVal),
+		shortSep(ikey("foo", 100, ktVal),
+			ikey("foo", 99, ktVal)))
+	assertBytes(t, ikey("foo", 100, ktVal),
+		shortSep(ikey("foo", 100, ktVal),
+			ikey("foo", 101, ktVal)))
+	assertBytes(t, ikey("foo", 100, ktVal),
+		shortSep(ikey("foo", 100, ktVal),
+			ikey("foo", 100, ktVal)))
+	assertBytes(t, ikey("foo", 100, ktVal),
+		shortSep(ikey("foo", 100, ktVal),
+			ikey("foo", 100, ktDel)))
 
 	// When user keys are misordered
-	assertBytes(t, ikey("foo", 100, tVal),
-		shortSep(ikey("foo", 100, tVal),
-			ikey("bar", 99, tVal)))
+	assertBytes(t, ikey("foo", 100, ktVal),
+		shortSep(ikey("foo", 100, ktVal),
+			ikey("bar", 99, ktVal)))
 
 	// When user keys are different, but correctly ordered
-	assertBytes(t, ikey("g", uint64(kMaxSeq), tSeek),
-		shortSep(ikey("foo", 100, tVal),
-			ikey("hello", 200, tVal)))
+	assertBytes(t, ikey("g", uint64(kMaxSeq), ktSeek),
+		shortSep(ikey("foo", 100, ktVal),
+			ikey("hello", 200, ktVal)))
 
 	// When start user key is prefix of limit user key
-	assertBytes(t, ikey("foo", 100, tVal),
-		shortSep(ikey("foo", 100, tVal),
-			ikey("foobar", 200, tVal)))
+	assertBytes(t, ikey("foo", 100, ktVal),
+		shortSep(ikey("foo", 100, ktVal),
+			ikey("foobar", 200, ktVal)))
 
 	// When limit user key is prefix of start user key
-	assertBytes(t, ikey("foobar", 100, tVal),
-		shortSep(ikey("foobar", 100, tVal),
-			ikey("foo", 200, tVal)))
+	assertBytes(t, ikey("foobar", 100, ktVal),
+		shortSep(ikey("foobar", 100, ktVal),
+			ikey("foo", 200, ktVal)))
 }
 
-func TestIKeyShortestSuccessor(t *testing.T) {
-	assertBytes(t, ikey("g", uint64(kMaxSeq), tSeek),
-		shortSuccessor(ikey("foo", 100, tVal)))
-	assertBytes(t, ikey("\xff\xff", 100, tVal),
-		shortSuccessor(ikey("\xff\xff", 100, tVal)))
+func TestIkeyShortestSuccessor(t *testing.T) {
+	assertBytes(t, ikey("g", uint64(kMaxSeq), ktSeek),
+		shortSuccessor(ikey("foo", 100, ktVal)))
+	assertBytes(t, ikey("\xff\xff", 100, ktVal),
+		shortSuccessor(ikey("\xff\xff", 100, ktVal)))
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/leveldb_suite_test.go

@@ -3,18 +3,9 @@ package leveldb
 import (
 	"testing"
 
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
 	"github.com/syndtr/goleveldb/leveldb/testutil"
 )
 
-func TestLeveldb(t *testing.T) {
-	testutil.RunDefer()
-
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Leveldb Suite")
-
-	RegisterTestingT(t)
-	testutil.RunDefer("teardown")
+func TestLevelDB(t *testing.T) {
+	testutil.RunSuite(t, "LevelDB Suite")
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go

@@ -12,12 +12,14 @@ import (
 	"sync"
 
 	"github.com/syndtr/goleveldb/leveldb/comparer"
+	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
 var (
-	ErrNotFound = util.ErrNotFound
+	ErrNotFound     = errors.ErrNotFound
+	ErrIterReleased = errors.New("leveldb/memdb: iterator released")
 )
 
 const tMaxHeight = 12
@@ -29,6 +31,7 @@ type dbIter struct {
 	node       int
 	forward    bool
 	key, value []byte
+	err        error
 }
 
 func (i *dbIter) fill(checkStart, checkLimit bool) bool {
@@ -59,6 +62,11 @@ func (i *dbIter) Valid() bool {
 }
 
 func (i *dbIter) First() bool {
+	if i.Released() {
+		i.err = ErrIterReleased
+		return false
+	}
+
 	i.forward = true
 	i.p.mu.RLock()
 	defer i.p.mu.RUnlock()
@@ -71,9 +79,11 @@ func (i *dbIter) First() bool {
 }
 
 func (i *dbIter) Last() bool {
-	if i.p == nil {
+	if i.Released() {
+		i.err = ErrIterReleased
 		return false
 	}
+
 	i.forward = false
 	i.p.mu.RLock()
 	defer i.p.mu.RUnlock()
@@ -86,9 +96,11 @@ func (i *dbIter) Last() bool {
 }
 
 func (i *dbIter) Seek(key []byte) bool {
-	if i.p == nil {
+	if i.Released() {
+		i.err = ErrIterReleased
 		return false
 	}
+
 	i.forward = true
 	i.p.mu.RLock()
 	defer i.p.mu.RUnlock()
@@ -100,9 +112,11 @@ func (i *dbIter) Seek(key []byte) bool {
 }
 
 func (i *dbIter) Next() bool {
-	if i.p == nil {
+	if i.Released() {
+		i.err = ErrIterReleased
 		return false
 	}
+
 	if i.node == 0 {
 		if !i.forward {
 			return i.First()
@@ -117,9 +131,11 @@ func (i *dbIter) Next() bool {
 }
 
 func (i *dbIter) Prev() bool {
-	if i.p == nil {
+	if i.Released() {
+		i.err = ErrIterReleased
 		return false
 	}
+
 	if i.node == 0 {
 		if i.forward {
 			return i.Last()
@@ -141,10 +157,10 @@ func (i *dbIter) Value() []byte {
 	return i.value
 }
 
-func (i *dbIter) Error() error { return nil }
+func (i *dbIter) Error() error { return i.err }
 
 func (i *dbIter) Release() {
-	if i.p != nil {
+	if !i.Released() {
 		i.p = nil
 		i.node = 0
 		i.key = nil

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb_suite_test.go

@@ -3,15 +3,9 @@ package memdb
 import (
 	"testing"
 
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
 	"github.com/syndtr/goleveldb/leveldb/testutil"
 )
 
-func TestMemdb(t *testing.T) {
-	testutil.RunDefer()
-
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Memdb Suite")
+func TestMemDB(t *testing.T) {
+	testutil.RunSuite(t, "MemDB Suite")
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb_test.go

@@ -129,7 +129,7 @@ var _ = testutil.Defer(func() {
 				}
 
 				return db
-			})
+			}, nil, nil)
 		})
 	})
 })

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go

@@ -11,6 +11,7 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/cache"
 	"github.com/syndtr/goleveldb/leveldb/comparer"
 	"github.com/syndtr/goleveldb/leveldb/filter"
+	"math"
 )
 
 const (
@@ -19,25 +20,56 @@ const (
 	GiB = MiB * 1024
 )
 
-const (
-	DefaultBlockCacheSize       = 8 * MiB
-	DefaultBlockRestartInterval = 16
-	DefaultBlockSize            = 4 * KiB
-	DefaultCompressionType      = SnappyCompression
-	DefaultMaxOpenFiles         = 1000
-	DefaultWriteBuffer          = 4 * MiB
+var (
+	DefaultBlockCacher                   = LRUCacher
+	DefaultBlockCacheCapacity            = 8 * MiB
+	DefaultBlockRestartInterval          = 16
+	DefaultBlockSize                     = 4 * KiB
+	DefaultCompactionExpandLimitFactor   = 25
+	DefaultCompactionGPOverlapsFactor    = 10
+	DefaultCompactionL0Trigger           = 4
+	DefaultCompactionSourceLimitFactor   = 1
+	DefaultCompactionTableSize           = 2 * MiB
+	DefaultCompactionTableSizeMultiplier = 1.0
+	DefaultCompactionTotalSize           = 10 * MiB
+	DefaultCompactionTotalSizeMultiplier = 10.0
+	DefaultCompressionType               = SnappyCompression
+	DefaultOpenFilesCacher               = LRUCacher
+	DefaultOpenFilesCacheCapacity        = 500
+	DefaultMaxMemCompationLevel          = 2
+	DefaultNumLevel                      = 7
+	DefaultWriteBuffer                   = 4 * MiB
+	DefaultWriteL0PauseTrigger           = 12
+	DefaultWriteL0SlowdownTrigger        = 8
 )
 
-type noCache struct{}
+// Cacher is a caching algorithm.
+type Cacher interface {
+	New(capacity int) cache.Cacher
+}
 
-func (noCache) SetCapacity(capacity int)               {}
-func (noCache) GetNamespace(id uint64) cache.Namespace { return nil }
-func (noCache) Purge(fin cache.PurgeFin)               {}
-func (noCache) Zap(closed bool)                        {}
+type CacherFunc struct {
+	NewFunc func(capacity int) cache.Cacher
+}
 
-var NoCache cache.Cache = noCache{}
+func (f *CacherFunc) New(capacity int) cache.Cacher {
+	if f.NewFunc != nil {
+		return f.NewFunc(capacity)
+	}
+	return nil
+}
 
-// Compression is the per-block compression algorithm to use.
+func noCacher(int) cache.Cacher { return nil }
+
+var (
+	// LRUCacher is the LRU-cache algorithm.
+	LRUCacher = &CacherFunc{cache.NewLRU}
+
+	// NoCacher is the value to disable caching algorithm.
+	NoCacher = &CacherFunc{}
+)
+
+// Compression is the 'sorted table' block compression algorithm to use.
 type Compression uint
 
 func (c Compression) String() string {
@@ -59,34 +91,47 @@ const (
 	nCompression
 )
 
-// Strict is the DB strict level.
+// Strict is the DB 'strict level'.
 type Strict uint
 
 const (
 	// If present then a corrupted or invalid chunk or block in manifest
-	// journal will cause an error istead of being dropped.
+	// journal will cause an error instead of being dropped.
+	// This will prevent database with corrupted manifest to be opened.
 	StrictManifest Strict = 1 << iota
 
-	// If present then a corrupted or invalid chunk or block in journal
-	// will cause an error istead of being dropped.
-	StrictJournal
-
 	// If present then journal chunk checksum will be verified.
 	StrictJournalChecksum
 
-	// If present then an invalid key/value pair will cause an error
-	// instead of being skipped.
-	StrictIterator
+	// If present then a corrupted or invalid chunk or block in journal
+	// will cause an error instead of being dropped.
+	// This will prevent database with corrupted journal to be opened.
+	StrictJournal
 
 	// If present then 'sorted table' block checksum will be verified.
+	// This has effect on both 'read operation' and compaction.
 	StrictBlockChecksum
 
+	// If present then a corrupted 'sorted table' will fails compaction.
+	// The database will enter read-only mode.
+	StrictCompaction
+
+	// If present then a corrupted 'sorted table' will halts 'read operation'.
+	StrictReader
+
+	// If present then leveldb.Recover will drop corrupted 'sorted table'.
+	StrictRecovery
+
+	// This only applicable for ReadOptions, if present then this ReadOptions
+	// 'strict level' will override global ones.
+	StrictOverride
+
 	// StrictAll enables all strict flags.
-	StrictAll = StrictManifest | StrictJournal | StrictJournalChecksum | StrictIterator | StrictBlockChecksum
+	StrictAll = StrictManifest | StrictJournalChecksum | StrictJournal | StrictBlockChecksum | StrictCompaction | StrictReader | StrictRecovery
 
 	// DefaultStrict is the default strict flags. Specify any strict flags
 	// will override default strict flags as whole (i.e. not OR'ed).
-	DefaultStrict = StrictJournalChecksum | StrictBlockChecksum
+	DefaultStrict = StrictJournalChecksum | StrictBlockChecksum | StrictCompaction | StrictReader
 
 	// NoStrict disables all strict flags. Override default strict flags.
 	NoStrict = ^StrictAll
@@ -101,11 +146,17 @@ type Options struct {
 	// The default value is nil
 	AltFilters []filter.Filter
 
-	// BlockCache provides per-block caching for LevelDB. Specify NoCache to
-	// disable block caching.
+	// BlockCacher provides cache algorithm for LevelDB 'sorted table' block caching.
+	// Specify NoCacher to disable caching algorithm.
 	//
-	// By default LevelDB will create LRU-cache with capacity of 8MiB.
-	BlockCache cache.Cache
+	// The default value is LRUCacher.
+	BlockCacher Cacher
+
+	// BlockCacheCapacity defines the capacity of the 'sorted table' block caching.
+	// Use -1 for zero, this has same effect with specifying NoCacher to BlockCacher.
+	//
+	// The default value is 8MiB.
+	BlockCacheCapacity int
 
 	// BlockRestartInterval is the number of keys between restart points for
 	// delta encoding of keys.
@@ -119,6 +170,73 @@ type Options struct {
 	// The default value is 4KiB.
 	BlockSize int
 
+	// CompactionExpandLimitFactor limits compaction size after expanded.
+	// This will be multiplied by table size limit at compaction target level.
+	//
+	// The default value is 25.
+	CompactionExpandLimitFactor int
+
+	// CompactionGPOverlapsFactor limits overlaps in grandparent (Level + 2) that a
+	// single 'sorted table' generates.
+	// This will be multiplied by table size limit at grandparent level.
+	//
+	// The default value is 10.
+	CompactionGPOverlapsFactor int
+
+	// CompactionL0Trigger defines number of 'sorted table' at level-0 that will
+	// trigger compaction.
+	//
+	// The default value is 4.
+	CompactionL0Trigger int
+
+	// CompactionSourceLimitFactor limits compaction source size. This doesn't apply to
+	// level-0.
+	// This will be multiplied by table size limit at compaction target level.
+	//
+	// The default value is 1.
+	CompactionSourceLimitFactor int
+
+	// CompactionTableSize limits size of 'sorted table' that compaction generates.
+	// The limits for each level will be calculated as:
+	//   CompactionTableSize * (CompactionTableSizeMultiplier ^ Level)
+	// The multiplier for each level can also fine-tuned using CompactionTableSizeMultiplierPerLevel.
+	//
+	// The default value is 2MiB.
+	CompactionTableSize int
+
+	// CompactionTableSizeMultiplier defines multiplier for CompactionTableSize.
+	//
+	// The default value is 1.
+	CompactionTableSizeMultiplier float64
+
+	// CompactionTableSizeMultiplierPerLevel defines per-level multiplier for
+	// CompactionTableSize.
+	// Use zero to skip a level.
+	//
+	// The default value is nil.
+	CompactionTableSizeMultiplierPerLevel []float64
+
+	// CompactionTotalSize limits total size of 'sorted table' for each level.
+	// The limits for each level will be calculated as:
+	//   CompactionTotalSize * (CompactionTotalSizeMultiplier ^ Level)
+	// The multiplier for each level can also fine-tuned using
+	// CompactionTotalSizeMultiplierPerLevel.
+	//
+	// The default value is 10MiB.
+	CompactionTotalSize int
+
+	// CompactionTotalSizeMultiplier defines multiplier for CompactionTotalSize.
+	//
+	// The default value is 10.
+	CompactionTotalSizeMultiplier float64
+
+	// CompactionTotalSizeMultiplierPerLevel defines per-level multiplier for
+	// CompactionTotalSize.
+	// Use zero to skip a level.
+	//
+	// The default value is nil.
+	CompactionTotalSizeMultiplierPerLevel []float64
+
 	// Comparer defines a total ordering over the space of []byte keys: a 'less
 	// than' relationship. The same comparison algorithm must be used for reads
 	// and writes over the lifetime of the DB.
@@ -126,11 +244,22 @@ type Options struct {
 	// The default value uses the same ordering as bytes.Compare.
 	Comparer comparer.Comparer
 
-	// Compression defines the per-block compression to use.
+	// Compression defines the 'sorted table' block compression to use.
 	//
 	// The default value (DefaultCompression) uses snappy compression.
 	Compression Compression
 
+	// DisableBlockCache allows disable use of cache.Cache functionality on
+	// 'sorted table' block.
+	//
+	// The default value is false.
+	DisableBlockCache bool
+
+	// DisableCompactionBackoff allows disable compaction retry backoff.
+	//
+	// The default value is false.
+	DisableCompactionBackoff bool
+
 	// ErrorIfExist defines whether an error should returned if the DB already
 	// exist.
 	//
@@ -159,12 +288,30 @@ type Options struct {
 	// The default value is nil.
 	Filter filter.Filter
 
-	// MaxOpenFiles defines maximum number of open files to kept around
-	// (cached). This is not an hard limit, actual open files may exceed
-	// the defined value.
+	// MaxMemCompationLevel defines maximum level a newly compacted 'memdb'
+	// will be pushed into if doesn't creates overlap. This should less than
+	// NumLevel. Use -1 for level-0.
 	//
-	// The default value is 1000.
-	MaxOpenFiles int
+	// The default is 2.
+	MaxMemCompationLevel int
+
+	// NumLevel defines number of database level. The level shouldn't changed
+	// between opens, or the database will panic.
+	//
+	// The default is 7.
+	NumLevel int
+
+	// OpenFilesCacher provides cache algorithm for open files caching.
+	// Specify NoCacher to disable caching algorithm.
+	//
+	// The default value is LRUCacher.
+	OpenFilesCacher Cacher
+
+	// OpenFilesCacheCapacity defines the capacity of the open files caching.
+	// Use -1 for zero, this has same effect with specifying NoCacher to OpenFilesCacher.
+	//
+	// The default value is 500.
+	OpenFilesCacheCapacity int
 
 	// Strict defines the DB strict level.
 	Strict Strict
@@ -177,6 +324,18 @@ type Options struct {
 	//
 	// The default value is 4MiB.
 	WriteBuffer int
+
+	// WriteL0StopTrigger defines number of 'sorted table' at level-0 that will
+	// pause write.
+	//
+	// The default value is 12.
+	WriteL0PauseTrigger int
+
+	// WriteL0SlowdownTrigger defines number of 'sorted table' at level-0 that
+	// will trigger write slowdown.
+	//
+	// The default value is 8.
+	WriteL0SlowdownTrigger int
 }
 
 func (o *Options) GetAltFilters() []filter.Filter {
@@ -186,11 +345,22 @@ func (o *Options) GetAltFilters() []filter.Filter {
 	return o.AltFilters
 }
 
-func (o *Options) GetBlockCache() cache.Cache {
-	if o == nil {
+func (o *Options) GetBlockCacher() Cacher {
+	if o == nil || o.BlockCacher == nil {
+		return DefaultBlockCacher
+	} else if o.BlockCacher == NoCacher {
 		return nil
 	}
-	return o.BlockCache
+	return o.BlockCacher
+}
+
+func (o *Options) GetBlockCacheCapacity() int {
+	if o == nil || o.BlockCacheCapacity <= 0 {
+		return DefaultBlockCacheCapacity
+	} else if o.BlockCacheCapacity == -1 {
+		return 0
+	}
+	return o.BlockCacheCapacity
 }
 
 func (o *Options) GetBlockRestartInterval() int {
@@ -207,6 +377,79 @@ func (o *Options) GetBlockSize() int {
 	return o.BlockSize
 }
 
+func (o *Options) GetCompactionExpandLimit(level int) int {
+	factor := DefaultCompactionExpandLimitFactor
+	if o != nil && o.CompactionExpandLimitFactor > 0 {
+		factor = o.CompactionExpandLimitFactor
+	}
+	return o.GetCompactionTableSize(level+1) * factor
+}
+
+func (o *Options) GetCompactionGPOverlaps(level int) int {
+	factor := DefaultCompactionGPOverlapsFactor
+	if o != nil && o.CompactionGPOverlapsFactor > 0 {
+		factor = o.CompactionGPOverlapsFactor
+	}
+	return o.GetCompactionTableSize(level+2) * factor
+}
+
+func (o *Options) GetCompactionL0Trigger() int {
+	if o == nil || o.CompactionL0Trigger == 0 {
+		return DefaultCompactionL0Trigger
+	}
+	return o.CompactionL0Trigger
+}
+
+func (o *Options) GetCompactionSourceLimit(level int) int {
+	factor := DefaultCompactionSourceLimitFactor
+	if o != nil && o.CompactionSourceLimitFactor > 0 {
+		factor = o.CompactionSourceLimitFactor
+	}
+	return o.GetCompactionTableSize(level+1) * factor
+}
+
+func (o *Options) GetCompactionTableSize(level int) int {
+	var (
+		base = DefaultCompactionTableSize
+		mult float64
+	)
+	if o != nil {
+		if o.CompactionTableSize > 0 {
+			base = o.CompactionTableSize
+		}
+		if len(o.CompactionTableSizeMultiplierPerLevel) > level && o.CompactionTableSizeMultiplierPerLevel[level] > 0 {
+			mult = o.CompactionTableSizeMultiplierPerLevel[level]
+		} else if o.CompactionTableSizeMultiplier > 0 {
+			mult = math.Pow(o.CompactionTableSizeMultiplier, float64(level))
+		}
+	}
+	if mult == 0 {
+		mult = math.Pow(DefaultCompactionTableSizeMultiplier, float64(level))
+	}
+	return int(float64(base) * mult)
+}
+
+func (o *Options) GetCompactionTotalSize(level int) int64 {
+	var (
+		base = DefaultCompactionTotalSize
+		mult float64
+	)
+	if o != nil {
+		if o.CompactionTotalSize > 0 {
+			base = o.CompactionTotalSize
+		}
+		if len(o.CompactionTotalSizeMultiplierPerLevel) > level && o.CompactionTotalSizeMultiplierPerLevel[level] > 0 {
+			mult = o.CompactionTotalSizeMultiplierPerLevel[level]
+		} else if o.CompactionTotalSizeMultiplier > 0 {
+			mult = math.Pow(o.CompactionTotalSizeMultiplier, float64(level))
+		}
+	}
+	if mult == 0 {
+		mult = math.Pow(DefaultCompactionTotalSizeMultiplier, float64(level))
+	}
+	return int64(float64(base) * mult)
+}
+
 func (o *Options) GetComparer() comparer.Comparer {
 	if o == nil || o.Comparer == nil {
 		return comparer.DefaultComparer
@@ -221,6 +464,13 @@ func (o *Options) GetCompression() Compression {
 	return o.Compression
 }
 
+func (o *Options) GetDisableCompactionBackoff() bool {
+	if o == nil {
+		return false
+	}
+	return o.DisableCompactionBackoff
+}
+
 func (o *Options) GetErrorIfExist() bool {
 	if o == nil {
 		return false
@@ -242,11 +492,45 @@ func (o *Options) GetFilter() filter.Filter {
 	return o.Filter
 }
 
-func (o *Options) GetMaxOpenFiles() int {
-	if o == nil || o.MaxOpenFiles <= 0 {
-		return DefaultMaxOpenFiles
+func (o *Options) GetMaxMemCompationLevel() int {
+	level := DefaultMaxMemCompationLevel
+	if o != nil {
+		if o.MaxMemCompationLevel > 0 {
+			level = o.MaxMemCompationLevel
+		} else if o.MaxMemCompationLevel == -1 {
+			level = 0
+		}
 	}
-	return o.MaxOpenFiles
+	if level >= o.GetNumLevel() {
+		return o.GetNumLevel() - 1
+	}
+	return level
+}
+
+func (o *Options) GetNumLevel() int {
+	if o == nil || o.NumLevel <= 0 {
+		return DefaultNumLevel
+	}
+	return o.NumLevel
+}
+
+func (o *Options) GetOpenFilesCacher() Cacher {
+	if o == nil || o.OpenFilesCacher == nil {
+		return DefaultOpenFilesCacher
+	}
+	if o.OpenFilesCacher == NoCacher {
+		return nil
+	}
+	return o.OpenFilesCacher
+}
+
+func (o *Options) GetOpenFilesCacheCapacity() int {
+	if o == nil || o.OpenFilesCacheCapacity <= 0 {
+		return DefaultOpenFilesCacheCapacity
+	} else if o.OpenFilesCacheCapacity == -1 {
+		return 0
+	}
+	return o.OpenFilesCacheCapacity
 }
 
 func (o *Options) GetStrict(strict Strict) bool {
@@ -263,6 +547,20 @@ func (o *Options) GetWriteBuffer() int {
 	return o.WriteBuffer
 }
 
+func (o *Options) GetWriteL0PauseTrigger() int {
+	if o == nil || o.WriteL0PauseTrigger == 0 {
+		return DefaultWriteL0PauseTrigger
+	}
+	return o.WriteL0PauseTrigger
+}
+
+func (o *Options) GetWriteL0SlowdownTrigger() int {
+	if o == nil || o.WriteL0SlowdownTrigger == 0 {
+		return DefaultWriteL0SlowdownTrigger
+	}
+	return o.WriteL0SlowdownTrigger
+}
+
 // ReadOptions holds the optional parameters for 'read operation'. The
 // 'read operation' includes Get, Find and NewIterator.
 type ReadOptions struct {
@@ -273,8 +571,8 @@ type ReadOptions struct {
 	// The default value is false.
 	DontFillCache bool
 
-	// Strict overrides global DB strict level. Only StrictIterator and
-	// StrictBlockChecksum that does have effects here.
+	// Strict will be OR'ed with global DB 'strict level' unless StrictOverride
+	// is present. Currently only StrictReader that has effect here.
 	Strict Strict
 }
 
@@ -316,3 +614,11 @@ func (wo *WriteOptions) GetSync() bool {
 	}
 	return wo.Sync
 }
+
+func GetStrict(o *Options, ro *ReadOptions, strict Strict) bool {
+	if ro.GetStrict(StrictOverride) {
+		return ro.GetStrict(strict)
+	} else {
+		return o.GetStrict(strict) || ro.GetStrict(strict)
+	}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/options.go

@@ -7,35 +7,86 @@
 package leveldb
 
 import (
-	"github.com/syndtr/goleveldb/leveldb/cache"
 	"github.com/syndtr/goleveldb/leveldb/filter"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 )
 
-func (s *session) setOptions(o *opt.Options) {
-	s.o = &opt.Options{}
+func dupOptions(o *opt.Options) *opt.Options {
+	newo := &opt.Options{}
 	if o != nil {
-		*s.o = *o
+		*newo = *o
 	}
+	if newo.Strict == 0 {
+		newo.Strict = opt.DefaultStrict
+	}
+	return newo
+}
+
+func (s *session) setOptions(o *opt.Options) {
+	no := dupOptions(o)
 	// Alternative filters.
 	if filters := o.GetAltFilters(); len(filters) > 0 {
-		s.o.AltFilters = make([]filter.Filter, len(filters))
+		no.AltFilters = make([]filter.Filter, len(filters))
 		for i, filter := range filters {
-			s.o.AltFilters[i] = &iFilter{filter}
+			no.AltFilters[i] = &iFilter{filter}
 		}
 	}
-	// Block cache.
-	switch o.GetBlockCache() {
-	case nil:
-		s.o.BlockCache = cache.NewLRUCache(opt.DefaultBlockCacheSize)
-	case opt.NoCache:
-		s.o.BlockCache = nil
-	}
 	// Comparer.
 	s.icmp = &iComparer{o.GetComparer()}
-	s.o.Comparer = s.icmp
+	no.Comparer = s.icmp
 	// Filter.
 	if filter := o.GetFilter(); filter != nil {
-		s.o.Filter = &iFilter{filter}
+		no.Filter = &iFilter{filter}
+	}
+
+	s.o = &cachedOptions{Options: no}
+	s.o.cache()
+}
+
+type cachedOptions struct {
+	*opt.Options
+
+	compactionExpandLimit []int
+	compactionGPOverlaps  []int
+	compactionSourceLimit []int
+	compactionTableSize   []int
+	compactionTotalSize   []int64
+}
+
+func (co *cachedOptions) cache() {
+	numLevel := co.Options.GetNumLevel()
+
+	co.compactionExpandLimit = make([]int, numLevel)
+	co.compactionGPOverlaps = make([]int, numLevel)
+	co.compactionSourceLimit = make([]int, numLevel)
+	co.compactionTableSize = make([]int, numLevel)
+	co.compactionTotalSize = make([]int64, numLevel)
+
+	for level := 0; level < numLevel; level++ {
+		co.compactionExpandLimit[level] = co.Options.GetCompactionExpandLimit(level)
+		co.compactionGPOverlaps[level] = co.Options.GetCompactionGPOverlaps(level)
+		co.compactionSourceLimit[level] = co.Options.GetCompactionSourceLimit(level)
+		co.compactionTableSize[level] = co.Options.GetCompactionTableSize(level)
+		co.compactionTotalSize[level] = co.Options.GetCompactionTotalSize(level)
 	}
 }
+
+func (co *cachedOptions) GetCompactionExpandLimit(level int) int {
+	return co.compactionExpandLimit[level]
+}
+
+func (co *cachedOptions) GetCompactionGPOverlaps(level int) int {
+	return co.compactionGPOverlaps[level]
+}
+
+func (co *cachedOptions) GetCompactionSourceLimit(level int) int {
+	return co.compactionSourceLimit[level]
+}
+
+func (co *cachedOptions) GetCompactionTableSize(level int) int {
+	return co.compactionTableSize[level]
+}
+
+func (co *cachedOptions) GetCompactionTotalSize(level int) int64 {
+	return co.compactionTotalSize[level]
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go

@@ -7,12 +7,13 @@
 package leveldb
 
 import (
-	"errors"
+	"fmt"
 	"io"
 	"os"
 	"sync"
 	"sync/atomic"
 
+	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/journal"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -20,18 +21,31 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
+type ErrManifestCorrupted struct {
+	Field  string
+	Reason string
+}
+
+func (e *ErrManifestCorrupted) Error() string {
+	return fmt.Sprintf("leveldb: manifest corrupted (field '%s'): %s", e.Field, e.Reason)
+}
+
+func newErrManifestCorrupted(f storage.File, field, reason string) error {
+	return errors.NewErrCorrupted(f, &ErrManifestCorrupted{field, reason})
+}
+
 // session represent a persistent database session.
 type session struct {
 	// Need 64-bit alignment.
-	stFileNum        uint64 // current unused file number
+	stNextFileNum    uint64 // current unused file number
 	stJournalNum     uint64 // current journal file number; need external synchronization
 	stPrevJournalNum uint64 // prev journal file number; no longer used; for compatibility with older version of leveldb
-	stSeq            uint64 // last mem compacted seq; need external synchronization
+	stSeqNum         uint64 // last mem compacted seq; need external synchronization
 	stTempFileNum    uint64
 
 	stor     storage.Storage
 	storLock util.Releaser
-	o        *opt.Options
+	o        *cachedOptions
 	icmp     *iComparer
 	tops     *tOps
 
@@ -39,9 +53,9 @@ type session struct {
 	manifestWriter storage.Writer
 	manifestFile   storage.File
 
-	stCptrs   [kNumLevels]iKey // compact pointers; need external synchronization
-	stVersion *version         // current version
-	vmu       sync.Mutex
+	stCompPtrs []iKey   // compaction pointers; need external synchronization
+	stVersion  *version // current version
+	vmu        sync.Mutex
 }
 
 // Creates new initialized session instance.
@@ -54,22 +68,20 @@ func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) {
 		return
 	}
 	s = &session{
-		stor:     stor,
-		storLock: storLock,
+		stor:       stor,
+		storLock:   storLock,
+		stCompPtrs: make([]iKey, o.GetNumLevel()),
 	}
 	s.setOptions(o)
-	s.tops = newTableOps(s, s.o.GetMaxOpenFiles())
-	s.setVersion(&version{s: s})
-	s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock D·DeletedEntry L·Level Q·SeqNum T·TimeElapsed")
+	s.tops = newTableOps(s)
+	s.setVersion(newVersion(s))
+	s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed")
 	return
 }
 
 // Close session.
 func (s *session) close() {
 	s.tops.close()
-	if bc := s.o.GetBlockCache(); bc != nil {
-		bc.Purge(nil)
-	}
 	if s.manifest != nil {
 		s.manifest.Close()
 	}
@@ -100,26 +112,26 @@ func (s *session) recover() (err error) {
 			// Don't return os.ErrNotExist if the underlying storage contains
 			// other files that belong to LevelDB. So the DB won't get trashed.
 			if files, _ := s.stor.GetFiles(storage.TypeAll); len(files) > 0 {
-				err = ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest file missing")}
+				err = &errors.ErrCorrupted{File: &storage.FileInfo{Type: storage.TypeManifest}, Err: &errors.ErrMissingFiles{}}
 			}
 		}
 	}()
 
-	file, err := s.stor.GetManifest()
+	m, err := s.stor.GetManifest()
 	if err != nil {
 		return
 	}
 
-	reader, err := file.Open()
+	reader, err := m.Open()
 	if err != nil {
 		return
 	}
 	defer reader.Close()
 	strict := s.o.GetStrict(opt.StrictManifest)
-	jr := journal.NewReader(reader, dropper{s, file}, strict, true)
+	jr := journal.NewReader(reader, dropper{s, m}, strict, true)
 
-	staging := s.version_NB().newStaging()
-	rec := &sessionRecord{}
+	staging := s.stVersion.newStaging()
+	rec := &sessionRecord{numLevel: s.o.GetNumLevel()}
 	for {
 		var r io.Reader
 		r, err = jr.Next()
@@ -128,51 +140,57 @@ func (s *session) recover() (err error) {
 				err = nil
 				break
 			}
-			return
+			return errors.SetFile(err, m)
 		}
 
 		err = rec.decode(r)
 		if err == nil {
 			// save compact pointers
-			for _, r := range rec.compactionPointers {
-				s.stCptrs[r.level] = iKey(r.ikey)
+			for _, r := range rec.compPtrs {
+				s.stCompPtrs[r.level] = iKey(r.ikey)
 			}
 			// commit record to version staging
 			staging.commit(rec)
-		} else if strict {
-			return ErrCorrupted{Type: CorruptedManifest, Err: err}
 		} else {
-			s.logf("manifest error: %v (skipped)", err)
+			err = errors.SetFile(err, m)
+			if strict || !errors.IsCorrupted(err) {
+				return
+			} else {
+				s.logf("manifest error: %v (skipped)", errors.SetFile(err, m))
+			}
 		}
-		rec.resetCompactionPointers()
+		rec.resetCompPtrs()
 		rec.resetAddedTables()
 		rec.resetDeletedTables()
 	}
 
 	switch {
 	case !rec.has(recComparer):
-		return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing comparer name")}
+		return newErrManifestCorrupted(m, "comparer", "missing")
 	case rec.comparer != s.icmp.uName():
-		return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: comparer mismatch, " + "want '" + s.icmp.uName() + "', " + "got '" + rec.comparer + "'")}
-	case !rec.has(recNextNum):
-		return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing next file number")}
+		return newErrManifestCorrupted(m, "comparer", fmt.Sprintf("mismatch: want '%s', got '%s'", s.icmp.uName(), rec.comparer))
+	case !rec.has(recNextFileNum):
+		return newErrManifestCorrupted(m, "next-file-num", "missing")
 	case !rec.has(recJournalNum):
-		return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing journal file number")}
-	case !rec.has(recSeq):
-		return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing seq number")}
+		return newErrManifestCorrupted(m, "journal-file-num", "missing")
+	case !rec.has(recSeqNum):
+		return newErrManifestCorrupted(m, "seq-num", "missing")
 	}
 
-	s.manifestFile = file
+	s.manifestFile = m
 	s.setVersion(staging.finish())
-	s.setFileNum(rec.nextNum)
+	s.setNextFileNum(rec.nextFileNum)
 	s.recordCommited(rec)
 	return nil
 }
 
 // Commit session; need external synchronization.
 func (s *session) commit(r *sessionRecord) (err error) {
+	v := s.version()
+	defer v.release()
+
 	// spawn new version based on current version
-	nv := s.version_NB().spawn(r)
+	nv := v.spawn(r)
 
 	if s.manifest == nil {
 		// manifest journal writer not yet created, create one
@@ -191,13 +209,13 @@ func (s *session) commit(r *sessionRecord) (err error) {
 
 // Pick a compaction based on current state; need external synchronization.
 func (s *session) pickCompaction() *compaction {
-	v := s.version_NB()
+	v := s.version()
 
 	var level int
 	var t0 tFiles
 	if v.cScore >= 1 {
 		level = v.cLevel
-		cptr := s.stCptrs[level]
+		cptr := s.stCompPtrs[level]
 		tables := v.tables[level]
 		for _, t := range tables {
 			if cptr == nil || s.icmp.Compare(t.imax, cptr) > 0 {
@@ -214,27 +232,21 @@ func (s *session) pickCompaction() *compaction {
 			level = ts.level
 			t0 = append(t0, ts.table)
 		} else {
+			v.release()
 			return nil
 		}
 	}
 
-	c := &compaction{s: s, v: v, level: level}
-	if level == 0 {
-		imin, imax := t0.getRange(s.icmp)
-		t0 = v.tables[0].getOverlaps(t0[:0], s.icmp, imin.ukey(), imax.ukey(), true)
-	}
-
-	c.tables[0] = t0
-	c.expand()
-	return c
+	return newCompaction(s, v, level, t0)
 }
 
 // Create compaction from given level and range; need external synchronization.
 func (s *session) getCompactionRange(level int, umin, umax []byte) *compaction {
-	v := s.version_NB()
+	v := s.version()
 
 	t0 := v.tables[level].getOverlaps(nil, s.icmp, umin, umax, level == 0)
 	if len(t0) == 0 {
+		v.release()
 		return nil
 	}
 
@@ -243,7 +255,7 @@ func (s *session) getCompactionRange(level int, umin, umax []byte) *compaction {
 	// and we must not pick one file and drop another older file if the
 	// two files overlap.
 	if level > 0 {
-		limit := uint64(kMaxTableSize)
+		limit := uint64(v.s.o.GetCompactionSourceLimit(level))
 		total := uint64(0)
 		for i, t := range t0 {
 			total += t.size
@@ -255,9 +267,20 @@ func (s *session) getCompactionRange(level int, umin, umax []byte) *compaction {
 		}
 	}
 
-	c := &compaction{s: s, v: v, level: level}
-	c.tables[0] = t0
+	return newCompaction(s, v, level, t0)
+}
+
+func newCompaction(s *session, v *version, level int, t0 tFiles) *compaction {
+	c := &compaction{
+		s:             s,
+		v:             v,
+		level:         level,
+		tables:        [2]tFiles{t0, nil},
+		maxGPOverlaps: uint64(s.o.GetCompactionGPOverlaps(level)),
+		tPtrs:         make([]int, s.o.GetNumLevel()),
+	}
 	c.expand()
+	c.save()
 	return c
 }
 
@@ -266,25 +289,57 @@ type compaction struct {
 	s *session
 	v *version
 
-	level  int
-	tables [2]tFiles
+	level         int
+	tables        [2]tFiles
+	maxGPOverlaps uint64
 
-	gp              tFiles
-	gpidx           int
-	seenKey         bool
-	overlappedBytes uint64
-	imin, imax      iKey
+	gp                tFiles
+	gpi               int
+	seenKey           bool
+	gpOverlappedBytes uint64
+	imin, imax        iKey
+	tPtrs             []int
+	released          bool
 
-	tPtrs [kNumLevels]int
+	snapGPI               int
+	snapSeenKey           bool
+	snapGPOverlappedBytes uint64
+	snapTPtrs             []int
+}
+
+func (c *compaction) save() {
+	c.snapGPI = c.gpi
+	c.snapSeenKey = c.seenKey
+	c.snapGPOverlappedBytes = c.gpOverlappedBytes
+	c.snapTPtrs = append(c.snapTPtrs[:0], c.tPtrs...)
+}
+
+func (c *compaction) restore() {
+	c.gpi = c.snapGPI
+	c.seenKey = c.snapSeenKey
+	c.gpOverlappedBytes = c.snapGPOverlappedBytes
+	c.tPtrs = append(c.tPtrs[:0], c.snapTPtrs...)
+}
+
+func (c *compaction) release() {
+	if !c.released {
+		c.released = true
+		c.v.release()
+	}
 }
 
 // Expand compacted tables; need external synchronization.
 func (c *compaction) expand() {
-	level := c.level
-	vt0, vt1 := c.v.tables[level], c.v.tables[level+1]
+	limit := uint64(c.s.o.GetCompactionExpandLimit(c.level))
+	vt0, vt1 := c.v.tables[c.level], c.v.tables[c.level+1]
 
 	t0, t1 := c.tables[0], c.tables[1]
 	imin, imax := t0.getRange(c.s.icmp)
+	// We expand t0 here just incase ukey hop across tables.
+	t0 = vt0.getOverlaps(t0, c.s.icmp, imin.ukey(), imax.ukey(), c.level == 0)
+	if len(t0) != len(c.tables[0]) {
+		imin, imax = t0.getRange(c.s.icmp)
+	}
 	t1 = vt1.getOverlaps(t1, c.s.icmp, imin.ukey(), imax.ukey(), false)
 	// Get entire range covered by compaction.
 	amin, amax := append(t0, t1...).getRange(c.s.icmp)
@@ -292,13 +347,13 @@ func (c *compaction) expand() {
 	// See if we can grow the number of inputs in "level" without
 	// changing the number of "level+1" files we pick up.
 	if len(t1) > 0 {
-		exp0 := vt0.getOverlaps(nil, c.s.icmp, amin.ukey(), amax.ukey(), level == 0)
-		if len(exp0) > len(t0) && t1.size()+exp0.size() < kExpCompactionMaxBytes {
+		exp0 := vt0.getOverlaps(nil, c.s.icmp, amin.ukey(), amax.ukey(), c.level == 0)
+		if len(exp0) > len(t0) && t1.size()+exp0.size() < limit {
 			xmin, xmax := exp0.getRange(c.s.icmp)
 			exp1 := vt1.getOverlaps(nil, c.s.icmp, xmin.ukey(), xmax.ukey(), false)
 			if len(exp1) == len(t1) {
 				c.s.logf("table@compaction expanding L%d+L%d (F·%d S·%s)+(F·%d S·%s) -> (F·%d S·%s)+(F·%d S·%s)",
-					level, level+1, len(t0), shortenb(int(t0.size())), len(t1), shortenb(int(t1.size())),
+					c.level, c.level+1, len(t0), shortenb(int(t0.size())), len(t1), shortenb(int(t1.size())),
 					len(exp0), shortenb(int(exp0.size())), len(exp1), shortenb(int(exp1.size())))
 				imin, imax = xmin, xmax
 				t0, t1 = exp0, exp1
@@ -309,8 +364,8 @@ func (c *compaction) expand() {
 
 	// Compute the set of grandparent files that overlap this compaction
 	// (parent == level+1; grandparent == level+2)
-	if level+2 < kNumLevels {
-		c.gp = c.v.tables[level+2].getOverlaps(c.gp, c.s.icmp, amin.ukey(), amax.ukey(), false)
+	if c.level+2 < c.s.o.GetNumLevel() {
+		c.gp = c.v.tables[c.level+2].getOverlaps(c.gp, c.s.icmp, amin.ukey(), amax.ukey(), false)
 	}
 
 	c.tables[0], c.tables[1] = t0, t1
@@ -319,7 +374,7 @@ func (c *compaction) expand() {
 
 // Check whether compaction is trivial.
 func (c *compaction) trivial() bool {
-	return len(c.tables[0]) == 1 && len(c.tables[1]) == 0 && c.gp.size() <= kMaxGrandParentOverlapBytes
+	return len(c.tables[0]) == 1 && len(c.tables[1]) == 0 && c.gp.size() <= c.maxGPOverlaps
 }
 
 func (c *compaction) baseLevelForKey(ukey []byte) bool {
@@ -341,20 +396,20 @@ func (c *compaction) baseLevelForKey(ukey []byte) bool {
 }
 
 func (c *compaction) shouldStopBefore(ikey iKey) bool {
-	for ; c.gpidx < len(c.gp); c.gpidx++ {
-		gp := c.gp[c.gpidx]
+	for ; c.gpi < len(c.gp); c.gpi++ {
+		gp := c.gp[c.gpi]
 		if c.s.icmp.Compare(ikey, gp.imax) <= 0 {
 			break
 		}
 		if c.seenKey {
-			c.overlappedBytes += gp.size
+			c.gpOverlappedBytes += gp.size
 		}
 	}
 	c.seenKey = true
 
-	if c.overlappedBytes > kMaxGrandParentOverlapBytes {
+	if c.gpOverlappedBytes > c.maxGPOverlaps {
 		// Too much overlap for current output; start new output.
-		c.overlappedBytes = 0
+		c.gpOverlappedBytes = 0
 		return true
 	}
 	return false
@@ -373,8 +428,12 @@ func (c *compaction) newIterator() iterator.Iterator {
 	// Options.
 	ro := &opt.ReadOptions{
 		DontFillCache: true,
+		Strict:        opt.StrictOverride,
+	}
+	strict := c.s.o.GetStrict(opt.StrictCompaction)
+	if strict {
+		ro.Strict |= opt.StrictReader
 	}
-	strict := c.s.o.GetStrict(opt.StrictIterator)
 
 	for i, tables := range c.tables {
 		if len(tables) == 0 {
@@ -387,10 +446,10 @@ func (c *compaction) newIterator() iterator.Iterator {
 				its = append(its, c.s.tops.newIterator(t, nil, ro))
 			}
 		} else {
-			it := iterator.NewIndexedIterator(tables.newIndexIterator(c.s.tops, c.s.icmp, nil, ro), strict, true)
+			it := iterator.NewIndexedIterator(tables.newIndexIterator(c.s.tops, c.s.icmp, nil, ro), strict)
 			its = append(its, it)
 		}
 	}
 
-	return iterator.NewMergedIterator(its, c.s.icmp, true)
+	return iterator.NewMergedIterator(its, c.s.icmp, strict)
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record.go

@@ -9,11 +9,11 @@ package leveldb
 import (
 	"bufio"
 	"encoding/binary"
-	"errors"
 	"io"
-)
+	"strings"
 
-var errCorruptManifest = errors.New("leveldb: corrupt manifest")
+	"github.com/syndtr/goleveldb/leveldb/errors"
+)
 
 type byteReader interface {
 	io.Reader
@@ -22,13 +22,13 @@ type byteReader interface {
 
 // These numbers are written to disk and should not be changed.
 const (
-	recComparer          = 1
-	recJournalNum        = 2
-	recNextNum           = 3
-	recSeq               = 4
-	recCompactionPointer = 5
-	recDeletedTable      = 6
-	recNewTable          = 7
+	recComparer    = 1
+	recJournalNum  = 2
+	recNextFileNum = 3
+	recSeqNum      = 4
+	recCompPtr     = 5
+	recDelTable    = 6
+	recAddTable    = 7
 	// 8 was used for large value refs
 	recPrevJournalNum = 9
 )
@@ -38,7 +38,7 @@ type cpRecord struct {
 	ikey  iKey
 }
 
-type ntRecord struct {
+type atRecord struct {
 	level int
 	num   uint64
 	size  uint64
@@ -46,27 +46,26 @@ type ntRecord struct {
 	imax  iKey
 }
 
-func (r ntRecord) makeFile(s *session) *tFile {
-	return newTableFile(s.getTableFile(r.num), r.size, r.imin, r.imax)
-}
-
 type dtRecord struct {
 	level int
 	num   uint64
 }
 
 type sessionRecord struct {
-	hasRec             int
-	comparer           string
-	journalNum         uint64
-	prevJournalNum     uint64
-	nextNum            uint64
-	seq                uint64
-	compactionPointers []cpRecord
-	addedTables        []ntRecord
-	deletedTables      []dtRecord
-	scratch            [binary.MaxVarintLen64]byte
-	err                error
+	numLevel int
+
+	hasRec         int
+	comparer       string
+	journalNum     uint64
+	prevJournalNum uint64
+	nextFileNum    uint64
+	seqNum         uint64
+	compPtrs       []cpRecord
+	addedTables    []atRecord
+	deletedTables  []dtRecord
+
+	scratch [binary.MaxVarintLen64]byte
+	err     error
 }
 
 func (p *sessionRecord) has(rec int) bool {
@@ -88,29 +87,29 @@ func (p *sessionRecord) setPrevJournalNum(num uint64) {
 	p.prevJournalNum = num
 }
 
-func (p *sessionRecord) setNextNum(num uint64) {
-	p.hasRec |= 1 << recNextNum
-	p.nextNum = num
+func (p *sessionRecord) setNextFileNum(num uint64) {
+	p.hasRec |= 1 << recNextFileNum
+	p.nextFileNum = num
 }
 
-func (p *sessionRecord) setSeq(seq uint64) {
-	p.hasRec |= 1 << recSeq
-	p.seq = seq
+func (p *sessionRecord) setSeqNum(num uint64) {
+	p.hasRec |= 1 << recSeqNum
+	p.seqNum = num
 }
 
-func (p *sessionRecord) addCompactionPointer(level int, ikey iKey) {
-	p.hasRec |= 1 << recCompactionPointer
-	p.compactionPointers = append(p.compactionPointers, cpRecord{level, ikey})
+func (p *sessionRecord) addCompPtr(level int, ikey iKey) {
+	p.hasRec |= 1 << recCompPtr
+	p.compPtrs = append(p.compPtrs, cpRecord{level, ikey})
 }
 
-func (p *sessionRecord) resetCompactionPointers() {
-	p.hasRec &= ^(1 << recCompactionPointer)
-	p.compactionPointers = p.compactionPointers[:0]
+func (p *sessionRecord) resetCompPtrs() {
+	p.hasRec &= ^(1 << recCompPtr)
+	p.compPtrs = p.compPtrs[:0]
 }
 
 func (p *sessionRecord) addTable(level int, num, size uint64, imin, imax iKey) {
-	p.hasRec |= 1 << recNewTable
-	p.addedTables = append(p.addedTables, ntRecord{level, num, size, imin, imax})
+	p.hasRec |= 1 << recAddTable
+	p.addedTables = append(p.addedTables, atRecord{level, num, size, imin, imax})
 }
 
 func (p *sessionRecord) addTableFile(level int, t *tFile) {
@@ -118,17 +117,17 @@ func (p *sessionRecord) addTableFile(level int, t *tFile) {
 }
 
 func (p *sessionRecord) resetAddedTables() {
-	p.hasRec &= ^(1 << recNewTable)
+	p.hasRec &= ^(1 << recAddTable)
 	p.addedTables = p.addedTables[:0]
 }
 
-func (p *sessionRecord) deleteTable(level int, num uint64) {
-	p.hasRec |= 1 << recDeletedTable
+func (p *sessionRecord) delTable(level int, num uint64) {
+	p.hasRec |= 1 << recDelTable
 	p.deletedTables = append(p.deletedTables, dtRecord{level, num})
 }
 
 func (p *sessionRecord) resetDeletedTables() {
-	p.hasRec &= ^(1 << recDeletedTable)
+	p.hasRec &= ^(1 << recDelTable)
 	p.deletedTables = p.deletedTables[:0]
 }
 
@@ -161,26 +160,26 @@ func (p *sessionRecord) encode(w io.Writer) error {
 		p.putUvarint(w, recJournalNum)
 		p.putUvarint(w, p.journalNum)
 	}
-	if p.has(recNextNum) {
-		p.putUvarint(w, recNextNum)
-		p.putUvarint(w, p.nextNum)
+	if p.has(recNextFileNum) {
+		p.putUvarint(w, recNextFileNum)
+		p.putUvarint(w, p.nextFileNum)
 	}
-	if p.has(recSeq) {
-		p.putUvarint(w, recSeq)
-		p.putUvarint(w, p.seq)
+	if p.has(recSeqNum) {
+		p.putUvarint(w, recSeqNum)
+		p.putUvarint(w, p.seqNum)
 	}
-	for _, r := range p.compactionPointers {
-		p.putUvarint(w, recCompactionPointer)
+	for _, r := range p.compPtrs {
+		p.putUvarint(w, recCompPtr)
 		p.putUvarint(w, uint64(r.level))
 		p.putBytes(w, r.ikey)
 	}
 	for _, r := range p.deletedTables {
-		p.putUvarint(w, recDeletedTable)
+		p.putUvarint(w, recDelTable)
 		p.putUvarint(w, uint64(r.level))
 		p.putUvarint(w, r.num)
 	}
 	for _, r := range p.addedTables {
-		p.putUvarint(w, recNewTable)
+		p.putUvarint(w, recAddTable)
 		p.putUvarint(w, uint64(r.level))
 		p.putUvarint(w, r.num)
 		p.putUvarint(w, r.size)
@@ -190,14 +189,16 @@ func (p *sessionRecord) encode(w io.Writer) error {
 	return p.err
 }
 
-func (p *sessionRecord) readUvarint(r io.ByteReader) uint64 {
+func (p *sessionRecord) readUvarintMayEOF(field string, r io.ByteReader, mayEOF bool) uint64 {
 	if p.err != nil {
 		return 0
 	}
 	x, err := binary.ReadUvarint(r)
 	if err != nil {
-		if err == io.EOF {
-			p.err = errCorruptManifest
+		if err == io.ErrUnexpectedEOF || (mayEOF == false && err == io.EOF) {
+			p.err = errors.NewErrCorrupted(nil, &ErrManifestCorrupted{field, "short read"})
+		} else if strings.HasPrefix(err.Error(), "binary:") {
+			p.err = errors.NewErrCorrupted(nil, &ErrManifestCorrupted{field, err.Error()})
 		} else {
 			p.err = err
 		}
@@ -206,35 +207,39 @@ func (p *sessionRecord) readUvarint(r io.ByteReader) uint64 {
 	return x
 }
 
-func (p *sessionRecord) readBytes(r byteReader) []byte {
+func (p *sessionRecord) readUvarint(field string, r io.ByteReader) uint64 {
+	return p.readUvarintMayEOF(field, r, false)
+}
+
+func (p *sessionRecord) readBytes(field string, r byteReader) []byte {
 	if p.err != nil {
 		return nil
 	}
-	n := p.readUvarint(r)
+	n := p.readUvarint(field, r)
 	if p.err != nil {
 		return nil
 	}
 	x := make([]byte, n)
 	_, p.err = io.ReadFull(r, x)
 	if p.err != nil {
-		if p.err == io.EOF {
-			p.err = errCorruptManifest
+		if p.err == io.ErrUnexpectedEOF {
+			p.err = errors.NewErrCorrupted(nil, &ErrManifestCorrupted{field, "short read"})
 		}
 		return nil
 	}
 	return x
 }
 
-func (p *sessionRecord) readLevel(r io.ByteReader) int {
+func (p *sessionRecord) readLevel(field string, r io.ByteReader) int {
 	if p.err != nil {
 		return 0
 	}
-	x := p.readUvarint(r)
+	x := p.readUvarint(field, r)
 	if p.err != nil {
 		return 0
 	}
-	if x >= kNumLevels {
-		p.err = errCorruptManifest
+	if x >= uint64(p.numLevel) {
+		p.err = errors.NewErrCorrupted(nil, &ErrManifestCorrupted{field, "invalid level number"})
 		return 0
 	}
 	return int(x)
@@ -247,59 +252,59 @@ func (p *sessionRecord) decode(r io.Reader) error {
 	}
 	p.err = nil
 	for p.err == nil {
-		rec, err := binary.ReadUvarint(br)
-		if err != nil {
-			if err == io.EOF {
-				err = nil
+		rec := p.readUvarintMayEOF("field-header", br, true)
+		if p.err != nil {
+			if p.err == io.EOF {
+				return nil
 			}
-			return err
+			return p.err
 		}
 		switch rec {
 		case recComparer:
-			x := p.readBytes(br)
+			x := p.readBytes("comparer", br)
 			if p.err == nil {
 				p.setComparer(string(x))
 			}
 		case recJournalNum:
-			x := p.readUvarint(br)
+			x := p.readUvarint("journal-num", br)
 			if p.err == nil {
 				p.setJournalNum(x)
 			}
 		case recPrevJournalNum:
-			x := p.readUvarint(br)
+			x := p.readUvarint("prev-journal-num", br)
 			if p.err == nil {
 				p.setPrevJournalNum(x)
 			}
-		case recNextNum:
-			x := p.readUvarint(br)
+		case recNextFileNum:
+			x := p.readUvarint("next-file-num", br)
 			if p.err == nil {
-				p.setNextNum(x)
+				p.setNextFileNum(x)
 			}
-		case recSeq:
-			x := p.readUvarint(br)
+		case recSeqNum:
+			x := p.readUvarint("seq-num", br)
 			if p.err == nil {
-				p.setSeq(x)
+				p.setSeqNum(x)
 			}
-		case recCompactionPointer:
-			level := p.readLevel(br)
-			ikey := p.readBytes(br)
+		case recCompPtr:
+			level := p.readLevel("comp-ptr.level", br)
+			ikey := p.readBytes("comp-ptr.ikey", br)
 			if p.err == nil {
-				p.addCompactionPointer(level, iKey(ikey))
+				p.addCompPtr(level, iKey(ikey))
 			}
-		case recNewTable:
-			level := p.readLevel(br)
-			num := p.readUvarint(br)
-			size := p.readUvarint(br)
-			imin := p.readBytes(br)
-			imax := p.readBytes(br)
+		case recAddTable:
+			level := p.readLevel("add-table.level", br)
+			num := p.readUvarint("add-table.num", br)
+			size := p.readUvarint("add-table.size", br)
+			imin := p.readBytes("add-table.imin", br)
+			imax := p.readBytes("add-table.imax", br)
 			if p.err == nil {
 				p.addTable(level, num, size, imin, imax)
 			}
-		case recDeletedTable:
-			level := p.readLevel(br)
-			num := p.readUvarint(br)
+		case recDelTable:
+			level := p.readLevel("del-table.level", br)
+			num := p.readUvarint("del-table.num", br)
 			if p.err == nil {
-				p.deleteTable(level, num)
+				p.delTable(level, num)
 			}
 		}
 	}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record_test.go

@@ -9,6 +9,8 @@ package leveldb
 import (
 	"bytes"
 	"testing"
+
+	"github.com/syndtr/goleveldb/leveldb/opt"
 )
 
 func decodeEncode(v *sessionRecord) (res bool, err error) {
@@ -17,7 +19,7 @@ func decodeEncode(v *sessionRecord) (res bool, err error) {
 	if err != nil {
 		return
 	}
-	v2 := new(sessionRecord)
+	v2 := &sessionRecord{numLevel: opt.DefaultNumLevel}
 	err = v.decode(b)
 	if err != nil {
 		return
@@ -32,7 +34,7 @@ func decodeEncode(v *sessionRecord) (res bool, err error) {
 
 func TestSessionRecord_EncodeDecode(t *testing.T) {
 	big := uint64(1) << 50
-	v := new(sessionRecord)
+	v := &sessionRecord{numLevel: opt.DefaultNumLevel}
 	i := uint64(0)
 	test := func() {
 		res, err := decodeEncode(v)
@@ -47,16 +49,16 @@ func TestSessionRecord_EncodeDecode(t *testing.T) {
 	for ; i < 4; i++ {
 		test()
 		v.addTable(3, big+300+i, big+400+i,
-			newIKey([]byte("foo"), big+500+1, tVal),
-			newIKey([]byte("zoo"), big+600+1, tDel))
-		v.deleteTable(4, big+700+i)
-		v.addCompactionPointer(int(i), newIKey([]byte("x"), big+900+1, tVal))
+			newIkey([]byte("foo"), big+500+1, ktVal),
+			newIkey([]byte("zoo"), big+600+1, ktDel))
+		v.delTable(4, big+700+i)
+		v.addCompPtr(int(i), newIkey([]byte("x"), big+900+1, ktVal))
 	}
 
 	v.setComparer("foo")
 	v.setJournalNum(big + 100)
 	v.setPrevJournalNum(big + 99)
-	v.setNextNum(big + 200)
-	v.setSeq(big + 1000)
+	v.setNextFileNum(big + 200)
+	v.setSeqNum(big + 1000)
 	test()
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go

@@ -22,7 +22,7 @@ type dropper struct {
 }
 
 func (d dropper) Drop(err error) {
-	if e, ok := err.(journal.DroppedError); ok {
+	if e, ok := err.(*journal.ErrCorrupted); ok {
 		d.s.logf("journal@drop %s-%d S·%s %q", d.file.Type(), d.file.Num(), shortenb(e.Size), e.Reason)
 	} else {
 		d.s.logf("journal@drop %s-%d %q", d.file.Type(), d.file.Num(), err)
@@ -51,9 +51,14 @@ func (s *session) newTemp() storage.File {
 	return s.stor.GetFile(num, storage.TypeTemp)
 }
 
+func (s *session) tableFileFromRecord(r atRecord) *tFile {
+	return newTableFile(s.getTableFile(r.num), r.size, r.imin, r.imax)
+}
+
 // Session state.
 
-// Get current version.
+// Get current version. This will incr version ref, must call
+// version.release (exactly once) after use.
 func (s *session) version() *version {
 	s.vmu.Lock()
 	defer s.vmu.Unlock()
@@ -61,61 +66,56 @@ func (s *session) version() *version {
 	return s.stVersion
 }
 
-// Get current version; no barrier.
-func (s *session) version_NB() *version {
-	return s.stVersion
-}
-
 // Set current version to v.
 func (s *session) setVersion(v *version) {
 	s.vmu.Lock()
-	v.ref = 1
+	v.ref = 1 // Holds by session.
 	if old := s.stVersion; old != nil {
-		v.ref++
+		v.ref++ // Holds by old version.
 		old.next = v
-		old.release_NB()
+		old.releaseNB()
 	}
 	s.stVersion = v
 	s.vmu.Unlock()
 }
 
 // Get current unused file number.
-func (s *session) fileNum() uint64 {
-	return atomic.LoadUint64(&s.stFileNum)
+func (s *session) nextFileNum() uint64 {
+	return atomic.LoadUint64(&s.stNextFileNum)
 }
 
-// Get current unused file number to num.
-func (s *session) setFileNum(num uint64) {
-	atomic.StoreUint64(&s.stFileNum, num)
+// Set current unused file number to num.
+func (s *session) setNextFileNum(num uint64) {
+	atomic.StoreUint64(&s.stNextFileNum, num)
 }
 
 // Mark file number as used.
 func (s *session) markFileNum(num uint64) {
-	num += 1
+	nextFileNum := num + 1
 	for {
-		old, x := s.stFileNum, num
+		old, x := s.stNextFileNum, nextFileNum
 		if old > x {
 			x = old
 		}
-		if atomic.CompareAndSwapUint64(&s.stFileNum, old, x) {
+		if atomic.CompareAndSwapUint64(&s.stNextFileNum, old, x) {
 			break
 		}
 	}
 }
 
 // Allocate a file number.
-func (s *session) allocFileNum() (num uint64) {
-	return atomic.AddUint64(&s.stFileNum, 1) - 1
+func (s *session) allocFileNum() uint64 {
+	return atomic.AddUint64(&s.stNextFileNum, 1) - 1
 }
 
 // Reuse given file number.
 func (s *session) reuseFileNum(num uint64) {
 	for {
-		old, x := s.stFileNum, num
+		old, x := s.stNextFileNum, num
 		if old != x+1 {
 			x = old
 		}
-		if atomic.CompareAndSwapUint64(&s.stFileNum, old, x) {
+		if atomic.CompareAndSwapUint64(&s.stNextFileNum, old, x) {
 			break
 		}
 	}
@@ -126,20 +126,20 @@ func (s *session) reuseFileNum(num uint64) {
 // Fill given session record obj with current states; need external
 // synchronization.
 func (s *session) fillRecord(r *sessionRecord, snapshot bool) {
-	r.setNextNum(s.fileNum())
+	r.setNextFileNum(s.nextFileNum())
 
 	if snapshot {
 		if !r.has(recJournalNum) {
 			r.setJournalNum(s.stJournalNum)
 		}
 
-		if !r.has(recSeq) {
-			r.setSeq(s.stSeq)
+		if !r.has(recSeqNum) {
+			r.setSeqNum(s.stSeqNum)
 		}
 
-		for level, ik := range s.stCptrs {
+		for level, ik := range s.stCompPtrs {
 			if ik != nil {
-				r.addCompactionPointer(level, ik)
+				r.addCompPtr(level, ik)
 			}
 		}
 
@@ -147,7 +147,7 @@ func (s *session) fillRecord(r *sessionRecord, snapshot bool) {
 	}
 }
 
-// Mark if record has been commited, this will update session state;
+// Mark if record has been committed, this will update session state;
 // need external synchronization.
 func (s *session) recordCommited(r *sessionRecord) {
 	if r.has(recJournalNum) {
@@ -158,12 +158,12 @@ func (s *session) recordCommited(r *sessionRecord) {
 		s.stPrevJournalNum = r.prevJournalNum
 	}
 
-	if r.has(recSeq) {
-		s.stSeq = r.seq
+	if r.has(recSeqNum) {
+		s.stSeqNum = r.seqNum
 	}
 
-	for _, p := range r.compactionPointers {
-		s.stCptrs[p.level] = iKey(p.ikey)
+	for _, p := range r.compPtrs {
+		s.stCompPtrs[p.level] = iKey(p.ikey)
 	}
 }
 
@@ -178,10 +178,11 @@ func (s *session) newManifest(rec *sessionRecord, v *version) (err error) {
 	jw := journal.NewWriter(writer)
 
 	if v == nil {
-		v = s.version_NB()
+		v = s.version()
+		defer v.release()
 	}
 	if rec == nil {
-		rec = new(sessionRecord)
+		rec = &sessionRecord{numLevel: s.o.GetNumLevel()}
 	}
 	s.fillRecord(rec, true)
 	v.fillRecord(rec)

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go

@@ -221,7 +221,7 @@ func (fs *fileStorage) GetManifest() (f File, err error) {
 					fs.log(fmt.Sprintf("skipping %s: invalid file name", fn))
 					continue
 				}
-				if _, e1 := strconv.ParseUint(fn[7:], 10, 0); e1 != nil {
+				if _, e1 := strconv.ParseUint(fn[8:], 10, 0); e1 != nil {
 					fs.log(fmt.Sprintf("skipping %s: invalid file num: %v", fn, e1))
 					continue
 				}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/storage.go

@@ -125,3 +125,33 @@ type Storage interface {
 	// Other methods should not be called after the storage has been closed.
 	Close() error
 }
+
+// FileInfo wraps basic file info.
+type FileInfo struct {
+	Type FileType
+	Num  uint64
+}
+
+func (fi FileInfo) String() string {
+	switch fi.Type {
+	case TypeManifest:
+		return fmt.Sprintf("MANIFEST-%06d", fi.Num)
+	case TypeJournal:
+		return fmt.Sprintf("%06d.log", fi.Num)
+	case TypeTable:
+		return fmt.Sprintf("%06d.ldb", fi.Num)
+	case TypeTemp:
+		return fmt.Sprintf("%06d.tmp", fi.Num)
+	default:
+		return fmt.Sprintf("%#x-%d", fi.Type, fi.Num)
+	}
+}
+
+// NewFileInfo creates new FileInfo from the given File. It will returns nil
+// if File is nil.
+func NewFileInfo(f File) *FileInfo {
+	if f == nil {
+		return nil
+	}
+	return &FileInfo{f.Type(), f.Num()}
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage_test.go

@@ -11,6 +11,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"sync"
@@ -28,11 +29,25 @@ var (
 )
 
 var (
-	tsFSEnv  = os.Getenv("GOLEVELDB_USEFS")
-	tsKeepFS = tsFSEnv == "2"
-	tsFS     = tsKeepFS || tsFSEnv == "" || tsFSEnv == "1"
-	tsMU     = &sync.Mutex{}
-	tsNum    = 0
+	tsFSEnv   = os.Getenv("GOLEVELDB_USEFS")
+	tsTempdir = os.Getenv("GOLEVELDB_TEMPDIR")
+	tsKeepFS  = tsFSEnv == "2"
+	tsFS      = tsKeepFS || tsFSEnv == "" || tsFSEnv == "1"
+	tsMU      = &sync.Mutex{}
+	tsNum     = 0
+)
+
+type tsOp uint
+
+const (
+	tsOpOpen tsOp = iota
+	tsOpCreate
+	tsOpRead
+	tsOpReadAt
+	tsOpWrite
+	tsOpSync
+
+	tsOpNum
 )
 
 type tsLock struct {
@@ -53,6 +68,9 @@ type tsReader struct {
 func (tr tsReader) Read(b []byte) (n int, err error) {
 	ts := tr.tf.ts
 	ts.countRead(tr.tf.Type())
+	if tr.tf.shouldErrLocked(tsOpRead) {
+		return 0, errors.New("leveldb.testStorage: emulated read error")
+	}
 	n, err = tr.Reader.Read(b)
 	if err != nil && err != io.EOF {
 		ts.t.Errorf("E: read error, num=%d type=%v n=%d: %v", tr.tf.Num(), tr.tf.Type(), n, err)
@@ -63,6 +81,9 @@ func (tr tsReader) Read(b []byte) (n int, err error) {
 func (tr tsReader) ReadAt(b []byte, off int64) (n int, err error) {
 	ts := tr.tf.ts
 	ts.countRead(tr.tf.Type())
+	if tr.tf.shouldErrLocked(tsOpReadAt) {
+		return 0, errors.New("leveldb.testStorage: emulated readAt error")
+	}
 	n, err = tr.Reader.ReadAt(b, off)
 	if err != nil && err != io.EOF {
 		ts.t.Errorf("E: readAt error, num=%d type=%v off=%d n=%d: %v", tr.tf.Num(), tr.tf.Type(), off, n, err)
@@ -82,15 +103,12 @@ type tsWriter struct {
 }
 
 func (tw tsWriter) Write(b []byte) (n int, err error) {
-	ts := tw.tf.ts
-	ts.mu.Lock()
-	defer ts.mu.Unlock()
-	if ts.emuWriteErr&tw.tf.Type() != 0 {
+	if tw.tf.shouldErrLocked(tsOpWrite) {
 		return 0, errors.New("leveldb.testStorage: emulated write error")
 	}
 	n, err = tw.Writer.Write(b)
 	if err != nil {
-		ts.t.Errorf("E: write error, num=%d type=%v n=%d: %v", tw.tf.Num(), tw.tf.Type(), n, err)
+		tw.tf.ts.t.Errorf("E: write error, num=%d type=%v n=%d: %v", tw.tf.Num(), tw.tf.Type(), n, err)
 	}
 	return
 }
@@ -98,23 +116,23 @@ func (tw tsWriter) Write(b []byte) (n int, err error) {
 func (tw tsWriter) Sync() (err error) {
 	ts := tw.tf.ts
 	ts.mu.Lock()
-	defer ts.mu.Unlock()
 	for ts.emuDelaySync&tw.tf.Type() != 0 {
 		ts.cond.Wait()
 	}
-	if ts.emuSyncErr&tw.tf.Type() != 0 {
+	ts.mu.Unlock()
+	if tw.tf.shouldErrLocked(tsOpSync) {
 		return errors.New("leveldb.testStorage: emulated sync error")
 	}
 	err = tw.Writer.Sync()
 	if err != nil {
-		ts.t.Errorf("E: sync error, num=%d type=%v: %v", tw.tf.Num(), tw.tf.Type(), err)
+		tw.tf.ts.t.Errorf("E: sync error, num=%d type=%v: %v", tw.tf.Num(), tw.tf.Type(), err)
 	}
 	return
 }
 
 func (tw tsWriter) Close() (err error) {
 	err = tw.Writer.Close()
-	tw.tf.close("reader", err)
+	tw.tf.close("writer", err)
 	return
 }
 
@@ -127,6 +145,16 @@ func (tf tsFile) x() uint64 {
 	return tf.Num()<<typeShift | uint64(tf.Type())
 }
 
+func (tf tsFile) shouldErr(op tsOp) bool {
+	return tf.ts.shouldErr(tf, op)
+}
+
+func (tf tsFile) shouldErrLocked(op tsOp) bool {
+	tf.ts.mu.Lock()
+	defer tf.ts.mu.Unlock()
+	return tf.shouldErr(op)
+}
+
 func (tf tsFile) checkOpen(m string) error {
 	ts := tf.ts
 	if writer, ok := ts.opens[tf.x()]; ok {
@@ -163,7 +191,7 @@ func (tf tsFile) Open() (r storage.Reader, err error) {
 	if err != nil {
 		return
 	}
-	if ts.emuOpenErr&tf.Type() != 0 {
+	if tf.shouldErr(tsOpOpen) {
 		err = errors.New("leveldb.testStorage: emulated open error")
 		return
 	}
@@ -190,7 +218,7 @@ func (tf tsFile) Create() (w storage.Writer, err error) {
 	if err != nil {
 		return
 	}
-	if ts.emuCreateErr&tf.Type() != 0 {
+	if tf.shouldErr(tsOpCreate) {
 		err = errors.New("leveldb.testStorage: emulated create error")
 		return
 	}
@@ -205,6 +233,23 @@ func (tf tsFile) Create() (w storage.Writer, err error) {
 	return
 }
 
+func (tf tsFile) Replace(newfile storage.File) (err error) {
+	ts := tf.ts
+	ts.mu.Lock()
+	defer ts.mu.Unlock()
+	err = tf.checkOpen("replace")
+	if err != nil {
+		return
+	}
+	err = tf.File.Replace(newfile.(tsFile).File)
+	if err != nil {
+		ts.t.Errorf("E: cannot replace file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
+	} else {
+		ts.t.Logf("I: file replace, num=%d type=%v", tf.Num(), tf.Type())
+	}
+	return
+}
+
 func (tf tsFile) Remove() (err error) {
 	ts := tf.ts
 	ts.mu.Lock()
@@ -231,25 +276,61 @@ type testStorage struct {
 	cond sync.Cond
 	// Open files, true=writer, false=reader
 	opens         map[uint64]bool
-	emuOpenErr    storage.FileType
-	emuCreateErr  storage.FileType
 	emuDelaySync  storage.FileType
-	emuWriteErr   storage.FileType
-	emuSyncErr    storage.FileType
 	ignoreOpenErr storage.FileType
 	readCnt       uint64
 	readCntEn     storage.FileType
+
+	emuErr         [tsOpNum]storage.FileType
+	emuErrOnce     [tsOpNum]storage.FileType
+	emuRandErr     [tsOpNum]storage.FileType
+	emuRandErrProb int
+	emuErrOnceMap  map[uint64]uint
+	emuRandRand    *rand.Rand
 }
 
-func (ts *testStorage) SetOpenErr(t storage.FileType) {
+func (ts *testStorage) shouldErr(tf tsFile, op tsOp) bool {
+	if ts.emuErr[op]&tf.Type() != 0 {
+		return true
+	} else if ts.emuRandErr[op]&tf.Type() != 0 || ts.emuErrOnce[op]&tf.Type() != 0 {
+		sop := uint(1) << op
+		eop := ts.emuErrOnceMap[tf.x()]
+		if eop&sop == 0 && (ts.emuRandRand.Int()%ts.emuRandErrProb == 0 || ts.emuErrOnce[op]&tf.Type() != 0) {
+			ts.emuErrOnceMap[tf.x()] = eop | sop
+			ts.t.Logf("I: emulated error: file=%d type=%v op=%v", tf.Num(), tf.Type(), op)
+			return true
+		}
+	}
+	return false
+}
+
+func (ts *testStorage) SetEmuErr(t storage.FileType, ops ...tsOp) {
 	ts.mu.Lock()
-	ts.emuOpenErr = t
+	for _, op := range ops {
+		ts.emuErr[op] = t
+	}
 	ts.mu.Unlock()
 }
 
-func (ts *testStorage) SetCreateErr(t storage.FileType) {
+func (ts *testStorage) SetEmuErrOnce(t storage.FileType, ops ...tsOp) {
 	ts.mu.Lock()
-	ts.emuCreateErr = t
+	for _, op := range ops {
+		ts.emuErrOnce[op] = t
+	}
+	ts.mu.Unlock()
+}
+
+func (ts *testStorage) SetEmuRandErr(t storage.FileType, ops ...tsOp) {
+	ts.mu.Lock()
+	for _, op := range ops {
+		ts.emuRandErr[op] = t
+	}
+	ts.mu.Unlock()
+}
+
+func (ts *testStorage) SetEmuRandErrProb(prob int) {
+	ts.mu.Lock()
+	ts.emuRandErrProb = prob
 	ts.mu.Unlock()
 }
 
@@ -267,18 +348,6 @@ func (ts *testStorage) ReleaseSync(t storage.FileType) {
 	ts.mu.Unlock()
 }
 
-func (ts *testStorage) SetWriteErr(t storage.FileType) {
-	ts.mu.Lock()
-	ts.emuWriteErr = t
-	ts.mu.Unlock()
-}
-
-func (ts *testStorage) SetSyncErr(t storage.FileType) {
-	ts.mu.Lock()
-	ts.emuSyncErr = t
-	ts.mu.Unlock()
-}
-
 func (ts *testStorage) ReadCounter() uint64 {
 	ts.mu.Lock()
 	defer ts.mu.Unlock()
@@ -413,7 +482,11 @@ func newTestStorage(t *testing.T) *testStorage {
 			num := tsNum
 			tsNum++
 			tsMU.Unlock()
-			path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num))
+			tempdir := tsTempdir
+			if tempdir == "" {
+				tempdir = os.TempDir()
+			}
+			path := filepath.Join(tempdir, fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num))
 			if _, err := os.Stat(path); err != nil {
 				stor, err = storage.OpenFile(path)
 				if err != nil {
@@ -436,6 +509,10 @@ func newTestStorage(t *testing.T) *testStorage {
 						}
 						f.Close()
 					}
+					if t.Failed() {
+						t.Logf("testing failed, test DB preserved at %s", path)
+						return nil
+					}
 					if tsKeepFS {
 						return nil
 					}
@@ -449,10 +526,13 @@ func newTestStorage(t *testing.T) *testStorage {
 		stor = storage.NewMemStorage()
 	}
 	ts := &testStorage{
-		t:       t,
-		Storage: stor,
-		closeFn: closeFn,
-		opens:   make(map[uint64]bool),
+		t:              t,
+		Storage:        stor,
+		closeFn:        closeFn,
+		opens:          make(map[uint64]bool),
+		emuErrOnceMap:  make(map[uint64]uint),
+		emuRandErrProb: 0x999,
+		emuRandRand:    rand.New(rand.NewSource(0xfacedead)),
 	}
 	ts.cond.L = &ts.mu
 	return ts

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go

@@ -7,6 +7,7 @@
 package leveldb
 
 import (
+	"fmt"
 	"sort"
 	"sync/atomic"
 
@@ -82,6 +83,18 @@ type tFiles []*tFile
 func (tf tFiles) Len() int      { return len(tf) }
 func (tf tFiles) Swap(i, j int) { tf[i], tf[j] = tf[j], tf[i] }
 
+func (tf tFiles) nums() string {
+	x := "[ "
+	for i, f := range tf {
+		if i != 0 {
+			x += ", "
+		}
+		x += fmt.Sprint(f.file.Num())
+	}
+	x += " ]"
+	return x
+}
+
 // Returns true if i smallest key is less than j.
 // This used for sort by key in ascending order.
 func (tf tFiles) lessByKey(icmp *iComparer, i, j int) bool {
@@ -149,7 +162,7 @@ func (tf tFiles) overlaps(icmp *iComparer, umin, umax []byte, unsorted bool) boo
 	i := 0
 	if len(umin) > 0 {
 		// Find the earliest possible internal key for min.
-		i = tf.searchMax(icmp, newIKey(umin, kMaxSeq, tSeek))
+		i = tf.searchMax(icmp, newIkey(umin, kMaxSeq, ktSeek))
 	}
 	if i >= len(tf) {
 		// Beginning of range is after all files, so no overlap.
@@ -159,24 +172,25 @@ func (tf tFiles) overlaps(icmp *iComparer, umin, umax []byte, unsorted bool) boo
 }
 
 // Returns tables whose its key range overlaps with given key range.
-// If overlapped is true then the search will be expanded to tables that
-// overlaps with each other.
+// Range will be expanded if ukey found hop across tables.
+// If overlapped is true then the search will be restarted if umax
+// expanded.
+// The dst content will be overwritten.
 func (tf tFiles) getOverlaps(dst tFiles, icmp *iComparer, umin, umax []byte, overlapped bool) tFiles {
-	x := len(dst)
+	dst = dst[:0]
 	for i := 0; i < len(tf); {
 		t := tf[i]
 		if t.overlaps(icmp, umin, umax) {
-			if overlapped {
-				// For overlapped files, check if the newly added file has
-				// expanded the range. If so, restart search.
-				if umin != nil && icmp.uCompare(t.imin.ukey(), umin) < 0 {
-					umin = t.imin.ukey()
-					dst = dst[:x]
-					i = 0
-					continue
-				} else if umax != nil && icmp.uCompare(t.imax.ukey(), umax) > 0 {
-					umax = t.imax.ukey()
-					dst = dst[:x]
+			if umin != nil && icmp.uCompare(t.imin.ukey(), umin) < 0 {
+				umin = t.imin.ukey()
+				dst = dst[:0]
+				i = 0
+				continue
+			} else if umax != nil && icmp.uCompare(t.imax.ukey(), umax) > 0 {
+				umax = t.imax.ukey()
+				// Restart search if it is overlapped.
+				if overlapped {
+					dst = dst[:0]
 					i = 0
 					continue
 				}
@@ -272,9 +286,10 @@ func (x *tFilesSortByNum) Less(i, j int) bool {
 
 // Table operations.
 type tOps struct {
-	s       *session
-	cache   cache.Cache
-	cacheNS cache.Namespace
+	s      *session
+	cache  *cache.Cache
+	bcache *cache.Cache
+	bpool  *util.BufferPool
 }
 
 // Creates an empty table and returns table writer.
@@ -288,7 +303,7 @@ func (t *tOps) create() (*tWriter, error) {
 		t:    t,
 		file: file,
 		w:    fw,
-		tw:   table.NewWriter(fw, t.s.o),
+		tw:   table.NewWriter(fw, t.s.o.Options),
 	}, nil
 }
 
@@ -296,7 +311,7 @@ func (t *tOps) create() (*tWriter, error) {
 func (t *tOps) createFrom(src iterator.Iterator) (f *tFile, n int, err error) {
 	w, err := t.create()
 	if err != nil {
-		return f, n, err
+		return
 	}
 
 	defer func() {
@@ -321,33 +336,32 @@ func (t *tOps) createFrom(src iterator.Iterator) (f *tFile, n int, err error) {
 	return
 }
 
-// Opens table. It returns a cache object, which should
+// Opens table. It returns a cache handle, which should
 // be released after use.
-func (t *tOps) open(f *tFile) (c cache.Object, err error) {
+func (t *tOps) open(f *tFile) (ch *cache.Handle, err error) {
 	num := f.file.Num()
-	c, ok := t.cacheNS.Get(num, func() (ok bool, value interface{}, charge int, fin cache.SetFin) {
+	ch = t.cache.Get(0, num, func() (size int, value cache.Value) {
 		var r storage.Reader
 		r, err = f.file.Open()
 		if err != nil {
-			return
+			return 0, nil
 		}
 
-		o := t.s.o
-
-		var cacheNS cache.Namespace
-		if bc := o.GetBlockCache(); bc != nil {
-			cacheNS = bc.GetNamespace(num)
+		var bcache *cache.CacheGetter
+		if t.bcache != nil {
+			bcache = &cache.CacheGetter{Cache: t.bcache, NS: num}
 		}
 
-		ok = true
-		value = table.NewReader(r, int64(f.size), cacheNS, o)
-		charge = 1
-		fin = func() {
+		var tr *table.Reader
+		tr, err = table.NewReader(r, int64(f.size), storage.NewFileInfo(f.file), bcache, t.bpool, t.s.o.Options)
+		if err != nil {
 			r.Close()
+			return 0, nil
 		}
-		return
+		return 1, tr
+
 	})
-	if !ok && err == nil {
+	if ch == nil && err == nil {
 		err = ErrClosed
 	}
 	return
@@ -356,34 +370,43 @@ func (t *tOps) open(f *tFile) (c cache.Object, err error) {
 // Finds key/value pair whose key is greater than or equal to the
 // given key.
 func (t *tOps) find(f *tFile, key []byte, ro *opt.ReadOptions) (rkey, rvalue []byte, err error) {
-	c, err := t.open(f)
+	ch, err := t.open(f)
 	if err != nil {
 		return nil, nil, err
 	}
-	defer c.Release()
-	return c.Value().(*table.Reader).Find(key, ro)
+	defer ch.Release()
+	return ch.Value().(*table.Reader).Find(key, true, ro)
+}
+
+// Finds key that is greater than or equal to the given key.
+func (t *tOps) findKey(f *tFile, key []byte, ro *opt.ReadOptions) (rkey []byte, err error) {
+	ch, err := t.open(f)
+	if err != nil {
+		return nil, err
+	}
+	defer ch.Release()
+	return ch.Value().(*table.Reader).FindKey(key, true, ro)
 }
 
 // Returns approximate offset of the given key.
 func (t *tOps) offsetOf(f *tFile, key []byte) (offset uint64, err error) {
-	c, err := t.open(f)
+	ch, err := t.open(f)
 	if err != nil {
 		return
 	}
-	_offset, err := c.Value().(*table.Reader).OffsetOf(key)
-	offset = uint64(_offset)
-	c.Release()
-	return
+	defer ch.Release()
+	offset_, err := ch.Value().(*table.Reader).OffsetOf(key)
+	return uint64(offset_), err
 }
 
 // Creates an iterator from the given table.
 func (t *tOps) newIterator(f *tFile, slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
-	c, err := t.open(f)
+	ch, err := t.open(f)
 	if err != nil {
 		return iterator.NewEmptyIterator(err)
 	}
-	iter := c.Value().(*table.Reader).NewIterator(slice, ro)
-	iter.SetReleaser(c)
+	iter := ch.Value().(*table.Reader).NewIterator(slice, ro)
+	iter.SetReleaser(ch)
 	return iter
 }
 
@@ -391,14 +414,14 @@ func (t *tOps) newIterator(f *tFile, slice *util.Range, ro *opt.ReadOptions) ite
 // no one use the the table.
 func (t *tOps) remove(f *tFile) {
 	num := f.file.Num()
-	t.cacheNS.Delete(num, func(exist bool) {
+	t.cache.Delete(0, num, func() {
 		if err := f.file.Remove(); err != nil {
 			t.s.logf("table@remove removing @%d %q", num, err)
 		} else {
 			t.s.logf("table@remove removed @%d", num)
 		}
-		if bc := t.s.o.GetBlockCache(); bc != nil {
-			bc.GetNamespace(num).Zap(false)
+		if t.bcache != nil {
+			t.bcache.EvictNS(num)
 		}
 	})
 }
@@ -406,14 +429,35 @@ func (t *tOps) remove(f *tFile) {
 // Closes the table ops instance. It will close all tables,
 // regadless still used or not.
 func (t *tOps) close() {
-	t.cache.Zap(true)
+	t.bpool.Close()
+	t.cache.Close()
+	if t.bcache != nil {
+		t.bcache.Close()
+	}
 }
 
 // Creates new initialized table ops instance.
-func newTableOps(s *session, cacheCap int) *tOps {
-	c := cache.NewLRUCache(cacheCap)
-	ns := c.GetNamespace(0)
-	return &tOps{s, c, ns}
+func newTableOps(s *session) *tOps {
+	var (
+		cacher cache.Cacher
+		bcache *cache.Cache
+	)
+	if s.o.GetOpenFilesCacheCapacity() > 0 {
+		cacher = cache.NewLRU(s.o.GetOpenFilesCacheCapacity())
+	}
+	if !s.o.DisableBlockCache {
+		var bcacher cache.Cacher
+		if s.o.GetBlockCacheCapacity() > 0 {
+			bcacher = cache.NewLRU(s.o.GetBlockCacheCapacity())
+		}
+		bcache = cache.NewCache(bcacher)
+	}
+	return &tOps{
+		s:      s,
+		cache:  cache.NewCache(cacher),
+		bcache: bcache,
+		bpool:  util.NewBufferPool(s.o.GetBlockSize() + 5),
+	}
 }
 
 // tWriter wraps the table writer. It keep track of file descriptor
@@ -442,28 +486,34 @@ func (w *tWriter) empty() bool {
 	return w.first == nil
 }
 
+// Closes the storage.Writer.
+func (w *tWriter) close() {
+	if w.w != nil {
+		w.w.Close()
+		w.w = nil
+	}
+}
+
 // Finalizes the table and returns table file.
 func (w *tWriter) finish() (f *tFile, err error) {
+	defer w.close()
 	err = w.tw.Close()
 	if err != nil {
 		return
 	}
 	err = w.w.Sync()
 	if err != nil {
-		w.w.Close()
 		return
 	}
-	w.w.Close()
 	f = newTableFile(w.file, uint64(w.tw.BytesLen()), iKey(w.first), iKey(w.last))
 	return
 }
 
 // Drops the table.
 func (w *tWriter) drop() {
-	w.w.Close()
+	w.close()
 	w.file.Remove()
 	w.t.s.reuseFileNum(w.file.Num())
-	w.w = nil
 	w.file = nil
 	w.tw = nil
 	w.first = nil

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/block_test.go

@@ -19,13 +19,18 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
-func (b *block) TestNewIterator(slice *util.Range) iterator.Iterator {
-	return b.newIterator(slice, false, nil)
+type blockTesting struct {
+	tr *Reader
+	b  *block
+}
+
+func (t *blockTesting) TestNewIterator(slice *util.Range) iterator.Iterator {
+	return t.tr.newBlockIter(t.b, nil, slice, false)
 }
 
 var _ = testutil.Defer(func() {
 	Describe("Block", func() {
-		Build := func(kv *testutil.KeyValue, restartInterval int) *block {
+		Build := func(kv *testutil.KeyValue, restartInterval int) *blockTesting {
 			// Building the block.
 			bw := &blockWriter{
 				restartInterval: restartInterval,
@@ -39,11 +44,13 @@ var _ = testutil.Defer(func() {
 			// Opening the block.
 			data := bw.buf.Bytes()
 			restartsLen := int(binary.LittleEndian.Uint32(data[len(data)-4:]))
-			return &block{
-				cmp:            comparer.DefaultComparer,
-				data:           data,
-				restartsLen:    restartsLen,
-				restartsOffset: len(data) - (restartsLen+1)*4,
+			return &blockTesting{
+				tr: &Reader{cmp: comparer.DefaultComparer},
+				b: &block{
+					data:           data,
+					restartsLen:    restartsLen,
+					restartsOffset: len(data) - (restartsLen+1)*4,
+				},
 			}
 		}
 
@@ -59,7 +66,7 @@ var _ = testutil.Defer(func() {
 						// Make block.
 						br := Build(kv, restartInterval)
 						// Do testing.
-						testutil.KeyValueTesting(nil, br, kv.Clone())
+						testutil.KeyValueTesting(nil, kv.Clone(), br, nil, nil)
 					}
 
 					Describe(Text(), Test)
@@ -102,11 +109,11 @@ var _ = testutil.Defer(func() {
 			for restartInterval := 1; restartInterval <= 5; restartInterval++ {
 				Describe(fmt.Sprintf("with restart interval of %d", restartInterval), func() {
 					// Make block.
-					br := Build(kv, restartInterval)
+					bt := Build(kv, restartInterval)
 
 					Test := func(r *util.Range) func(done Done) {
 						return func(done Done) {
-							iter := br.newIterator(r, false, nil)
+							iter := bt.TestNewIterator(r)
 							Expect(iter.Error()).ShouldNot(HaveOccurred())
 
 							t := testutil.IteratorTesting{
@@ -115,6 +122,7 @@ var _ = testutil.Defer(func() {
 							}
 
 							testutil.DoIteratorTesting(&t)
+							iter.Release()
 							done <- true
 						}
 					}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table.go

@@ -133,9 +133,9 @@ Filter block trailer:
 
       +- 4-bytes -+
      /             \
-    +---------------+---------------+---------------+-------------------------+------------------+
-    |    offset 1   |      ....     |    offset n   | filter offset (4-bytes) | base Lg (1-byte) |
-    +-------------- +---------------+---------------+-------------------------+------------------+
+    +---------------+---------------+---------------+-------------------------------+------------------+
+    | data 1 offset |      ....     | data n offset | data-offsets offset (4-bytes) | base Lg (1-byte) |
+    +-------------- +---------------+---------------+-------------------------------+------------------+
 
 
 NOTE: All fixed-length integer are little-endian.

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table_suite_test.go

@@ -3,15 +3,9 @@ package table
 import (
 	"testing"
 
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
 	"github.com/syndtr/goleveldb/leveldb/testutil"
 )
 
 func TestTable(t *testing.T) {
-	testutil.RunDefer()
-
-	RegisterFailHandler(Fail)
-	RunSpecs(t, "Table Suite")
+	testutil.RunSuite(t, "Table Suite")
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table_test.go

@@ -23,7 +23,7 @@ type tableWrapper struct {
 }
 
 func (t tableWrapper) TestFind(key []byte) (rkey, rvalue []byte, err error) {
-	return t.Reader.Find(key, nil)
+	return t.Reader.Find(key, false, nil)
 }
 
 func (t tableWrapper) TestGet(key []byte) (value []byte, err error) {
@@ -59,7 +59,8 @@ var _ = testutil.Defer(func() {
 			It("Should be able to approximate offset of a key correctly", func() {
 				Expect(err).ShouldNot(HaveOccurred())
 
-				tr := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, o)
+				tr, err := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, nil, nil, o)
+				Expect(err).ShouldNot(HaveOccurred())
 				CheckOffset := func(key string, expect, threshold int) {
 					offset, err := tr.OffsetOf([]byte(key))
 					Expect(err).ShouldNot(HaveOccurred())
@@ -95,7 +96,7 @@ var _ = testutil.Defer(func() {
 				tw.Close()
 
 				// Opening the table.
-				tr := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, o)
+				tr, _ := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, nil, nil, o)
 				return tableWrapper{tr}
 			}
 			Test := func(kv *testutil.KeyValue, body func(r *Reader)) func() {
@@ -104,14 +105,16 @@ var _ = testutil.Defer(func() {
 					if body != nil {
 						body(db.(tableWrapper).Reader)
 					}
-					testutil.KeyValueTesting(nil, db, *kv)
+					testutil.KeyValueTesting(nil, *kv, db, nil, nil)
 				}
 			}
 
-			testutil.AllKeyValueTesting(nil, Build)
+			testutil.AllKeyValueTesting(nil, Build, nil, nil)
 			Describe("with one key per block", Test(testutil.KeyValue_Generate(nil, 9, 1, 10, 512, 512), func(r *Reader) {
 				It("should have correct blocks number", func() {
-					Expect(r.indexBlock.restartsLen).Should(Equal(9))
+					indexBlock, err := r.readBlock(r.indexBH, true)
+					Expect(err).To(BeNil())
+					Expect(indexBlock.restartsLen).Should(Equal(9))
 				})
 			}))
 		})

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/writer.go

@@ -12,7 +12,7 @@ import (
 	"fmt"
 	"io"
 
-	"code.google.com/p/snappy-go/snappy"
+	"github.com/syndtr/gosnappy/snappy"
 
 	"github.com/syndtr/goleveldb/leveldb/comparer"
 	"github.com/syndtr/goleveldb/leveldb/filter"

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/db.go

@@ -12,6 +12,7 @@ import (
 
 	. "github.com/onsi/gomega"
 
+	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
@@ -34,6 +35,10 @@ type Get interface {
 	TestGet(key []byte) (value []byte, err error)
 }
 
+type Has interface {
+	TestHas(key []byte) (ret bool, err error)
+}
+
 type NewIterator interface {
 	TestNewIterator(slice *util.Range) iterator.Iterator
 }
@@ -110,7 +115,7 @@ func (t *DBTesting) TestAllPresent() {
 
 func (t *DBTesting) TestDeletedKey(key []byte) {
 	_, err := t.DB.TestGet(key)
-	Expect(err).Should(Equal(util.ErrNotFound), "Get on deleted key %q, %s", key, t.text())
+	Expect(err).Should(Equal(errors.ErrNotFound), "Get on deleted key %q, %s", key, t.text())
 }
 
 func (t *DBTesting) TestAllDeleted() {
@@ -212,5 +217,6 @@ func DoDBTesting(t *DBTesting) {
 		}
 
 		DoIteratorTesting(&it)
+		iter.Release()
 	}
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/ginkgo.go

@@ -0,0 +1,21 @@
+package testutil
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+func RunSuite(t GinkgoTestingT, name string) {
+	RunDefer()
+
+	SynchronizedBeforeSuite(func() []byte {
+		RunDefer("setup")
+		return nil
+	}, func(data []byte) {})
+	SynchronizedAfterSuite(func() {
+		RunDefer("teardown")
+	}, func() {})
+
+	RegisterFailHandler(Fail)
+	RunSpecs(t, name)
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/kvtest.go

@@ -13,16 +13,28 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
+	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
-func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
+func KeyValueTesting(rnd *rand.Rand, kv KeyValue, p DB, setup func(KeyValue) DB, teardown func(DB)) {
 	if rnd == nil {
 		rnd = NewRand()
 	}
 
-	if db, ok := p.(Find); ok {
-		It("Should find all keys with Find", func() {
+	if p == nil {
+		BeforeEach(func() {
+			p = setup(kv)
+		})
+		if teardown != nil {
+			AfterEach(func() {
+				teardown(p)
+			})
+		}
+	}
+
+	It("Should find all keys with Find", func() {
+		if db, ok := p.(Find); ok {
 			ShuffledIndex(nil, kv.Len(), 1, func(i int) {
 				key_, key, value := kv.IndexInexact(i)
 
@@ -38,9 +50,11 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
 				Expect(rkey).Should(Equal(key))
 				Expect(rvalue).Should(Equal(value), "Value for key %q (%q)", key_, key)
 			})
-		})
+		}
+	})
 
-		It("Should return error if the key is not present", func() {
+	It("Should return error if the key is not present", func() {
+		if db, ok := p.(Find); ok {
 			var key []byte
 			if kv.Len() > 0 {
 				key_, _ := kv.Index(kv.Len() - 1)
@@ -48,12 +62,12 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
 			}
 			rkey, _, err := db.TestFind(key)
 			Expect(err).Should(HaveOccurred(), "Find for key %q yield key %q", key, rkey)
-			Expect(err).Should(Equal(util.ErrNotFound))
-		})
-	}
+			Expect(err).Should(Equal(errors.ErrNotFound))
+		}
+	})
 
-	if db, ok := p.(Get); ok {
-		It("Should only find exact key with Get", func() {
+	It("Should only find exact key with Get", func() {
+		if db, ok := p.(Get); ok {
 			ShuffledIndex(nil, kv.Len(), 1, func(i int) {
 				key_, key, value := kv.IndexInexact(i)
 
@@ -66,14 +80,34 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
 				if len(key_) > 0 {
 					_, err = db.TestGet(key_)
 					Expect(err).Should(HaveOccurred(), "Error for key %q", key_)
-					Expect(err).Should(Equal(util.ErrNotFound))
+					Expect(err).Should(Equal(errors.ErrNotFound))
 				}
 			})
-		})
-	}
+		}
+	})
 
-	if db, ok := p.(NewIterator); ok {
-		TestIter := func(r *util.Range, _kv KeyValue) {
+	It("Should only find present key with Has", func() {
+		if db, ok := p.(Has); ok {
+			ShuffledIndex(nil, kv.Len(), 1, func(i int) {
+				key_, key, _ := kv.IndexInexact(i)
+
+				// Using exact key.
+				ret, err := db.TestHas(key)
+				Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key)
+				Expect(ret).Should(BeTrue(), "False for key %q", key)
+
+				// Using inexact key.
+				if len(key_) > 0 {
+					ret, err = db.TestHas(key_)
+					Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key_)
+					Expect(ret).ShouldNot(BeTrue(), "True for key %q", key)
+				}
+			})
+		}
+	})
+
+	TestIter := func(r *util.Range, _kv KeyValue) {
+		if db, ok := p.(NewIterator); ok {
 			iter := db.TestNewIterator(r)
 			Expect(iter.Error()).ShouldNot(HaveOccurred())
 
@@ -83,46 +117,62 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
 			}
 
 			DoIteratorTesting(&t)
+			iter.Release()
+		}
+	}
+
+	It("Should iterates and seeks correctly", func(done Done) {
+		TestIter(nil, kv.Clone())
+		done <- true
+	}, 3.0)
+
+	RandomIndex(rnd, kv.Len(), Min(kv.Len(), 50), func(i int) {
+		type slice struct {
+			r            *util.Range
+			start, limit int
 		}
 
-		It("Should iterates and seeks correctly", func(done Done) {
-			TestIter(nil, kv.Clone())
-			done <- true
-		}, 3.0)
-
-		RandomIndex(rnd, kv.Len(), kv.Len(), func(i int) {
-			type slice struct {
-				r            *util.Range
-				start, limit int
-			}
-
-			key_, _, _ := kv.IndexInexact(i)
-			for _, x := range []slice{
-				{&util.Range{Start: key_, Limit: nil}, i, kv.Len()},
-				{&util.Range{Start: nil, Limit: key_}, 0, i},
-			} {
-				It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", x.start, x.limit), func(done Done) {
-					TestIter(x.r, kv.Slice(x.start, x.limit))
-					done <- true
-				}, 3.0)
-			}
-		})
-
-		RandomRange(rnd, kv.Len(), kv.Len(), func(start, limit int) {
-			It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", start, limit), func(done Done) {
-				r := kv.Range(start, limit)
-				TestIter(&r, kv.Slice(start, limit))
+		key_, _, _ := kv.IndexInexact(i)
+		for _, x := range []slice{
+			{&util.Range{Start: key_, Limit: nil}, i, kv.Len()},
+			{&util.Range{Start: nil, Limit: key_}, 0, i},
+		} {
+			It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", x.start, x.limit), func(done Done) {
+				TestIter(x.r, kv.Slice(x.start, x.limit))
 				done <- true
 			}, 3.0)
-		})
-	}
+		}
+	})
+
+	RandomRange(rnd, kv.Len(), Min(kv.Len(), 50), func(start, limit int) {
+		It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", start, limit), func(done Done) {
+			r := kv.Range(start, limit)
+			TestIter(&r, kv.Slice(start, limit))
+			done <- true
+		}, 3.0)
+	})
 }
 
-func AllKeyValueTesting(rnd *rand.Rand, body func(kv KeyValue) DB) {
+func AllKeyValueTesting(rnd *rand.Rand, body, setup func(KeyValue) DB, teardown func(DB)) {
 	Test := func(kv *KeyValue) func() {
 		return func() {
-			db := body(*kv)
-			KeyValueTesting(rnd, db, *kv)
+			var p DB
+			if setup != nil {
+				Defer("setup", func() {
+					p = setup(*kv)
+				})
+			}
+			if teardown != nil {
+				Defer("teardown", func() {
+					teardown(p)
+				})
+			}
+			if body != nil {
+				p = body(*kv)
+			}
+			KeyValueTesting(rnd, *kv, p, func(KeyValue) DB {
+				return p
+			}, nil)
 		}
 	}
 
@@ -133,4 +183,5 @@ func AllKeyValueTesting(rnd *rand.Rand, body func(kv KeyValue) DB) {
 	Describe("with big value", Test(KeyValue_BigValue()))
 	Describe("with special key", Test(KeyValue_SpecialKey()))
 	Describe("with multiple key/value", Test(KeyValue_MultipleKeyValue()))
+	Describe("with generated key/value", Test(KeyValue_Generate(nil, 120, 1, 50, 10, 120)))
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/storage.go

@@ -397,6 +397,7 @@ func (s *Storage) logI(format string, args ...interface{}) {
 
 func (s *Storage) Log(str string) {
 	s.log(1, "Log: "+str)
+	s.Storage.Log(str)
 }
 
 func (s *Storage) Lock() (r util.Releaser, err error) {

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/util.go

@@ -155,3 +155,17 @@ func RandomRange(rnd *rand.Rand, n, round int, fn func(start, limit int)) {
 	}
 	return
 }
+
+func Max(x, y int) int {
+	if x > y {
+		return x
+	}
+	return y
+}
+
+func Min(x, y int) int {
+	if x < y {
+		return x
+	}
+	return y
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil_test.go

@@ -34,6 +34,10 @@ func (t *testingDB) TestGet(key []byte) (value []byte, err error) {
 	return t.Get(key, t.ro)
 }
 
+func (t *testingDB) TestHas(key []byte) (ret bool, err error) {
+	return t.Has(key, t.ro)
+}
+
 func (t *testingDB) TestNewIterator(slice *util.Range) iterator.Iterator {
 	return t.NewIterator(slice, t.ro)
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util.go

@@ -14,10 +14,10 @@ import (
 )
 
 func shorten(str string) string {
-	if len(str) <= 4 {
+	if len(str) <= 8 {
 		return str
 	}
-	return str[:1] + ".." + str[len(str)-1:]
+	return str[:3] + ".." + str[len(str)-3:]
 }
 
 var bunits = [...]string{"", "Ki", "Mi", "Gi"}