Revert "Use a monitor process to handle panics and restarts (fixes #586)"

This reverts commit 10f0713257.

Conflicts:
	cmd/syncthing/monitor.go

.gitattributes

@@ -1,9 +0,0 @@
-# Text files use LF line endings in this repository
-*   text=auto
-
-# Except the dependencies, which we leave alone
-Godeps/**   -text=auto
-
-# Diffs on these files are meaningless
-gui.files.go    -diff
-*.svg           -diff

.gitignore

@@ -1,16 +1,12 @@
 syncthing
-!gui/syncthing
-!Godeps/_workspace/src/github.com/syncthing
 syncthing.exe
 *.tar.gz
 *.zip
 *.asc
+*.sublime*
 .jshintrc
 coverage.out
 files/pidx
 bin
 perfstats*.csv
 coverage.xml
-syncthing.sig
-RELEASE
-deb

.mailmap

@@ -1 +0,0 @@
-NICKS

AUTHORS

@@ -1,72 +0,0 @@
-# This is the official list of Syncthing authors for copyright purposes.
-
-Aaron Bieber <qbit@deftly.net>
-Adam Piggott	<aD@simplypeachy.co.uk> <simplypeachy@users.noreply.github.com>
-Alexander Graf <register-github@alex-graf.de>
-Andrew Dunham <andrew@du.nham.ca>
-Antony Male <antony.male@gmail.com>
-Arthur Axel fREW Schmidt <frew@afoolishmanifesto.com> <frioux@gmail.com>
-Audrius Butkevicius <audrius.butkevicius@gmail.com>
-Bart De Vries <devriesb@gmail.com>
-Ben Curthoys <ben@bencurthoys.com>
-Ben Schulz <ueomkail@gmail.com> <uok@users.noreply.github.com>
-Ben Sidhom <bsidhom@gmail.com>
-Brandon Philips <brandon@ifup.org>
-Brendan Long <self@brendanlong.com>
-Brian R. Becker <brbecker@gmail.com>
-Caleb Callaway <enlightened.despot@gmail.com>
-Carsten Hagemann <moter8@gmail.com>
-Cathryne Linenweaver <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
-Chris Howie <me@chrishowie.com>
-Chris Joel <chris@scriptolo.gy>
-Colin Kennedy <moshen.colin@gmail.com>
-Daniel Bergmann <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
-Daniel Martí <mvdan@mvdan.cc>
-Denis A. <denisva@gmail.com>
-Dennis Wilson <dw@risu.io>
-Dominik Heidler <dominik@heidler.eu>
-Elias Jarlebring <jarlebring@gmail.com>
-Emil Hessman <emil@hessman.se>
-Erik Meitner <e.meitner@willystreet.coop>
-Federico Castagnini <federico.castagnini@gmail.com>
-Felix Ableitner <me@nutomic.com>
-Felix Unterpaintner <bigbear2nd@gmail.com>
-Francois-Xavier Gsell <fxgsell@gmail.com>
-Frank Isemann <frank@isemann.name>
-Gilli Sigurdsson <gilli@vx.is>
-Jacek Szafarkiewicz <szafar@linux.pl>
-Jakob Borg <jakob@nym.se>
-Jake Peterson <jake@acogdev.com>
-James Patterson <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
-Jaroslav Malec <dzardacz@gmail.com>
-Jens Diemer <github.com@jensdiemer.de> <git@jensdiemer.de>
-Jochen Voss <voss@seehuhn.de>
-Johan Vromans <jvromans@squirrel.nl>
-Karol Różycki <rozycki.karol@gmail.com>
-Ken'ichi Kamada <kamada@nanohz.org>
-Lode Hoste <zillode@zillode.be>
-Lord Landon Agahnim <lordlandon@gmail.com>
-Marc Laporte <marc@marclaporte.com> <marc@laporte.name>
-Marc Pujol <kilburn@la3.org>
-Marcin Dziadus <dziadus.marcin@gmail.com>
-Mateusz Naściszewski <matin1111@wp.pl>
-Matt Burke <mburke@amplify.com> <burkemw3@gmail.com>
-Michael Jephcote <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
-Michael Tilli <pyfisch@gmail.com>
-Pascal Jungblut <github@pascalj.com> <mail@pascal-jungblut.com>
-Peter Hoeg <peter@speartail.com>
-Philippe Schommers <philippe@schommers.be>
-Phill Luby <phill.luby@newredo.com>
-Piotr Bejda <piotrb10@gmail.com>
-Ryan Sullivan <kayoticsully@gmail.com>
-Sergey Mishin <ralder@yandex.ru>
-Stefan Tatschner <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
-Stefan Kuntz <stefan.github@gmail.com> <Stefan.github@gmail.com>
-Tim Abell <tim@timwise.co.uk>
-Tobias Nygren <tnn@nygren.pp.se>
-Tomas Cerveny <kozec@kozec.com>
-Tully Robinson <tully@tojr.org>
-Tyler Brazier <tyler@tylerbrazier.com>
-Veeti Paananen <veeti.paananen@rojekti.fi>
-Vil Brekin <vilbrekin@gmail.com>
-Yannic A. <eipiminusone+github@gmail.com> <eipiminus1@users.noreply.github.com>

CONDUCT.md

@@ -1,92 +0,0 @@
-## Conduct
-
-* We are committed to providing a friendly, safe and welcoming
-  environment for all, regardless of gender, sexual orientation,
-  disability, ethnicity, religion, or similar personal characteristic.
-
-* On IRC, please avoid using overtly sexual nicknames or other nicknames
-  that might detract from a friendly, safe and welcoming environment for
-  all.
-
-* Please be kind and courteous. There's no need to be mean or rude.
-
-* Respect that people have differences of opinion and that every design
-  or implementation choice carries a trade-off and numerous costs. There
-  is seldom a right answer.
-
-* Please keep unstructured critique to a minimum. If you have solid
-  ideas you want to experiment with, make a fork and see how it works.
-
-* We will exclude you from interaction if you insult, demean or harass
-  anyone. That is not welcome behaviour. We interpret the term
-  "harassment" as including the definition in the <a
-  href="http://citizencodeofconduct.org/">Citizen Code of Conduct</a>;
-  if you have any lack of clarity about what might be included in that
-  concept, please read their definition. In particular, we don't
-  tolerate behavior that excludes people in socially marginalized
-  groups.
-
-* Private harassment is also unacceptable. No matter who you are, if you
-  feel you have been or are being harassed or made uncomfortable by a
-  community member, please contact one of the channel ops or any of the
-  Syncthing core team immediately. Whether you're a regular  contributor
-  or a newcomer, we care about making this community a safe place for
-  you and we've got your back.
-
-* Likewise any spamming, trolling, flaming, baiting or other
-  attention-stealing behaviour is not welcome.
-
-## Moderation
-
-These are the policies for upholding our community's standards of
-conduct in our communication channels, most notably in Syncthing-related
-IRC channels and on the web forum.
-
-1. Remarks that violate the Syncthing standards of conduct, including
-   hateful, hurtful, oppressive, or exclusionary remarks, are not
-   allowed. (Cursing is allowed, but never targeting another user, and
-   never in a hateful manner.)
-
-2. Remarks that moderators find inappropriate, whether listed in the
-   code of conduct or not, are also not allowed.
-
-3. Moderators will first respond to such remarks with a warning.
-
-4. If the warning is unheeded, the user will be "kicked," i.e., kicked
-   out of the communication channel to cool off.
-
-5. If the user comes back and continues to make trouble, they will be
-   banned, i.e., indefinitely excluded.
-
-6. Moderators may choose at their discretion to un-ban the user if it
-   was a first offense and they offer the offended party a genuine
-   apology.
-
-7. If a moderator bans someone and you think it was unjustified, please
-   take it up with that moderator, or with a different moderator, **in
-   private**. Complaints about bans in-channel are not allowed.
-
-8. Moderators are held to a higher standard than other community
-   members. If a moderator creates an inappropriate situation, they
-   should expect less leeway than others.
-
-In the Syncthing community we strive to go the extra step to look out
-for each other. Don't just aim to be technically unimpeachable, try to
-be your best self. In particular, avoid flirting with offensive or
-sensitive issues, particularly if they're off-topic; this all too
-often leads to unnecessary fights, hurt feelings, and damaged trust;
-worse, it can drive people away from the community entirely.
-
-And if someone takes issue with something you said or did, resist the
-urge to be defensive. Just stop doing what it was they complained about
-and apologize. Even if you feel you were misinterpreted or unfairly
-accused, chances are good there was something you could've communicated
-better — remember that it's your responsibility to make your fellow
-community members comfortable. Everyone wants to get along and we are
-all here first and foremost because we want to talk about cool
-technology. You will find that people will be eager to assume good
-intent and forgive as long as you earn their trust.
-
-*Adapted from the [Rust Code of Conduct](https://github.com/rust-lang/rust/wiki/Note-development-policy#conduct)*
-
-*Adapted from the [Node.js Policy on Trolling](http://blog.izs.me/post/30036893703/policy-on-trolling)*

CONTRIBUTING.md

@@ -12,7 +12,7 @@ least the following:
  - What operating system, operating system version and version of
    Syncthing you are running
 
- - The same for other connected devices, where relevant
+ - The same for other connected nodes, where relevant
 
  - Screenshot if the issue concerns something visible in the GUI
 
@@ -31,22 +31,63 @@ latest info on Transifex.
 
 ## Contributing Code
 
-Every contribution is welcome. If you want to contribute but are unsure
-where to start, any open issues are fair game! See the [Contribution
-Guidelines](http://docs.syncthing.net/dev/contributing.html) for the full
-story on committing code.
-
-## Contributing Documentation
-
-Updates to the [documentation site](http://docs.syncthing.net/) can be
-made as pull requests on the [documentation
-repository](https://github.com/syncthing/docs).
+Please do contribute! If you want to contribute but are unsure where to
+start, the [Contributions Needed
+topic](http://discourse.syncthing.net/t/49) lists areas in need of
+attention. In general, any open issues are fair game! Be prepared for a
+[certain amount of
+review](https://discourse.syncthing.net/t/733); it's all in the name of
+quality. :)
 
 ## Licensing
 
-All contributions are made under the same MPLv2 license as the rest of
-the project, except documentation, user interface text and translation
-strings which are licensed under the Creative Commons Attribution 4.0
-International License. You retain the copyright to code you have
-written.
+All contributions are made under the same MIT License as the rest of the
+project, except documentation which is licensed under the Creative
+Commons Attribution 4.0 International License. You retain the copyright
+to code you have written.
 
+When accepting your first contribution, the maintainer of the project
+will ensure that you are added to the CONTRIBUTORS file. You are welcome
+to add yourself as a separate commit in your first pull request.
+
+## Building
+
+[See the documentation](http://discourse.syncthing.net/t/44) on how to
+get started with a build environment.
+
+## Branches
+
+ - `master` is the main branch containing good code that will end up in
+   the next release. You should base your work on it. It won't ever be
+   rebased or force-pushed to.
+
+ - `vx.y` branches exist to make patch releases on otherwise obsolete
+   minor releases. Should only contain fixes cherry picked from master.
+   Don't base any work on them.
+
+ - Other branches are probably topic branches and may be subject to
+   rebasing. Don't base any work on them unless you specifically know
+   otherwise.
+
+## Tags
+
+All releases are tagged semver style as `vx.y.z`. Release tags are
+signed by GPG key BCE524C7.
+
+## Tests
+
+Yes please!
+
+## Style
+
+ - `go fmt`
+
+ - Unix line breaks
+
+## Documentation
+
+[Over here!](http://discourse.syncthing.net/category/documentation)
+
+## License
+
+MIT

CONTRIBUTORS

@@ -0,0 +1,16 @@
+Aaron Bieber <qbit@deftly.net>
+Alexander Graf <register-github@alex-graf.de>
+Andrew Dunham <andrew@du.nham.ca>
+Audrius Butkevicius <audrius.butkevicius@gmail.com>
+Arthur Axel fREW Schmidt <frew@afoolishmanifesto.com> <frioux@gmail.com>
+Ben Sidhom <bsidhom@gmail.com>
+Brandon Philips <brandon@ifup.org>
+Gilli Sigurdsson <gilli@vx.is>
+James Patterson <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
+Jens Diemer <github.com@jensdiemer.de> <git@jensdiemer.de>
+Marcin Dziadus <dziadus.marcin@gmail.com>
+Michael Tilli <pyfisch@gmail.com>
+Philippe Schommers <philippe@schommers.be>
+Ryan Sullivan <kayoticsully@gmail.com>
+Tully Robinson <tully@tojr.org>
+Veeti Paananen <veeti.paananen@rojekti.fi>

Godeps/Godeps.json

@@ -1,46 +1,55 @@
 {
 	"ImportPath": "github.com/syncthing/syncthing",
-	"GoVersion": "go1.5.1",
+	"GoVersion": "go1.3.1",
 	"Packages": [
 		"./cmd/..."
 	],
 	"Deps": [
+		{
+			"ImportPath": "bitbucket.org/kardianos/osext",
+			"Comment": "null-13",
+			"Rev": "5d3ddcf53a508cc2f7404eaebf546ef2cb5cdb6e"
+		},
+		{
+			"ImportPath": "code.google.com/p/go.crypto/bcrypt",
+			"Comment": "null-216",
+			"Rev": "41cd4647fccc72b0b79ef1bd1fe6735e718257cd"
+		},
+		{
+			"ImportPath": "code.google.com/p/go.crypto/blowfish",
+			"Comment": "null-216",
+			"Rev": "41cd4647fccc72b0b79ef1bd1fe6735e718257cd"
+		},
+		{
+			"ImportPath": "code.google.com/p/go.text/transform",
+			"Comment": "null-90",
+			"Rev": "d65bffbc88a153d23a6d2a864531e6e7c2cde59b"
+		},
+		{
+			"ImportPath": "code.google.com/p/go.text/unicode/norm",
+			"Comment": "null-90",
+			"Rev": "d65bffbc88a153d23a6d2a864531e6e7c2cde59b"
+		},
+		{
+			"ImportPath": "code.google.com/p/snappy-go/snappy",
+			"Comment": "null-15",
+			"Rev": "12e4b4183793ac4b061921e7980845e750679fd0"
+		},
 		{
 			"ImportPath": "github.com/bkaradzic/go-lz4",
-			"Rev": "74ddf82598bc4745b965729e9c6a463bedd33049"
-		},
-		{
-			"ImportPath": "github.com/calmh/du",
-			"Rev": "3c0690cca16228b97741327b1b6781397afbdb24"
-		},
-		{
-			"ImportPath": "github.com/calmh/luhn",
-			"Rev": "0c8388ff95fa92d4094011e5a04fc99dea3d1632"
+			"Rev": "77e2ba877bde9da31213bec75dbbe197fa507c21"
 		},
 		{
 			"ImportPath": "github.com/calmh/xdr",
-			"Rev": "47c0042d09a827b81ee62497f99e5e0c7f0bd31c"
-		},
-		{
-			"ImportPath": "github.com/golang/snappy",
-			"Rev": "723cc1e459b8eea2dea4583200fd60757d40097a"
+			"Rev": "a597b63b87d6140f79084c8aab214b4d533833a1"
 		},
 		{
 			"ImportPath": "github.com/juju/ratelimit",
-			"Rev": "772f5c38e468398c4511514f4f6aa9a4185bc0a0"
-		},
-		{
-			"ImportPath": "github.com/kardianos/osext",
-			"Rev": "6e7f843663477789fac7c02def0d0909e969b4e5"
+			"Rev": "f9f36d11773655c0485207f0ad30dc2655f69d56"
 		},
 		{
 			"ImportPath": "github.com/syndtr/goleveldb/leveldb",
-			"Rev": "1a9d62f03ea92815b46fcaab357cfd4df264b1a0"
-		},
-		{
-			"ImportPath": "github.com/thejerf/suture",
-			"Comment": "v1.0.1",
-			"Rev": "99c1f2d613756768fc4299acd9dc621e11ed3fd7"
+			"Rev": "2b99e8d4757bf06eeab1b0485d80b8ae1c088874"
 		},
 		{
 			"ImportPath": "github.com/vitrun/qart/coding",
@@ -53,34 +62,6 @@
 		{
 			"ImportPath": "github.com/vitrun/qart/qr",
 			"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
-		},
-		{
-			"ImportPath": "golang.org/x/crypto/bcrypt",
-			"Rev": "81bf7719a6b7ce9b665598222362b50122dfc13b"
-		},
-		{
-			"ImportPath": "golang.org/x/crypto/blowfish",
-			"Rev": "81bf7719a6b7ce9b665598222362b50122dfc13b"
-		},
-		{
-			"ImportPath": "golang.org/x/net/internal/iana",
-			"Rev": "4b709d93778b93d2f34943e3142c71578d83ad31"
-		},
-		{
-			"ImportPath": "golang.org/x/net/ipv6",
-			"Rev": "4b709d93778b93d2f34943e3142c71578d83ad31"
-		},
-		{
-			"ImportPath": "golang.org/x/net/proxy",
-			"Rev": "4b709d93778b93d2f34943e3142c71578d83ad31"
-		},
-		{
-			"ImportPath": "golang.org/x/text/transform",
-			"Rev": "723492b65e225eafcba054e76ba18bb9c5ac1ea2"
-		},
-		{
-			"ImportPath": "golang.org/x/text/unicode/norm",
-			"Rev": "723492b65e225eafcba054e76ba18bb9c5ac1ea2"
 		}
 	]
 }

Godeps/_workspace/src/bitbucket.org/kardianos/osext/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Daniel Theophanes
+
+This software is provided 'as-is', without any express or implied
+warranty. In no event will the authors be held liable for any damages
+arising from the use of this software.
+
+Permission is granted to anyone to use this software for any purpose,
+including commercial applications, and to alter it and redistribute it
+freely, subject to the following restrictions:
+
+   1. The origin of this software must not be misrepresented; you must not
+   claim that you wrote the original software. If you use this software
+   in a product, an acknowledgment in the product documentation would be
+   appreciated but is not required.
+
+   2. Altered source versions must be plainly marked as such, and must not be
+   misrepresented as being the original software.
+
+   3. This notice may not be removed or altered from any source
+   distribution.

Godeps/_workspace/src/bitbucket.org/kardianos/osext/osext.go

@@ -16,12 +16,17 @@ func Executable() (string, error) {
 }
 
 // Returns same path as Executable, returns just the folder
-// path. Excludes the executable name and any trailing slash.
+// path. Excludes the executable name.
 func ExecutableFolder() (string, error) {
 	p, err := Executable()
 	if err != nil {
 		return "", err
 	}
-
-	return filepath.Dir(p), nil
+	folder, _ := filepath.Split(p)
+	return folder, nil
+}
+
+// Depricated. Same as Executable().
+func GetExePath() (exePath string, err error) {
+	return Executable()
 }

Godeps/_workspace/src/bitbucket.org/kardianos/osext/osext_plan9.go

@@ -5,16 +5,16 @@
 package osext
 
 import (
-	"os"
-	"strconv"
-	"syscall"
+    "syscall"
+    "os"
+    "strconv"
 )
 
 func executable() (string, error) {
-	f, err := os.Open("/proc/" + strconv.Itoa(os.Getpid()) + "/text")
-	if err != nil {
-		return "", err
-	}
-	defer f.Close()
-	return syscall.Fd2path(int(f.Fd()))
+    f, err := os.Open("/proc/" + strconv.Itoa(os.Getpid()) + "/text")
+    if err != nil {
+        return "", err
+    }
+    defer f.Close()
+    return syscall.Fd2path(int(f.Fd()))
 }

Godeps/_workspace/src/bitbucket.org/kardianos/osext/osext_procfs.go

@@ -2,35 +2,24 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build linux netbsd openbsd solaris dragonfly
+// +build linux netbsd openbsd
 
 package osext
 
 import (
 	"errors"
-	"fmt"
 	"os"
 	"runtime"
-	"strings"
 )
 
 func executable() (string, error) {
 	switch runtime.GOOS {
 	case "linux":
-		const deletedTag = " (deleted)"
-		execpath, err := os.Readlink("/proc/self/exe")
-		if err != nil {
-			return execpath, err
-		}
-		execpath = strings.TrimSuffix(execpath, deletedTag)
-		execpath = strings.TrimPrefix(execpath, deletedTag)
-		return execpath, nil
+		return os.Readlink("/proc/self/exe")
 	case "netbsd":
 		return os.Readlink("/proc/curproc/exe")
-	case "openbsd", "dragonfly":
+	case "openbsd":
 		return os.Readlink("/proc/curproc/file")
-	case "solaris":
-		return os.Readlink(fmt.Sprintf("/proc/%d/path/a.out", os.Getpid()))
 	}
 	return "", errors.New("ExecPath not implemented for " + runtime.GOOS)
 }

Godeps/_workspace/src/bitbucket.org/kardianos/osext/osext_test.go

@@ -0,0 +1,79 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin linux freebsd netbsd windows
+
+package osext
+
+import (
+	"fmt"
+	"os"
+	oexec "os/exec"
+	"path/filepath"
+	"runtime"
+	"testing"
+)
+
+const execPath_EnvVar = "OSTEST_OUTPUT_EXECPATH"
+
+func TestExecPath(t *testing.T) {
+	ep, err := Executable()
+	if err != nil {
+		t.Fatalf("ExecPath failed: %v", err)
+	}
+	// we want fn to be of the form "dir/prog"
+	dir := filepath.Dir(filepath.Dir(ep))
+	fn, err := filepath.Rel(dir, ep)
+	if err != nil {
+		t.Fatalf("filepath.Rel: %v", err)
+	}
+	cmd := &oexec.Cmd{}
+	// make child start with a relative program path
+	cmd.Dir = dir
+	cmd.Path = fn
+	// forge argv[0] for child, so that we can verify we could correctly
+	// get real path of the executable without influenced by argv[0].
+	cmd.Args = []string{"-", "-test.run=XXXX"}
+	cmd.Env = []string{fmt.Sprintf("%s=1", execPath_EnvVar)}
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("exec(self) failed: %v", err)
+	}
+	outs := string(out)
+	if !filepath.IsAbs(outs) {
+		t.Fatalf("Child returned %q, want an absolute path", out)
+	}
+	if !sameFile(outs, ep) {
+		t.Fatalf("Child returned %q, not the same file as %q", out, ep)
+	}
+}
+
+func sameFile(fn1, fn2 string) bool {
+	fi1, err := os.Stat(fn1)
+	if err != nil {
+		return false
+	}
+	fi2, err := os.Stat(fn2)
+	if err != nil {
+		return false
+	}
+	return os.SameFile(fi1, fi2)
+}
+
+func init() {
+	if e := os.Getenv(execPath_EnvVar); e != "" {
+		// first chdir to another path
+		dir := "/"
+		if runtime.GOOS == "windows" {
+			dir = filepath.VolumeName(".")
+		}
+		os.Chdir(dir)
+		if ep, err := Executable(); err != nil {
+			fmt.Fprint(os.Stderr, "ERROR: ", err)
+		} else {
+			fmt.Fprint(os.Stderr, ep)
+		}
+		os.Exit(0)
+	}
+}

Godeps/_workspace/src/code.google.com/p/go.crypto/bcrypt/bcrypt.go

@@ -8,11 +8,11 @@ package bcrypt
 
 // The code is a port of Provos and Mazières's C implementation.
 import (
+	"code.google.com/p/go.crypto/blowfish"
 	"crypto/rand"
 	"crypto/subtle"
 	"errors"
 	"fmt"
-	"golang.org/x/crypto/blowfish"
 	"io"
 	"strconv"
 )

Godeps/_workspace/src/code.google.com/p/go.text/transform/examples_test.go

@@ -8,8 +8,8 @@ import (
 	"fmt"
 	"unicode"
 
-	"golang.org/x/text/transform"
-	"golang.org/x/text/unicode/norm"
+	"code.google.com/p/go.text/transform"
+	"code.google.com/p/go.text/unicode/norm"
 )
 
 func ExampleRemoveFunc() {

Godeps/_workspace/src/code.google.com/p/go.text/transform/transform.go

@@ -55,17 +55,10 @@ type Transformer interface {
 	// either error may be returned. Other than the error conditions listed
 	// here, implementations are free to report other errors that arise.
 	Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error)
-
-	// Reset resets the state and allows a Transformer to be reused.
-	Reset()
 }
 
-// NopResetter can be embedded by implementations of Transformer to add a nop
-// Reset method.
-type NopResetter struct{}
-
-// Reset implements the Reset method of the Transformer interface.
-func (NopResetter) Reset() {}
+// TODO: Do we require that a Transformer be reusable if it returns a nil error
+// or do we always require a reset after use?  Is Reset mandatory or optional?
 
 // Reader wraps another io.Reader by transforming the bytes read.
 type Reader struct {
@@ -91,9 +84,8 @@ type Reader struct {
 const defaultBufSize = 4096
 
 // NewReader returns a new Reader that wraps r by transforming the bytes read
-// via t. It calls Reset on t.
+// via t.
 func NewReader(r io.Reader, t Transformer) *Reader {
-	t.Reset()
 	return &Reader{
 		r:   r,
 		t:   t,
@@ -178,9 +170,8 @@ type Writer struct {
 }
 
 // NewWriter returns a new Writer that wraps w by transforming the bytes written
-// via t. It calls Reset on t.
+// via t.
 func NewWriter(w io.Writer, t Transformer) *Writer {
-	t.Reset()
 	return &Writer{
 		w:   w,
 		t:   t,
@@ -256,7 +247,7 @@ func (w *Writer) Close() error {
 	return nil
 }
 
-type nop struct{ NopResetter }
+type nop struct{}
 
 func (nop) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	n := copy(dst, src)
@@ -266,7 +257,7 @@ func (nop) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	return n, n, err
 }
 
-type discard struct{ NopResetter }
+type discard struct{}
 
 func (discard) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	return 0, len(src), nil
@@ -336,16 +327,6 @@ func Chain(t ...Transformer) Transformer {
 	return c
 }
 
-// Reset resets the state of Chain. It calls Reset on all the Transformers.
-func (c *chain) Reset() {
-	for i, l := range c.link {
-		if l.t != nil {
-			l.t.Reset()
-		}
-		c.link[i].p, c.link[i].n = 0, 0
-	}
-}
-
 // Transform applies the transformers of c in sequence.
 func (c *chain) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	// Set up src and dst in the chain.
@@ -444,8 +425,6 @@ func RemoveFunc(f func(r rune) bool) Transformer {
 
 type removeF func(r rune) bool
 
-func (removeF) Reset() {}
-
 // Transform implements the Transformer interface.
 func (t removeF) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	for r, sz := rune(0), 0; len(src) > 0; src = src[sz:] {
@@ -457,7 +436,7 @@ func (t removeF) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err err
 
 			if sz == 1 {
 				// Invalid rune.
-				if !atEOF && !utf8.FullRune(src) {
+				if !atEOF && !utf8.FullRune(src[nSrc:]) {
 					err = ErrShortSrc
 					break
 				}
@@ -506,14 +485,12 @@ func grow(b []byte, n int) []byte {
 const initialBufSize = 128
 
 // String returns a string with the result of converting s[:n] using t, where
-// n <= len(s). If err == nil, n will be len(s). It calls Reset on t.
+// n <= len(s). If err == nil, n will be len(s).
 func String(t Transformer, s string) (result string, n int, err error) {
 	if s == "" {
 		return "", 0, nil
 	}
 
-	t.Reset()
-
 	// Allocate only once. Note that both dst and src escape when passed to
 	// Transform.
 	buf := [2 * initialBufSize]byte{}
@@ -594,9 +571,8 @@ func String(t Transformer, s string) (result string, n int, err error) {
 }
 
 // Bytes returns a new byte slice with the result of converting b[:n] using t,
-// where n <= len(b). If err == nil, n will be len(b). It calls Reset on t.
+// where n <= len(b). If err == nil, n will be len(b).
 func Bytes(t Transformer, b []byte) (result []byte, n int, err error) {
-	t.Reset()
 	dst := make([]byte, len(b))
 	pDst, pSrc := 0, 0
 	for {

Godeps/_workspace/src/code.google.com/p/go.text/transform/transform_test.go

@@ -16,7 +16,7 @@ import (
 	"unicode/utf8"
 )
 
-type lowerCaseASCII struct{ NopResetter }
+type lowerCaseASCII struct{}
 
 func (lowerCaseASCII) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	n := len(src)
@@ -34,7 +34,7 @@ func (lowerCaseASCII) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, er
 
 var errYouMentionedX = errors.New("you mentioned X")
 
-type dontMentionX struct{ NopResetter }
+type dontMentionX struct{}
 
 func (dontMentionX) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	n := len(src)
@@ -52,7 +52,7 @@ func (dontMentionX) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err
 
 // doublerAtEOF is a strange Transformer that transforms "this" to "tthhiiss",
 // but only if atEOF is true.
-type doublerAtEOF struct{ NopResetter }
+type doublerAtEOF struct{}
 
 func (doublerAtEOF) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	if !atEOF {
@@ -71,7 +71,7 @@ func (doublerAtEOF) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err
 // rleDecode and rleEncode implement a toy run-length encoding: "aabbbbbbbbbb"
 // is encoded as "2a10b". The decoding is assumed to not contain any numbers.
 
-type rleDecode struct{ NopResetter }
+type rleDecode struct{}
 
 func (rleDecode) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 loop:
@@ -104,8 +104,6 @@ loop:
 }
 
 type rleEncode struct {
-	NopResetter
-
 	// allowStutter means that "xxxxxxxx" can be encoded as "5x3x"
 	// instead of always as "8x".
 	allowStutter bool
@@ -138,10 +136,6 @@ func (e rleEncode) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err e
 // trickler consumes all input bytes, but writes a single byte at a time to dst.
 type trickler []byte
 
-func (t *trickler) Reset() {
-	*t = nil
-}
-
 func (t *trickler) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	*t = append(*t, src...)
 	if len(*t) == 0 {
@@ -163,9 +157,6 @@ func (t *trickler) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err e
 // to have some tolerance as long as progress can be detected.
 type delayedTrickler []byte
 
-func (t *delayedTrickler) Reset() {
-	*t = nil
-}
 func (t *delayedTrickler) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	if len(*t) > 0 && len(dst) > 0 {
 		dst[0] = (*t)[0]
@@ -456,6 +447,7 @@ var testCases = []testCase{
 
 func TestReader(t *testing.T) {
 	for _, tc := range testCases {
+		reset(tc.t)
 		r := NewReader(strings.NewReader(tc.src), tc.t)
 		// Differently sized dst and src buffers are not part of the
 		// exported API. We override them manually.
@@ -469,6 +461,13 @@ func TestReader(t *testing.T) {
 	}
 }
 
+func reset(t Transformer) {
+	var dst [128]byte
+	for err := ErrShortDst; err != nil; {
+		_, _, err = t.Transform(dst[:], nil, true)
+	}
+}
+
 func TestWriter(t *testing.T) {
 	tests := append(testCases, chainTests()...)
 	for _, tc := range tests {
@@ -478,6 +477,7 @@ func TestWriter(t *testing.T) {
 		}
 		for _, sz := range sizes {
 			bb := &bytes.Buffer{}
+			reset(tc.t)
 			w := NewWriter(bb, tc.t)
 			// Differently sized dst and src buffers are not part of the
 			// exported API. We override them manually.
@@ -735,7 +735,7 @@ func chainTests() []testCase {
 }
 
 func doTransform(tc testCase) (res string, iter int, err error) {
-	tc.t.Reset()
+	reset(tc.t)
 	dst := make([]byte, tc.dstSize)
 	out, in := make([]byte, 0, 2*len(tc.src)), []byte(tc.src)
 	for {
@@ -884,14 +884,6 @@ func TestRemoveFunc(t *testing.T) {
 			wantIter: 2,
 		},
 
-		{
-			// Test a long buffer greater than the internal buffer size
-			src:      "hello\xcc\xcc\xccworld",
-			srcSize:  13,
-			wantStr:  "hello\uFFFD\uFFFD\uFFFDworld",
-			wantIter: 1,
-		},
-
 		{
 			src:     "\u2345",
 			dstSize: 2,
@@ -955,6 +947,7 @@ func testString(t *testing.T, f func(Transformer, string) (string, int, error))
 			// that depend on a specific buffer size being set.
 			continue
 		}
+		reset(tt.t)
 		if tt.wantErr == ErrShortDst || tt.wantErr == ErrShortSrc {
 			// The result string will be different.
 			continue

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/Makefile

@@ -0,0 +1,30 @@
+# Copyright 2011 The Go Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+maketables: maketables.go triegen.go
+	go build $^
+
+maketesttables: maketesttables.go triegen.go
+	go build $^
+
+normregtest: normregtest.go
+	go build $^
+
+tables:	maketables
+	./maketables > tables.go
+	gofmt -w tables.go
+
+trietesttables: maketesttables
+	./maketesttables > triedata_test.go
+	gofmt -w triedata_test.go
+
+# Downloads from www.unicode.org, so not part
+# of standard test scripts.
+test: testtables regtest
+
+testtables: maketables
+	./maketables -test > data_test.go && go test -tags=test
+
+regtest: normregtest
+	./normregtest

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/example_iter_test.go

@@ -9,7 +9,7 @@ import (
 	"fmt"
 	"unicode/utf8"
 
-	"golang.org/x/text/unicode/norm"
+	"code.google.com/p/go.text/unicode/norm"
 )
 
 // EqualSimple uses a norm.Iter to compare two non-normalized

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/forminfo.go

@@ -201,17 +201,17 @@ func lookupInfoNFKC(b input, i int) Properties {
 // Properties returns properties for the first rune in s.
 func (f Form) Properties(s []byte) Properties {
 	if f == NFC || f == NFD {
-		return compInfo(nfcData.lookup(s))
+		return compInfo(nfcTrie.lookup(s))
 	}
-	return compInfo(nfkcData.lookup(s))
+	return compInfo(nfkcTrie.lookup(s))
 }
 
 // PropertiesString returns properties for the first rune in s.
 func (f Form) PropertiesString(s string) Properties {
 	if f == NFC || f == NFD {
-		return compInfo(nfcData.lookupString(s))
+		return compInfo(nfcTrie.lookupString(s))
 	}
-	return compInfo(nfkcData.lookupString(s))
+	return compInfo(nfkcTrie.lookupString(s))
 }
 
 // compInfo converts the information contained in v and sz

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/input.go

@@ -77,16 +77,16 @@ func (in *input) copySlice(buf []byte, b, e int) int {
 
 func (in *input) charinfoNFC(p int) (uint16, int) {
 	if in.bytes == nil {
-		return nfcData.lookupString(in.str[p:])
+		return nfcTrie.lookupString(in.str[p:])
 	}
-	return nfcData.lookup(in.bytes[p:])
+	return nfcTrie.lookup(in.bytes[p:])
 }
 
 func (in *input) charinfoNFKC(p int) (uint16, int) {
 	if in.bytes == nil {
-		return nfkcData.lookupString(in.str[p:])
+		return nfkcTrie.lookupString(in.str[p:])
 	}
-	return nfkcData.lookup(in.bytes[p:])
+	return nfkcTrie.lookup(in.bytes[p:])
 }
 
 func (in *input) hangul(p int) (r rune) {

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/iter.go

@@ -9,8 +9,6 @@ import (
 	"unicode/utf8"
 )
 
-// MaxSegmentSize is the maximum size of a byte buffer needed to consider any
-// sequence of starter and non-starter runes for the purpose of normalization.
 const MaxSegmentSize = maxByteBufferSize
 
 // An Iter iterates over a string or byte slice, while normalizing it

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/maketables.go

@@ -16,17 +16,19 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net/http"
+	"os"
+	"regexp"
 	"sort"
 	"strconv"
 	"strings"
+	"unicode"
 
-	"golang.org/x/text/internal/gen"
-	"golang.org/x/text/internal/triegen"
-	"golang.org/x/text/internal/ucd"
+	"code.google.com/p/go.text/internal/ucd"
 )
 
 func main() {
-	gen.Init()
+	flag.Parse()
 	loadUnicodeData()
 	compactCCC()
 	loadCompositionExclusions()
@@ -43,18 +45,24 @@ func main() {
 	}
 }
 
-var (
-	tablelist = flag.String("tables",
-		"all",
-		"comma-separated list of which tables to generate; "+
-			"can be 'decomp', 'recomp', 'info' and 'all'")
-	test = flag.Bool("test",
-		false,
-		"test existing tables against DerivedNormalizationProps and generate test data for regression testing")
-	verbose = flag.Bool("verbose",
-		false,
-		"write data to stdout as it is parsed")
-)
+var url = flag.String("url",
+	"http://www.unicode.org/Public/"+unicode.Version+"/ucd/",
+	"URL of Unicode database directory")
+var tablelist = flag.String("tables",
+	"all",
+	"comma-separated list of which tables to generate; "+
+		"can be 'decomp', 'recomp', 'info' and 'all'")
+var test = flag.Bool("test",
+	false,
+	"test existing tables against DerivedNormalizationProps and generate test data for regression testing")
+var verbose = flag.Bool("verbose",
+	false,
+	"write data to stdout as it is parsed")
+var localFiles = flag.Bool("local",
+	false,
+	"data files have been copied to the current directory; for debugging only")
+
+var logger = log.New(os.Stderr, "", log.Lshortfile)
 
 const MaxChar = 0x10FFFF // anything above this shouldn't exist
 
@@ -180,6 +188,27 @@ func (f FormInfo) String() string {
 
 type Decomposition []rune
 
+func openReader(file string) (input io.ReadCloser) {
+	if *localFiles {
+		f, err := os.Open(file)
+		if err != nil {
+			logger.Fatal(err)
+		}
+		input = f
+	} else {
+		path := *url + file
+		resp, err := http.Get(path)
+		if err != nil {
+			logger.Fatal(err)
+		}
+		if resp.StatusCode != 200 {
+			logger.Fatal("bad GET status for "+file, resp.Status)
+		}
+		input = resp.Body
+	}
+	return
+}
+
 func parseDecomposition(s string, skipfirst bool) (a []rune, err error) {
 	decomp := strings.Split(s, " ")
 	if len(decomp) > 0 && skipfirst {
@@ -196,7 +225,7 @@ func parseDecomposition(s string, skipfirst bool) (a []rune, err error) {
 }
 
 func loadUnicodeData() {
-	f := gen.OpenUCDFile("UnicodeData.txt")
+	f := openReader("UnicodeData.txt")
 	defer f.Close()
 	p := ucd.New(f)
 	for p.Next() {
@@ -212,7 +241,7 @@ func loadUnicodeData() {
 			if len(decmap) > 0 {
 				exp, err = parseDecomposition(decmap, true)
 				if err != nil {
-					log.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err)
+					logger.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err)
 				}
 				isCompat = true
 			}
@@ -231,7 +260,7 @@ func loadUnicodeData() {
 		}
 	}
 	if err := p.Err(); err != nil {
-		log.Fatal(err)
+		logger.Fatal(err)
 	}
 }
 
@@ -266,18 +295,18 @@ func compactCCC() {
 // 0958    # ...
 // See http://unicode.org/reports/tr44/ for full explanation
 func loadCompositionExclusions() {
-	f := gen.OpenUCDFile("CompositionExclusions.txt")
+	f := openReader("CompositionExclusions.txt")
 	defer f.Close()
 	p := ucd.New(f)
 	for p.Next() {
 		c := &chars[p.Rune(0)]
 		if c.excludeInComp {
-			log.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint)
+			logger.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint)
 		}
 		c.excludeInComp = true
 	}
 	if e := p.Err(); e != nil {
-		log.Fatal(e)
+		logger.Fatal(e)
 	}
 }
 
@@ -471,22 +500,29 @@ func computeNonStarterCounts() {
 		if exp := c.forms[FCompatibility].expandedDecomp; len(exp) > 0 {
 			runes = exp
 		}
-		// We consider runes that combine backwards to be non-starters for the
-		// purpose of Stream-Safe Text Processing.
 		for _, r := range runes {
-			if cr := &chars[r]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward {
+			if chars[r].ccc == 0 {
 				break
 			}
 			c.nLeadingNonStarters++
 		}
 		for i := len(runes) - 1; i >= 0; i-- {
-			if cr := &chars[runes[i]]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward {
+			if chars[runes[i]].ccc == 0 {
 				break
 			}
 			c.nTrailingNonStarters++
 		}
-		if c.nTrailingNonStarters > 3 {
-			log.Fatalf("%U: Decomposition with more than 3 (%d) trailing modifiers (%U)", i, c.nTrailingNonStarters, runes)
+
+		// We consider runes that combine backwards to be non-starters for the
+		// purpose of Stream-Safe Text Processing.
+		for _, f := range c.forms {
+			if c.ccc == 0 && f.combinesBackward {
+				if len(c.forms[FCompatibility].expandedDecomp) > 0 {
+					log.Fatalf("%U: CCC==0 modifier with an expansion is not supported.", i)
+				}
+				c.nTrailingNonStarters = 1
+				c.nLeadingNonStarters = 1
+			}
 		}
 
 		if isHangul(rune(i)) {
@@ -505,19 +541,19 @@ func computeNonStarterCounts() {
 	}
 }
 
-func printBytes(w io.Writer, b []byte, name string) {
-	fmt.Fprintf(w, "// %s: %d bytes\n", name, len(b))
-	fmt.Fprintf(w, "var %s = [...]byte {", name)
+func printBytes(b []byte, name string) {
+	fmt.Printf("// %s: %d bytes\n", name, len(b))
+	fmt.Printf("var %s = [...]byte {", name)
 	for i, c := range b {
 		switch {
 		case i%64 == 0:
-			fmt.Fprintf(w, "\n// Bytes %x - %x\n", i, i+63)
+			fmt.Printf("\n// Bytes %x - %x\n", i, i+63)
 		case i%8 == 0:
-			fmt.Fprintf(w, "\n")
+			fmt.Printf("\n")
 		}
-		fmt.Fprintf(w, "0x%.2X, ", c)
+		fmt.Printf("0x%.2X, ", c)
 	}
-	fmt.Fprint(w, "\n}\n\n")
+	fmt.Print("\n}\n\n")
 }
 
 // See forminfo.go for format.
@@ -573,13 +609,13 @@ func (m *decompSet) insert(key int, s string) {
 	m[key][s] = true
 }
 
-func printCharInfoTables(w io.Writer) int {
+func printCharInfoTables() int {
 	mkstr := func(r rune, f *FormInfo) (int, string) {
 		d := f.expandedDecomp
 		s := string([]rune(d))
 		if max := 1 << 6; len(s) >= max {
 			const msg = "%U: too many bytes in decomposition: %d >= %d"
-			log.Fatalf(msg, r, len(s), max)
+			logger.Fatalf(msg, r, len(s), max)
 		}
 		head := uint8(len(s))
 		if f.quickCheck[MComposed] != QCYes {
@@ -594,11 +630,11 @@ func printCharInfoTables(w io.Writer) int {
 		tccc := ccc(d[len(d)-1])
 		cc := ccc(r)
 		if cc != 0 && lccc == 0 && tccc == 0 {
-			log.Fatalf("%U: trailing and leading ccc are 0 for non-zero ccc %d", r, cc)
+			logger.Fatalf("%U: trailing and leading ccc are 0 for non-zero ccc %d", r, cc)
 		}
 		if tccc < lccc && lccc != 0 {
 			const msg = "%U: lccc (%d) must be <= tcc (%d)"
-			log.Fatalf(msg, r, lccc, tccc)
+			logger.Fatalf(msg, r, lccc, tccc)
 		}
 		index := normalDecomp
 		nTrail := chars[r].nTrailingNonStarters
@@ -615,13 +651,13 @@ func printCharInfoTables(w io.Writer) int {
 			if lccc > 0 {
 				s += string([]byte{lccc})
 				if index == firstCCC {
-					log.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r)
+					logger.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r)
 				}
 				index = firstLeadingCCC
 			}
 			if cc != lccc {
 				if cc != 0 {
-					log.Fatalf("%U: for lccc != ccc, expected ccc to be 0; was %d", r, cc)
+					logger.Fatalf("%U: for lccc != ccc, expected ccc to be 0; was %d", r, cc)
 				}
 				index = firstCCCZeroExcept
 			}
@@ -643,7 +679,7 @@ func printCharInfoTables(w io.Writer) int {
 				continue
 			}
 			if f.combinesBackward {
-				log.Fatalf("%U: combinesBackward and decompose", c.codePoint)
+				logger.Fatalf("%U: combinesBackward and decompose", c.codePoint)
 			}
 			index, s := mkstr(c.codePoint, &f)
 			decompSet.insert(index, s)
@@ -654,7 +690,7 @@ func printCharInfoTables(w io.Writer) int {
 	size := 0
 	positionMap := make(map[string]uint16)
 	decompositions.WriteString("\000")
-	fmt.Fprintln(w, "const (")
+	fmt.Println("const (")
 	for i, m := range decompSet {
 		sa := []string{}
 		for s := range m {
@@ -667,44 +703,39 @@ func printCharInfoTables(w io.Writer) int {
 			positionMap[s] = uint16(p)
 		}
 		if cname[i] != "" {
-			fmt.Fprintf(w, "%s = 0x%X\n", cname[i], decompositions.Len())
+			fmt.Printf("%s = 0x%X\n", cname[i], decompositions.Len())
 		}
 	}
-	fmt.Fprintln(w, "maxDecomp = 0x8000")
-	fmt.Fprintln(w, ")")
+	fmt.Println("maxDecomp = 0x8000")
+	fmt.Println(")")
 	b := decompositions.Bytes()
-	printBytes(w, b, "decomps")
+	printBytes(b, "decomps")
 	size += len(b)
 
 	varnames := []string{"nfc", "nfkc"}
 	for i := 0; i < FNumberOfFormTypes; i++ {
-		trie := triegen.NewTrie(varnames[i])
-
+		trie := newNode()
 		for r, c := range chars {
 			f := c.forms[i]
 			d := f.expandedDecomp
 			if len(d) != 0 {
 				_, key := mkstr(c.codePoint, &f)
-				trie.Insert(rune(r), uint64(positionMap[key]))
+				trie.insert(rune(r), positionMap[key])
 				if c.ccc != ccc(d[0]) {
 					// We assume the lead ccc of a decomposition !=0 in this case.
 					if ccc(d[0]) == 0 {
-						log.Fatalf("Expected leading CCC to be non-zero; ccc is %d", c.ccc)
+						logger.Fatalf("Expected leading CCC to be non-zero; ccc is %d", c.ccc)
 					}
 				}
 			} else if c.nLeadingNonStarters > 0 && len(f.expandedDecomp) == 0 && c.ccc == 0 && !f.combinesBackward {
 				// Handle cases where it can't be detected that the nLead should be equal
 				// to nTrail.
-				trie.Insert(c.codePoint, uint64(positionMap[nLeadStr]))
+				trie.insert(c.codePoint, positionMap[nLeadStr])
 			} else if v := makeEntry(&f, &c)<<8 | uint16(c.ccc); v != 0 {
-				trie.Insert(c.codePoint, uint64(0x8000|v))
+				trie.insert(c.codePoint, 0x8000|v)
 			}
 		}
-		sz, err := trie.Gen(w, triegen.Compact(&normCompacter{name: varnames[i]}))
-		if err != nil {
-			log.Fatal(err)
-		}
-		size += sz
+		size += trie.printTables(varnames[i])
 	}
 	return size
 }
@@ -718,9 +749,30 @@ func contains(sa []string, s string) bool {
 	return false
 }
 
-func makeTables() {
-	w := &bytes.Buffer{}
+// Extract the version number from the URL.
+func version() string {
+	// From http://www.unicode.org/standard/versions/#Version_Numbering:
+	// for the later Unicode versions, data files are located in
+	// versioned directories.
+	fields := strings.Split(*url, "/")
+	for _, f := range fields {
+		if match, _ := regexp.MatchString(`[0-9]\.[0-9]\.[0-9]`, f); match {
+			return f
+		}
+	}
+	logger.Fatal("unknown version")
+	return "Unknown"
+}
 
+const fileHeader = `// Generated by running
+//	maketables --tables=%s --url=%s
+// DO NOT EDIT
+
+package norm
+
+`
+
+func makeTables() {
 	size := 0
 	if *tablelist == "" {
 		return
@@ -729,6 +781,7 @@ func makeTables() {
 	if *tablelist == "all" {
 		list = []string{"recomp", "info"}
 	}
+	fmt.Printf(fileHeader, *tablelist, *url)
 
 	// Compute maximum decomposition size.
 	max := 0
@@ -738,30 +791,30 @@ func makeTables() {
 		}
 	}
 
-	fmt.Fprintln(w, "const (")
-	fmt.Fprintln(w, "\t// Version is the Unicode edition from which the tables are derived.")
-	fmt.Fprintf(w, "\tVersion = %q\n", gen.UnicodeVersion())
-	fmt.Fprintln(w)
-	fmt.Fprintln(w, "\t// MaxTransformChunkSize indicates the maximum number of bytes that Transform")
-	fmt.Fprintln(w, "\t// may need to write atomically for any Form. Making a destination buffer at")
-	fmt.Fprintln(w, "\t// least this size ensures that Transform can always make progress and that")
-	fmt.Fprintln(w, "\t// the user does not need to grow the buffer on an ErrShortDst.")
-	fmt.Fprintf(w, "\tMaxTransformChunkSize = %d+maxNonStarters*4\n", len(string(0x034F))+max)
-	fmt.Fprintln(w, ")\n")
+	fmt.Println("const (")
+	fmt.Println("\t// Version is the Unicode edition from which the tables are derived.")
+	fmt.Printf("\tVersion = %q\n", version())
+	fmt.Println()
+	fmt.Println("\t// MaxTransformChunkSize indicates the maximum number of bytes that Transform")
+	fmt.Println("\t// may need to write atomically for any Form. Making a destination buffer at")
+	fmt.Println("\t// least this size ensures that Transform can always make progress and that")
+	fmt.Println("\t// the user does not need to grow the buffer on an ErrShortDst.")
+	fmt.Printf("\tMaxTransformChunkSize = %d+maxNonStarters*4\n", len(string(0x034F))+max)
+	fmt.Println(")\n")
 
 	// Print the CCC remap table.
 	size += len(cccMap)
-	fmt.Fprintf(w, "var ccc = [%d]uint8{", len(cccMap))
+	fmt.Printf("var ccc = [%d]uint8{", len(cccMap))
 	for i := 0; i < len(cccMap); i++ {
 		if i%8 == 0 {
-			fmt.Fprintln(w)
+			fmt.Println()
 		}
-		fmt.Fprintf(w, "%3d, ", cccMap[uint8(i)])
+		fmt.Printf("%3d, ", cccMap[uint8(i)])
 	}
-	fmt.Fprintln(w, "\n}\n")
+	fmt.Println("\n}\n")
 
 	if contains(list, "info") {
-		size += printCharInfoTables(w)
+		size += printCharInfoTables()
 	}
 
 	if contains(list, "recomp") {
@@ -783,21 +836,20 @@ func makeTables() {
 		}
 		sz := nrentries * 8
 		size += sz
-		fmt.Fprintf(w, "// recompMap: %d bytes (entries only)\n", sz)
-		fmt.Fprintln(w, "var recompMap = map[uint32]rune{")
+		fmt.Printf("// recompMap: %d bytes (entries only)\n", sz)
+		fmt.Println("var recompMap = map[uint32]rune{")
 		for i, c := range chars {
 			f := c.forms[FCanonical]
 			d := f.decomp
 			if !f.isOneWay && len(d) > 0 {
 				key := uint32(uint16(d[0]))<<16 + uint32(uint16(d[1]))
-				fmt.Fprintf(w, "0x%.8X: 0x%.4X,\n", key, i)
+				fmt.Printf("0x%.8X: 0x%.4X,\n", key, i)
 			}
 		}
-		fmt.Fprintf(w, "}\n\n")
+		fmt.Printf("}\n\n")
 	}
 
-	fmt.Fprintf(w, "// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size)
-	gen.WriteGoFile("tables.go", "norm", w.Bytes())
+	fmt.Printf("// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size)
 }
 
 func printChars() {
@@ -832,16 +884,10 @@ func verifyComputed() {
 				continue
 			}
 			if a, b := c.nLeadingNonStarters > 0, (c.ccc > 0 || f.combinesBackward); a != b {
-				// We accept these runes to be treated differently (it only affects
-				// segment breaking in iteration, most likely on improper use), but
+				// We accept these two runes to be treated differently (it only affects
+				// segment breaking in iteration, most likely on inproper use), but
 				// reconsider if more characters are added.
-				// U+FF9E HALFWIDTH KATAKANA VOICED SOUND MARK;Lm;0;L;<narrow> 3099;;;;N;;;;;
-				// U+FF9F HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK;Lm;0;L;<narrow> 309A;;;;N;;;;;
-				// U+3133 HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<compat> 11AA;;;;N;HANGUL LETTER GIYEOG SIOS;;;;
-				// U+318E HANGUL LETTER ARAEAE;Lo;0;L;<compat> 11A1;;;;N;HANGUL LETTER ALAE AE;;;;
-				// U+FFA3 HALFWIDTH HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<narrow> 3133;;;;N;HALFWIDTH HANGUL LETTER GIYEOG SIOS;;;;
-				// U+FFDC HALFWIDTH HANGUL LETTER I;Lo;0;L;<narrow> 3163;;;;N;;;;;
-				if i != 0xFF9E && i != 0xFF9F && !(0x3133 <= i && i <= 0x318E) && !(0xFFA3 <= i && i <= 0xFFDC) {
+				if i != 0xFF9E && i != 0xFF9F {
 					log.Fatalf("%U: nLead was %v; want %v", i, a, b)
 				}
 			}
@@ -849,7 +895,7 @@ func verifyComputed() {
 		nfc := c.forms[FCanonical]
 		nfkc := c.forms[FCompatibility]
 		if nfc.combinesBackward != nfkc.combinesBackward {
-			log.Fatalf("%U: Cannot combine combinesBackward\n", c.codePoint)
+			logger.Fatalf("%U: Cannot combine combinesBackward\n", c.codePoint)
 		}
 	}
 }
@@ -861,7 +907,7 @@ func verifyComputed() {
 // 0374          ; NFD_QC; N # ...
 // See http://unicode.org/reports/tr44/ for full explanation
 func testDerived() {
-	f := gen.OpenUCDFile("DerivedNormalizationProps.txt")
+	f := openReader("DerivedNormalizationProps.txt")
 	defer f.Close()
 	p := ucd.New(f)
 	for p.Next() {
@@ -894,12 +940,12 @@ func testDerived() {
 			log.Fatalf(`Unexpected quick check value "%s"`, p.String(2))
 		}
 		if got := c.forms[ftype].quickCheck[mode]; got != qr {
-			log.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr)
+			logger.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr)
 		}
 		c.forms[ftype].verified[mode] = true
 	}
 	if err := p.Err(); err != nil {
-		log.Fatal(err)
+		logger.Fatal(err)
 	}
 	// Any unspecified value must be QCYes. Verify this.
 	for i, c := range chars {
@@ -907,14 +953,20 @@ func testDerived() {
 			for k, qr := range fd.quickCheck {
 				if !fd.verified[k] && qr != QCYes {
 					m := "%U: FAIL F:%d M:%d (was %v need Yes) %s\n"
-					log.Printf(m, i, j, k, qr, c.name)
+					logger.Printf(m, i, j, k, qr, c.name)
 				}
 			}
 		}
 	}
 }
 
-var testHeader = `const (
+var testHeader = `// Generated by running
+//   maketables --test --url=%s
+// +build test
+
+package norm
+
+const (
 	Yes = iota
 	No
 	Maybe
@@ -952,10 +1004,8 @@ func printTestdata() {
 		nTrail uint8
 		f      string
 	}
-
 	last := lastInfo{}
-	w := &bytes.Buffer{}
-	fmt.Fprintf(w, testHeader)
+	fmt.Printf(testHeader, *url)
 	for r, c := range chars {
 		f := c.forms[FCanonical]
 		qc, cf, d := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp)
@@ -969,10 +1019,9 @@ func printTestdata() {
 		}
 		current := lastInfo{c.ccc, c.nLeadingNonStarters, c.nTrailingNonStarters, s}
 		if last != current {
-			fmt.Fprintf(w, "\t{0x%x, %d, %d, %d, %s},\n", r, c.origCCC, c.nLeadingNonStarters, c.nTrailingNonStarters, s)
+			fmt.Printf("\t{0x%x, %d, %d, %d, %s},\n", r, c.origCCC, c.nLeadingNonStarters, c.nTrailingNonStarters, s)
 			last = current
 		}
 	}
-	fmt.Fprintln(w, "}")
-	gen.WriteGoFile("data_test.go", "norm", w.Bytes())
+	fmt.Println("}")
 }

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/maketesttables.go

@@ -0,0 +1,45 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// Generate test data for trie code.
+
+package main
+
+import (
+	"fmt"
+)
+
+func main() {
+	printTestTables()
+}
+
+// We take the smallest, largest and an arbitrary value for each
+// of the UTF-8 sequence lengths.
+var testRunes = []rune{
+	0x01, 0x0C, 0x7F, // 1-byte sequences
+	0x80, 0x100, 0x7FF, // 2-byte sequences
+	0x800, 0x999, 0xFFFF, // 3-byte sequences
+	0x10000, 0x10101, 0x10FFFF, // 4-byte sequences
+	0x200, 0x201, 0x202, 0x210, 0x215, // five entries in one sparse block
+}
+
+const fileHeader = `// Generated by running
+//	maketesttables
+// DO NOT EDIT
+
+package norm
+
+`
+
+func printTestTables() {
+	fmt.Print(fileHeader)
+	fmt.Printf("var testRunes = %#v\n\n", testRunes)
+	t := newNode()
+	for i, r := range testRunes {
+		t.insert(r, uint16(i))
+	}
+	t.printTables("testdata")
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/normalize.go

@@ -2,9 +2,6 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-//go:generate go run maketables.go triegen.go
-//go:generate go run maketables.go triegen.go -test
-
 // Package norm contains types and functions for normalizing Unicode strings.
 package norm
 

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/normalize_test.go

@@ -113,25 +113,7 @@ var decomposeSegmentTests = []PositionTest{
 	{"\u00C0b", 2, "A\u0300"},
 	// long
 	{grave(31), 60, grave(30) + cgj},
-	{"a" + grave(31), 61, "a" + grave(30) + cgj},
-
-	// Stability tests: see http://www.unicode.org/review/pr-29.html.
-	// U+0300 COMBINING GRAVE ACCENT;Mn;230;NSM;;;;;N;NON-SPACING GRAVE;;;;
-	// U+0B47 ORIYA VOWEL SIGN E;Mc;0;L;;;;;N;;;;;
-	// U+0B3E ORIYA VOWEL SIGN AA;Mc;0;L;;;;;N;;;;;
-	// U+1100 HANGUL CHOSEONG KIYEOK;Lo;0;L;;;;;N;;;;;
-	// U+1161 HANGUL JUNGSEONG A;Lo;0;L;;;;;N;;;;;
-	{"\u0B47\u0300\u0B3E", 8, "\u0B47\u0300\u0B3E"},
-	{"\u1100\u0300\u1161", 8, "\u1100\u0300\u1161"},
-	{"\u0B47\u0B3E", 6, "\u0B47\u0B3E"},
-	{"\u1100\u1161", 6, "\u1100\u1161"},
-
-	// U+04DA MALAYALAM VOWEL SIGN O;Mc;0;L;0D46 0D3E;;;;N;;;;;
-	// Sequence of decomposing characters that are starters and modifiers.
-	{"\u0d4a" + strings.Repeat("\u0d3e", 31), 90, "\u0d46" + strings.Repeat("\u0d3e", 30) + cgj},
-
 	{grave(30), 60, grave(30)},
-	// U+FF9E is a starter, but decomposes to U+3099, which is not.
 	{grave(30) + "\uff9e", 60, grave(30) + cgj},
 	// ends with incomplete UTF-8 encoding
 	{"\xCC", 0, ""},
@@ -570,44 +552,6 @@ var appendTestsNFC = []AppendTest{
 		"a" + rep(0x0305, maxNonStarters+4) + "\u0316",
 		"a" + rep(0x0305, maxNonStarters) + cgj + "\u0316" + rep(0x305, 4),
 	},
-
-	{ // Combine across non-blocking non-starters.
-		// U+0327 COMBINING CEDILLA;Mn;202;NSM;;;;;N;NON-SPACING CEDILLA;;;;
-		// U+0325 COMBINING RING BELOW;Mn;220;NSM;;;;;N;NON-SPACING RING BELOW;;;;
-		"", "a\u0327\u0325", "\u1e01\u0327",
-	},
-
-	{ // Jamo V+T does not combine.
-		"",
-		"\u1161\u11a8",
-		"\u1161\u11a8",
-	},
-
-	// Stability tests: see http://www.unicode.org/review/pr-29.html.
-	{"", "\u0b47\u0300\u0b3e", "\u0b47\u0300\u0b3e"},
-	{"", "\u1100\u0300\u1161", "\u1100\u0300\u1161"},
-	{"", "\u0b47\u0b3e", "\u0b4b"},
-	{"", "\u1100\u1161", "\uac00"},
-
-	// U+04DA MALAYALAM VOWEL SIGN O;Mc;0;L;0D46 0D3E;;;;N;;;;;
-	{ // 0d4a starts a new segment.
-		"",
-		"\u0d4a" + strings.Repeat("\u0d3e", 15) + "\u0d4a" + strings.Repeat("\u0d3e", 15),
-		"\u0d4a" + strings.Repeat("\u0d3e", 15) + "\u0d4a" + strings.Repeat("\u0d3e", 15),
-	},
-
-	{ // Split combining characters.
-		// TODO: don't insert CGJ before starters.
-		"",
-		"\u0d46" + strings.Repeat("\u0d3e", 31),
-		"\u0d4a" + strings.Repeat("\u0d3e", 29) + cgj + "\u0d3e",
-	},
-
-	{ // Split combining characters.
-		"",
-		"\u0d4a" + strings.Repeat("\u0d3e", 30),
-		"\u0d4a" + strings.Repeat("\u0d3e", 29) + cgj + "\u0d3e",
-	},
 }
 
 var appendTestsNFD = []AppendTest{

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/normregtest.go

@@ -2,37 +2,52 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package norm
+// +build ignore
+
+package main
 
 import (
 	"bufio"
 	"bytes"
 	"flag"
 	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"path"
 	"regexp"
 	"runtime"
 	"strconv"
 	"strings"
-	"sync"
-	"testing"
 	"time"
+	"unicode"
 	"unicode/utf8"
 
-	"golang.org/x/text/internal/gen"
+	"code.google.com/p/go.text/unicode/norm"
 )
 
-var long = flag.Bool("long", false,
-	"run time-consuming tests, such as tests that fetch data online")
-
-var once sync.Once
-
-func skipShort(t *testing.T) {
-	if !gen.IsLocal() && !*long {
-		t.Skip("skipping test to prevent downloading; to run use -long or use -local to specify a local source")
+func main() {
+	flag.Parse()
+	loadTestData()
+	CharacterByCharacterTests()
+	StandardTests()
+	PerformanceTest()
+	if errorCount == 0 {
+		fmt.Println("PASS")
 	}
-	once.Do(func() { loadTestData(t) })
 }
 
+const file = "NormalizationTest.txt"
+
+var url = flag.String("url",
+	"http://www.unicode.org/Public/"+unicode.Version+"/ucd/"+file,
+	"URL of Unicode database directory")
+var localFiles = flag.Bool("local",
+	false,
+	"data files have been copied to the current directory; for debugging only")
+
+var logger = log.New(os.Stderr, "", log.Lshortfile)
+
 // This regression test runs the test set in NormalizationTest.txt
 // (taken from http://www.unicode.org/Public/<unicode.Version>/ucd/).
 //
@@ -109,8 +124,22 @@ var testRe = regexp.MustCompile(`^` + strings.Repeat(`([\dA-F ]+);`, 5) + ` # (.
 var counter int
 
 // Load the data form NormalizationTest.txt
-func loadTestData(t *testing.T) {
-	f := gen.OpenUCDFile("NormalizationTest.txt")
+func loadTestData() {
+	if *localFiles {
+		pwd, _ := os.Getwd()
+		*url = "file://" + path.Join(pwd, file)
+	}
+	t := &http.Transport{}
+	t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/")))
+	c := &http.Client{Transport: t}
+	resp, err := c.Get(*url)
+	if err != nil {
+		logger.Fatal(err)
+	}
+	if resp.StatusCode != 200 {
+		logger.Fatal("bad GET status for "+file, resp.Status)
+	}
+	f := resp.Body
 	defer f.Close()
 	scanner := bufio.NewScanner(f)
 	for scanner.Scan() {
@@ -121,11 +150,11 @@ func loadTestData(t *testing.T) {
 		m := partRe.FindStringSubmatch(line)
 		if m != nil {
 			if len(m) < 3 {
-				t.Fatal("Failed to parse Part: ", line)
+				logger.Fatal("Failed to parse Part: ", line)
 			}
 			i, err := strconv.Atoi(m[1])
 			if err != nil {
-				t.Fatal(err)
+				logger.Fatal(err)
 			}
 			name := m[2]
 			part = append(part, Part{name: name[:len(name)-1], number: i})
@@ -133,7 +162,7 @@ func loadTestData(t *testing.T) {
 		}
 		m = testRe.FindStringSubmatch(line)
 		if m == nil || len(m) < 7 {
-			t.Fatalf(`Failed to parse: "%s" result: %#v`, line, m)
+			logger.Fatalf(`Failed to parse: "%s" result: %#v`, line, m)
 		}
 		test := Test{name: m[6], partnr: len(part) - 1, number: counter}
 		counter++
@@ -141,7 +170,7 @@ func loadTestData(t *testing.T) {
 			for _, split := range strings.Split(m[j], " ") {
 				r, err := strconv.ParseUint(split, 16, 64)
 				if err != nil {
-					t.Fatal(err)
+					logger.Fatal(err)
 				}
 				if test.r == 0 {
 					// save for CharacterByCharacterTests
@@ -156,38 +185,50 @@ func loadTestData(t *testing.T) {
 		part.tests = append(part.tests, test)
 	}
 	if scanner.Err() != nil {
-		t.Fatal(scanner.Err())
+		logger.Fatal(scanner.Err())
 	}
 }
 
-func cmpResult(t *testing.T, tc *Test, name string, f Form, gold, test, result string) {
+var fstr = []string{"NFC", "NFD", "NFKC", "NFKD"}
+
+var errorCount int
+
+func cmpResult(t *Test, name string, f norm.Form, gold, test, result string) {
 	if gold != result {
-		t.Errorf("%s:%s: %s(%+q)=%+q; want %+q: %s",
-			tc.Name(), name, fstr[f], test, result, gold, tc.name)
+		errorCount++
+		if errorCount > 20 {
+			return
+		}
+		logger.Printf("%s:%s: %s(%+q)=%+q; want %+q: %s",
+			t.Name(), name, fstr[f], test, result, gold, t.name)
 	}
 }
 
-func cmpIsNormal(t *testing.T, tc *Test, name string, f Form, test string, result, want bool) {
+func cmpIsNormal(t *Test, name string, f norm.Form, test string, result, want bool) {
 	if result != want {
-		t.Errorf("%s:%s: %s(%+q)=%v; want %v", tc.Name(), name, fstr[f], test, result, want)
+		errorCount++
+		if errorCount > 20 {
+			return
+		}
+		logger.Printf("%s:%s: %s(%+q)=%v; want %v", t.Name(), name, fstr[f], test, result, want)
 	}
 }
 
-func doTest(t *testing.T, tc *Test, f Form, gold, test string) {
+func doTest(t *Test, f norm.Form, gold, test string) {
 	testb := []byte(test)
 	result := f.Bytes(testb)
-	cmpResult(t, tc, "Bytes", f, gold, test, string(result))
+	cmpResult(t, "Bytes", f, gold, test, string(result))
 
 	sresult := f.String(test)
-	cmpResult(t, tc, "String", f, gold, test, sresult)
+	cmpResult(t, "String", f, gold, test, sresult)
 
 	acc := []byte{}
-	i := Iter{}
+	i := norm.Iter{}
 	i.InitString(f, test)
 	for !i.Done() {
 		acc = append(acc, i.Next()...)
 	}
-	cmpResult(t, tc, "Iter.Next", f, gold, test, string(acc))
+	cmpResult(t, "Iter.Next", f, gold, test, string(acc))
 
 	buf := make([]byte, 128)
 	acc = nil
@@ -196,33 +237,32 @@ func doTest(t *testing.T, tc *Test, f Form, gold, test string) {
 		acc = append(acc, buf[:nDst]...)
 		p += nSrc
 	}
-	cmpResult(t, tc, "Transform", f, gold, test, string(acc))
+	cmpResult(t, "Transform", f, gold, test, string(acc))
 
 	for i := range test {
 		out := f.Append(f.Bytes([]byte(test[:i])), []byte(test[i:])...)
-		cmpResult(t, tc, fmt.Sprintf(":Append:%d", i), f, gold, test, string(out))
+		cmpResult(t, fmt.Sprintf(":Append:%d", i), f, gold, test, string(out))
 	}
-	cmpIsNormal(t, tc, "IsNormal", f, test, f.IsNormal([]byte(test)), test == gold)
-	cmpIsNormal(t, tc, "IsNormalString", f, test, f.IsNormalString(test), test == gold)
+	cmpIsNormal(t, "IsNormal", f, test, f.IsNormal([]byte(test)), test == gold)
+	cmpIsNormal(t, "IsNormalString", f, test, f.IsNormalString(test), test == gold)
 }
 
-func doConformanceTests(t *testing.T, tc *Test, partn int) {
+func doConformanceTests(t *Test, partn int) {
 	for i := 0; i <= 2; i++ {
-		doTest(t, tc, NFC, tc.cols[1], tc.cols[i])
-		doTest(t, tc, NFD, tc.cols[2], tc.cols[i])
-		doTest(t, tc, NFKC, tc.cols[3], tc.cols[i])
-		doTest(t, tc, NFKD, tc.cols[4], tc.cols[i])
+		doTest(t, norm.NFC, t.cols[1], t.cols[i])
+		doTest(t, norm.NFD, t.cols[2], t.cols[i])
+		doTest(t, norm.NFKC, t.cols[3], t.cols[i])
+		doTest(t, norm.NFKD, t.cols[4], t.cols[i])
 	}
 	for i := 3; i <= 4; i++ {
-		doTest(t, tc, NFC, tc.cols[3], tc.cols[i])
-		doTest(t, tc, NFD, tc.cols[4], tc.cols[i])
-		doTest(t, tc, NFKC, tc.cols[3], tc.cols[i])
-		doTest(t, tc, NFKD, tc.cols[4], tc.cols[i])
+		doTest(t, norm.NFC, t.cols[3], t.cols[i])
+		doTest(t, norm.NFD, t.cols[4], t.cols[i])
+		doTest(t, norm.NFKC, t.cols[3], t.cols[i])
+		doTest(t, norm.NFKD, t.cols[4], t.cols[i])
 	}
 }
 
-func TestCharacterByCharacter(t *testing.T) {
-	skipShort(t)
+func CharacterByCharacterTests() {
 	tests := part[1].tests
 	var last rune = 0
 	for i := 0; i <= len(tests); i++ { // last one is special case
@@ -234,39 +274,37 @@ func TestCharacterByCharacter(t *testing.T) {
 		}
 		for last++; last < r; last++ {
 			// Check all characters that were not explicitly listed in the test.
-			tc := &Test{partnr: 1, number: -1}
+			t := &Test{partnr: 1, number: -1}
 			char := string(last)
-			doTest(t, tc, NFC, char, char)
-			doTest(t, tc, NFD, char, char)
-			doTest(t, tc, NFKC, char, char)
-			doTest(t, tc, NFKD, char, char)
+			doTest(t, norm.NFC, char, char)
+			doTest(t, norm.NFD, char, char)
+			doTest(t, norm.NFKC, char, char)
+			doTest(t, norm.NFKD, char, char)
 		}
 		if i < len(tests) {
-			doConformanceTests(t, &tests[i], 1)
+			doConformanceTests(&tests[i], 1)
 		}
 	}
 }
 
-func TestStandardTests(t *testing.T) {
-	skipShort(t)
+func StandardTests() {
 	for _, j := range []int{0, 2, 3} {
 		for _, test := range part[j].tests {
-			doConformanceTests(t, &test, j)
+			doConformanceTests(&test, j)
 		}
 	}
 }
 
-// TestPerformance verifies that normalization is O(n). If any of the
+// PerformanceTest verifies that normalization is O(n). If any of the
 // code does not properly check for maxCombiningChars, normalization
 // may exhibit O(n**2) behavior.
-func TestPerformance(t *testing.T) {
-	skipShort(t)
+func PerformanceTest() {
 	runtime.GOMAXPROCS(2)
 	success := make(chan bool, 1)
 	go func() {
 		buf := bytes.Repeat([]byte("\u035D"), 1024*1024)
 		buf = append(buf, "\u035B"...)
-		NFC.Append(nil, buf...)
+		norm.NFC.Append(nil, buf...)
 		success <- true
 	}()
 	timeout := time.After(1 * time.Second)
@@ -274,6 +312,7 @@ func TestPerformance(t *testing.T) {
 	case <-success:
 		// test completed before the timeout
 	case <-timeout:
-		t.Errorf(`unexpectedly long time to complete PerformanceTest`)
+		errorCount++
+		logger.Printf(`unexpectedly long time to complete PerformanceTest`)
 	}
 }

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/transform.go

@@ -7,16 +7,13 @@ package norm
 import (
 	"unicode/utf8"
 
-	"golang.org/x/text/transform"
+	"code.google.com/p/go.text/transform"
 )
 
-// Reset implements the Reset method of the transform.Transformer interface.
-func (Form) Reset() {}
-
-// Transform implements the Transform method of the transform.Transformer
-// interface. It may need to write segments of up to MaxSegmentSize at once.
-// Users should either catch ErrShortDst and allow dst to grow or have dst be at
-// least of size MaxTransformChunkSize to be guaranteed of progress.
+// Transform implements the transform.Transformer interface. It may need to
+// write segments of up to MaxSegmentSize at once. Users should either catch
+// ErrShortDst and allow dst to grow or have dst be at least of size
+// MaxTransformChunkSize to be guaranteed of progress.
 func (f Form) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
 	n := 0
 	// Cap the maximum number of src bytes to check.

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/transform_test.go

@@ -8,7 +8,7 @@ import (
 	"fmt"
 	"testing"
 
-	"golang.org/x/text/transform"
+	"code.google.com/p/go.text/transform"
 )
 
 func TestTransform(t *testing.T) {

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/trie.go

@@ -0,0 +1,232 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+type valueRange struct {
+	value  uint16 // header: value:stride
+	lo, hi byte   // header: lo:n
+}
+
+type trie struct {
+	index        []uint8
+	values       []uint16
+	sparse       []valueRange
+	sparseOffset []uint16
+	cutoff       uint8 // indices >= cutoff are sparse
+}
+
+// lookupValue determines the type of block n and looks up the value for b.
+// For n < t.cutoff, the block is a simple lookup table. Otherwise, the block
+// is a list of ranges with an accompanying value. Given a matching range r,
+// the value for b is by r.value + (b - r.lo) * stride.
+func (t *trie) lookupValue(n uint8, b byte) uint16 {
+	if n < t.cutoff {
+		return t.values[uint16(n)<<6+uint16(b)]
+	}
+	offset := t.sparseOffset[n-t.cutoff]
+	header := t.sparse[offset]
+	lo := offset + 1
+	hi := lo + uint16(header.lo)
+	for lo < hi {
+		m := lo + (hi-lo)/2
+		r := t.sparse[m]
+		if r.lo <= b && b <= r.hi {
+			return r.value + uint16(b-r.lo)*header.value
+		}
+		if b < r.lo {
+			hi = m
+		} else {
+			lo = m + 1
+		}
+	}
+	return 0
+}
+
+const (
+	t1 = 0x00 // 0000 0000
+	tx = 0x80 // 1000 0000
+	t2 = 0xC0 // 1100 0000
+	t3 = 0xE0 // 1110 0000
+	t4 = 0xF0 // 1111 0000
+	t5 = 0xF8 // 1111 1000
+	t6 = 0xFC // 1111 1100
+	te = 0xFE // 1111 1110
+)
+
+// lookup returns the trie value for the first UTF-8 encoding in s and
+// the width in bytes of this encoding. The size will be 0 if s does not
+// hold enough bytes to complete the encoding. len(s) must be greater than 0.
+func (t *trie) lookup(s []byte) (v uint16, sz int) {
+	c0 := s[0]
+	switch {
+	case c0 < tx:
+		return t.values[c0], 1
+	case c0 < t2:
+		return 0, 1
+	case c0 < t3:
+		if len(s) < 2 {
+			return 0, 0
+		}
+		i := t.index[c0]
+		c1 := s[1]
+		if c1 < tx || t2 <= c1 {
+			return 0, 1
+		}
+		return t.lookupValue(i, c1), 2
+	case c0 < t4:
+		if len(s) < 3 {
+			return 0, 0
+		}
+		i := t.index[c0]
+		c1 := s[1]
+		if c1 < tx || t2 <= c1 {
+			return 0, 1
+		}
+		o := uint16(i)<<6 + uint16(c1)
+		i = t.index[o]
+		c2 := s[2]
+		if c2 < tx || t2 <= c2 {
+			return 0, 2
+		}
+		return t.lookupValue(i, c2), 3
+	case c0 < t5:
+		if len(s) < 4 {
+			return 0, 0
+		}
+		i := t.index[c0]
+		c1 := s[1]
+		if c1 < tx || t2 <= c1 {
+			return 0, 1
+		}
+		o := uint16(i)<<6 + uint16(c1)
+		i = t.index[o]
+		c2 := s[2]
+		if c2 < tx || t2 <= c2 {
+			return 0, 2
+		}
+		o = uint16(i)<<6 + uint16(c2)
+		i = t.index[o]
+		c3 := s[3]
+		if c3 < tx || t2 <= c3 {
+			return 0, 3
+		}
+		return t.lookupValue(i, c3), 4
+	}
+	// Illegal rune
+	return 0, 1
+}
+
+// lookupString returns the trie value for the first UTF-8 encoding in s and
+// the width in bytes of this encoding. The size will be 0 if s does not
+// hold enough bytes to complete the encoding. len(s) must be greater than 0.
+func (t *trie) lookupString(s string) (v uint16, sz int) {
+	c0 := s[0]
+	switch {
+	case c0 < tx:
+		return t.values[c0], 1
+	case c0 < t2:
+		return 0, 1
+	case c0 < t3:
+		if len(s) < 2 {
+			return 0, 0
+		}
+		i := t.index[c0]
+		c1 := s[1]
+		if c1 < tx || t2 <= c1 {
+			return 0, 1
+		}
+		return t.lookupValue(i, c1), 2
+	case c0 < t4:
+		if len(s) < 3 {
+			return 0, 0
+		}
+		i := t.index[c0]
+		c1 := s[1]
+		if c1 < tx || t2 <= c1 {
+			return 0, 1
+		}
+		o := uint16(i)<<6 + uint16(c1)
+		i = t.index[o]
+		c2 := s[2]
+		if c2 < tx || t2 <= c2 {
+			return 0, 2
+		}
+		return t.lookupValue(i, c2), 3
+	case c0 < t5:
+		if len(s) < 4 {
+			return 0, 0
+		}
+		i := t.index[c0]
+		c1 := s[1]
+		if c1 < tx || t2 <= c1 {
+			return 0, 1
+		}
+		o := uint16(i)<<6 + uint16(c1)
+		i = t.index[o]
+		c2 := s[2]
+		if c2 < tx || t2 <= c2 {
+			return 0, 2
+		}
+		o = uint16(i)<<6 + uint16(c2)
+		i = t.index[o]
+		c3 := s[3]
+		if c3 < tx || t2 <= c3 {
+			return 0, 3
+		}
+		return t.lookupValue(i, c3), 4
+	}
+	// Illegal rune
+	return 0, 1
+}
+
+// lookupUnsafe returns the trie value for the first UTF-8 encoding in s.
+// s must hold a full encoding.
+func (t *trie) lookupUnsafe(s []byte) uint16 {
+	c0 := s[0]
+	if c0 < tx {
+		return t.values[c0]
+	}
+	if c0 < t2 {
+		return 0
+	}
+	i := t.index[c0]
+	if c0 < t3 {
+		return t.lookupValue(i, s[1])
+	}
+	i = t.index[uint16(i)<<6+uint16(s[1])]
+	if c0 < t4 {
+		return t.lookupValue(i, s[2])
+	}
+	i = t.index[uint16(i)<<6+uint16(s[2])]
+	if c0 < t5 {
+		return t.lookupValue(i, s[3])
+	}
+	return 0
+}
+
+// lookupStringUnsafe returns the trie value for the first UTF-8 encoding in s.
+// s must hold a full encoding.
+func (t *trie) lookupStringUnsafe(s string) uint16 {
+	c0 := s[0]
+	if c0 < tx {
+		return t.values[c0]
+	}
+	if c0 < t2 {
+		return 0
+	}
+	i := t.index[c0]
+	if c0 < t3 {
+		return t.lookupValue(i, s[1])
+	}
+	i = t.index[uint16(i)<<6+uint16(s[1])]
+	if c0 < t4 {
+		return t.lookupValue(i, s[2])
+	}
+	i = t.index[uint16(i)<<6+uint16(s[2])]
+	if c0 < t5 {
+		return t.lookupValue(i, s[3])
+	}
+	return 0
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/trie_test.go

@@ -0,0 +1,152 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import (
+	"testing"
+	"unicode/utf8"
+)
+
+// Test data is located in triedata_test.go; generated by maketesttables.
+var testdata = testdataTrie
+
+type rangeTest struct {
+	block   uint8
+	lookup  byte
+	result  uint16
+	table   []valueRange
+	offsets []uint16
+}
+
+var range1Off = []uint16{0, 2}
+var range1 = []valueRange{
+	{0, 1, 0},
+	{1, 0x80, 0x80},
+	{0, 2, 0},
+	{1, 0x80, 0x80},
+	{9, 0xff, 0xff},
+}
+
+var rangeTests = []rangeTest{
+	{10, 0x80, 1, range1, range1Off},
+	{10, 0x00, 0, range1, range1Off},
+	{11, 0x80, 1, range1, range1Off},
+	{11, 0xff, 9, range1, range1Off},
+	{11, 0x00, 0, range1, range1Off},
+}
+
+func TestLookupSparse(t *testing.T) {
+	for i, test := range rangeTests {
+		n := trie{sparse: test.table, sparseOffset: test.offsets, cutoff: 10}
+		v := n.lookupValue(test.block, test.lookup)
+		if v != test.result {
+			t.Errorf("LookupSparse:%d: found %X; want %X", i, v, test.result)
+		}
+	}
+}
+
+// Test cases for illegal runes.
+type trietest struct {
+	size  int
+	bytes []byte
+}
+
+var tests = []trietest{
+	// illegal runes
+	{1, []byte{0x80}},
+	{1, []byte{0xFF}},
+	{1, []byte{t2, tx - 1}},
+	{1, []byte{t2, t2}},
+	{2, []byte{t3, tx, tx - 1}},
+	{2, []byte{t3, tx, t2}},
+	{1, []byte{t3, tx - 1, tx}},
+	{3, []byte{t4, tx, tx, tx - 1}},
+	{3, []byte{t4, tx, tx, t2}},
+	{1, []byte{t4, t2, tx, tx - 1}},
+	{2, []byte{t4, tx, t2, tx - 1}},
+
+	// short runes
+	{0, []byte{t2}},
+	{0, []byte{t3, tx}},
+	{0, []byte{t4, tx, tx}},
+
+	// we only support UTF-8 up to utf8.UTFMax bytes (4 bytes)
+	{1, []byte{t5, tx, tx, tx, tx}},
+	{1, []byte{t6, tx, tx, tx, tx, tx}},
+}
+
+func mkUTF8(r rune) ([]byte, int) {
+	var b [utf8.UTFMax]byte
+	sz := utf8.EncodeRune(b[:], r)
+	return b[:sz], sz
+}
+
+func TestLookup(t *testing.T) {
+	for i, tt := range testRunes {
+		b, szg := mkUTF8(tt)
+		v, szt := testdata.lookup(b)
+		if int(v) != i {
+			t.Errorf("lookup(%U): found value %#x, expected %#x", tt, v, i)
+		}
+		if szt != szg {
+			t.Errorf("lookup(%U): found size %d, expected %d", tt, szt, szg)
+		}
+	}
+	for i, tt := range tests {
+		v, sz := testdata.lookup(tt.bytes)
+		if v != 0 {
+			t.Errorf("lookup of illegal rune, case %d: found value %#x, expected 0", i, v)
+		}
+		if sz != tt.size {
+			t.Errorf("lookup of illegal rune, case %d: found size %d, expected %d", i, sz, tt.size)
+		}
+	}
+	// Verify defaults.
+	if v, _ := testdata.lookup([]byte{0xC1, 0x8C}); v != 0 {
+		t.Errorf("lookup of non-existing rune should be 0; found %X", v)
+	}
+}
+
+func TestLookupUnsafe(t *testing.T) {
+	for i, tt := range testRunes {
+		b, _ := mkUTF8(tt)
+		v := testdata.lookupUnsafe(b)
+		if int(v) != i {
+			t.Errorf("lookupUnsafe(%U): found value %#x, expected %#x", i, v, i)
+		}
+	}
+}
+
+func TestLookupString(t *testing.T) {
+	for i, tt := range testRunes {
+		b, szg := mkUTF8(tt)
+		v, szt := testdata.lookupString(string(b))
+		if int(v) != i {
+			t.Errorf("lookup(%U): found value %#x, expected %#x", i, v, i)
+		}
+		if szt != szg {
+			t.Errorf("lookup(%U): found size %d, expected %d", i, szt, szg)
+		}
+	}
+	for i, tt := range tests {
+		v, sz := testdata.lookupString(string(tt.bytes))
+		if int(v) != 0 {
+			t.Errorf("lookup of illegal rune, case %d: found value %#x, expected 0", i, v)
+		}
+		if sz != tt.size {
+			t.Errorf("lookup of illegal rune, case %d: found size %d, expected %d", i, sz, tt.size)
+		}
+	}
+}
+
+func TestLookupStringUnsafe(t *testing.T) {
+	for i, tt := range testRunes {
+		b, _ := mkUTF8(tt)
+		v := testdata.lookupStringUnsafe(string(b))
+		if int(v) != i {
+			t.Errorf("lookupUnsafe(%U): found value %#x, expected %#x", i, v, i)
+		}
+	}
+}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/triedata_test.go

@@ -0,0 +1,85 @@
+// Generated by running
+//	maketesttables
+// DO NOT EDIT
+
+package norm
+
+var testRunes = []int32{1, 12, 127, 128, 256, 2047, 2048, 2457, 65535, 65536, 65793, 1114111, 512, 513, 514, 528, 533}
+
+// testdataValues: 192 entries, 384 bytes
+// Block 2 is the null block.
+var testdataValues = [192]uint16{
+	// Block 0x0, offset 0x0
+	0x000c: 0x0001,
+	// Block 0x1, offset 0x40
+	0x007f: 0x0002,
+	// Block 0x2, offset 0x80
+}
+
+// testdataSparseOffset: 10 entries, 20 bytes
+var testdataSparseOffset = []uint16{0x0, 0x2, 0x4, 0x8, 0xa, 0xc, 0xe, 0x10, 0x12, 0x14}
+
+// testdataSparseValues: 22 entries, 88 bytes
+var testdataSparseValues = [22]valueRange{
+	// Block 0x0, offset 0x1
+	{value: 0x0000, lo: 0x01},
+	{value: 0x0003, lo: 0x80, hi: 0x80},
+	// Block 0x1, offset 0x2
+	{value: 0x0000, lo: 0x01},
+	{value: 0x0004, lo: 0x80, hi: 0x80},
+	// Block 0x2, offset 0x3
+	{value: 0x0001, lo: 0x03},
+	{value: 0x000c, lo: 0x80, hi: 0x82},
+	{value: 0x000f, lo: 0x90, hi: 0x90},
+	{value: 0x0010, lo: 0x95, hi: 0x95},
+	// Block 0x3, offset 0x4
+	{value: 0x0000, lo: 0x01},
+	{value: 0x0005, lo: 0xbf, hi: 0xbf},
+	// Block 0x4, offset 0x5
+	{value: 0x0000, lo: 0x01},
+	{value: 0x0006, lo: 0x80, hi: 0x80},
+	// Block 0x5, offset 0x6
+	{value: 0x0000, lo: 0x01},
+	{value: 0x0007, lo: 0x99, hi: 0x99},
+	// Block 0x6, offset 0x7
+	{value: 0x0000, lo: 0x01},
+	{value: 0x0008, lo: 0xbf, hi: 0xbf},
+	// Block 0x7, offset 0x8
+	{value: 0x0000, lo: 0x01},
+	{value: 0x0009, lo: 0x80, hi: 0x80},
+	// Block 0x8, offset 0x9
+	{value: 0x0000, lo: 0x01},
+	{value: 0x000a, lo: 0x81, hi: 0x81},
+	// Block 0x9, offset 0xa
+	{value: 0x0000, lo: 0x01},
+	{value: 0x000b, lo: 0xbf, hi: 0xbf},
+}
+
+// testdataLookup: 640 bytes
+// Block 0 is the null block.
+var testdataLookup = [640]uint8{
+	// Block 0x0, offset 0x0
+	// Block 0x1, offset 0x40
+	// Block 0x2, offset 0x80
+	// Block 0x3, offset 0xc0
+	0x0c2: 0x01, 0x0c4: 0x02,
+	0x0c8: 0x03,
+	0x0df: 0x04,
+	0x0e0: 0x02,
+	0x0ef: 0x03,
+	0x0f0: 0x05, 0x0f4: 0x07,
+	// Block 0x4, offset 0x100
+	0x120: 0x05, 0x126: 0x06,
+	// Block 0x5, offset 0x140
+	0x17f: 0x07,
+	// Block 0x6, offset 0x180
+	0x180: 0x08, 0x184: 0x09,
+	// Block 0x7, offset 0x1c0
+	0x1d0: 0x04,
+	// Block 0x8, offset 0x200
+	0x23f: 0x0a,
+	// Block 0x9, offset 0x240
+	0x24f: 0x06,
+}
+
+var testdataTrie = trie{testdataLookup[:], testdataValues[:], testdataSparseValues[:], testdataSparseOffset[:], 1}

Godeps/_workspace/src/code.google.com/p/go.text/unicode/norm/triegen.go

@@ -0,0 +1,317 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// Trie table generator.
+// Used by make*tables tools to generate a go file with trie data structures
+// for mapping UTF-8 to a 16-bit value. All but the last byte in a UTF-8 byte
+// sequence are used to lookup offsets in the index table to be used for the
+// next byte. The last byte is used to index into a table with 16-bit values.
+
+package main
+
+import (
+	"fmt"
+	"hash/crc32"
+	"log"
+	"unicode/utf8"
+)
+
+const (
+	blockSize        = 64
+	blockOffset      = 2 // Subtract two blocks to compensate for the 0x80 added to continuation bytes.
+	maxSparseEntries = 16
+)
+
+// Intermediate trie structure
+type trieNode struct {
+	table [256]*trieNode
+	value int
+	b     byte
+	leaf  bool
+}
+
+func newNode() *trieNode {
+	return new(trieNode)
+}
+
+func (n trieNode) String() string {
+	s := fmt.Sprint("trieNode{table: { non-nil at index: ")
+	for i, v := range n.table {
+		if v != nil {
+			s += fmt.Sprintf("%d, ", i)
+		}
+	}
+	s += fmt.Sprintf("}, value:%#x, b:%#x leaf:%v}", n.value, n.b, n.leaf)
+	return s
+}
+
+func (n trieNode) isInternal() bool {
+	internal := true
+	for i := 0; i < 256; i++ {
+		if nn := n.table[i]; nn != nil {
+			if !internal && !nn.leaf {
+				log.Fatalf("triegen: isInternal: node contains both leaf and non-leaf children (%v)", n)
+			}
+			internal = internal && !nn.leaf
+		}
+	}
+	return internal
+}
+
+func (n trieNode) mostFrequentStride() int {
+	counts := make(map[int]int)
+	v := 0
+	for _, t := range n.table[0x80 : 0x80+blockSize] {
+		if t != nil {
+			if stride := t.value - v; v != 0 && stride >= 0 {
+				counts[stride]++
+			}
+			v = t.value
+		} else {
+			v = 0
+		}
+	}
+	var maxs, maxc int
+	for stride, cnt := range counts {
+		if cnt > maxc || (cnt == maxc && stride < maxs) {
+			maxs, maxc = stride, cnt
+		}
+	}
+	return maxs
+}
+
+func (n trieNode) countSparseEntries() int {
+	stride := n.mostFrequentStride()
+	var count, v int
+	for _, t := range n.table[0x80 : 0x80+blockSize] {
+		tv := 0
+		if t != nil {
+			tv = t.value
+		}
+		if tv-v != stride {
+			if tv != 0 {
+				count++
+			}
+		}
+		v = tv
+	}
+	return count
+}
+
+func (n *trieNode) insert(r rune, value uint16) {
+	var p [utf8.UTFMax]byte
+	sz := utf8.EncodeRune(p[:], r)
+
+	for i := 0; i < sz; i++ {
+		if n.leaf {
+			log.Fatalf("triegen: insert: node (%#v) should not be a leaf", n)
+		}
+		nn := n.table[p[i]]
+		if nn == nil {
+			nn = newNode()
+			nn.b = p[i]
+			n.table[p[i]] = nn
+		}
+		n = nn
+	}
+	n.value = int(value)
+	n.leaf = true
+}
+
+type nodeIndex struct {
+	lookupBlocks []*trieNode
+	valueBlocks  []*trieNode
+	sparseBlocks []*trieNode
+	sparseOffset []uint16
+	sparseCount  int
+
+	lookupBlockIdx map[uint32]int
+	valueBlockIdx  map[uint32]int
+}
+
+func newIndex() *nodeIndex {
+	index := &nodeIndex{}
+	index.lookupBlocks = make([]*trieNode, 0)
+	index.valueBlocks = make([]*trieNode, 0)
+	index.sparseBlocks = make([]*trieNode, 0)
+	index.sparseOffset = make([]uint16, 1)
+	index.lookupBlockIdx = make(map[uint32]int)
+	index.valueBlockIdx = make(map[uint32]int)
+	return index
+}
+
+func computeOffsets(index *nodeIndex, n *trieNode) int {
+	if n.leaf {
+		return n.value
+	}
+	hasher := crc32.New(crc32.MakeTable(crc32.IEEE))
+	// We only index continuation bytes.
+	for i := 0; i < blockSize; i++ {
+		v := 0
+		if nn := n.table[0x80+i]; nn != nil {
+			v = computeOffsets(index, nn)
+		}
+		hasher.Write([]byte{uint8(v >> 8), uint8(v)})
+	}
+	h := hasher.Sum32()
+	if n.isInternal() {
+		v, ok := index.lookupBlockIdx[h]
+		if !ok {
+			v = len(index.lookupBlocks) - blockOffset
+			index.lookupBlocks = append(index.lookupBlocks, n)
+			index.lookupBlockIdx[h] = v
+		}
+		n.value = v
+	} else {
+		v, ok := index.valueBlockIdx[h]
+		if !ok {
+			if c := n.countSparseEntries(); c > maxSparseEntries {
+				v = len(index.valueBlocks) - blockOffset
+				index.valueBlocks = append(index.valueBlocks, n)
+				index.valueBlockIdx[h] = v
+			} else {
+				v = -len(index.sparseOffset)
+				index.sparseBlocks = append(index.sparseBlocks, n)
+				index.sparseOffset = append(index.sparseOffset, uint16(index.sparseCount))
+				index.sparseCount += c + 1
+				index.valueBlockIdx[h] = v
+			}
+		}
+		n.value = v
+	}
+	return n.value
+}
+
+func printValueBlock(nr int, n *trieNode, offset int) {
+	boff := nr * blockSize
+	fmt.Printf("\n// Block %#x, offset %#x", nr, boff)
+	var printnewline bool
+	for i := 0; i < blockSize; i++ {
+		if i%6 == 0 {
+			printnewline = true
+		}
+		v := 0
+		if nn := n.table[i+offset]; nn != nil {
+			v = nn.value
+		}
+		if v != 0 {
+			if printnewline {
+				fmt.Printf("\n")
+				printnewline = false
+			}
+			fmt.Printf("%#04x:%#04x, ", boff+i, v)
+		}
+	}
+}
+
+func printSparseBlock(nr int, n *trieNode) {
+	boff := -n.value
+	fmt.Printf("\n// Block %#x, offset %#x", nr, boff)
+	v := 0
+	//stride := f(n)
+	stride := n.mostFrequentStride()
+	c := n.countSparseEntries()
+	fmt.Printf("\n{value:%#04x,lo:%#02x},", stride, uint8(c))
+	for i, nn := range n.table[0x80 : 0x80+blockSize] {
+		nv := 0
+		if nn != nil {
+			nv = nn.value
+		}
+		if nv-v != stride {
+			if v != 0 {
+				fmt.Printf(",hi:%#02x},", 0x80+i-1)
+			}
+			if nv != 0 {
+				fmt.Printf("\n{value:%#04x,lo:%#02x", nv, nn.b)
+			}
+		}
+		v = nv
+	}
+	if v != 0 {
+		fmt.Printf(",hi:%#02x},", 0x80+blockSize-1)
+	}
+}
+
+func printLookupBlock(nr int, n *trieNode, offset, cutoff int) {
+	boff := nr * blockSize
+	fmt.Printf("\n// Block %#x, offset %#x", nr, boff)
+	var printnewline bool
+	for i := 0; i < blockSize; i++ {
+		if i%8 == 0 {
+			printnewline = true
+		}
+		v := 0
+		if nn := n.table[i+offset]; nn != nil {
+			v = nn.value
+		}
+		if v != 0 {
+			if v < 0 {
+				v = -v - 1 + cutoff
+			}
+			if printnewline {
+				fmt.Printf("\n")
+				printnewline = false
+			}
+			fmt.Printf("%#03x:%#02x, ", boff+i, v)
+		}
+	}
+}
+
+// printTables returns the size in bytes of the generated tables.
+func (t *trieNode) printTables(name string) int {
+	index := newIndex()
+	// Values for 7-bit ASCII are stored in first two block, followed by nil block.
+	index.valueBlocks = append(index.valueBlocks, nil, nil, nil)
+	// First byte of multi-byte UTF-8 codepoints are indexed in 4th block.
+	index.lookupBlocks = append(index.lookupBlocks, nil, nil, nil, nil)
+	// Index starter bytes of multi-byte UTF-8.
+	for i := 0xC0; i < 0x100; i++ {
+		if t.table[i] != nil {
+			computeOffsets(index, t.table[i])
+		}
+	}
+
+	nv := len(index.valueBlocks) * blockSize
+	fmt.Printf("// %sValues: %d entries, %d bytes\n", name, nv, nv*2)
+	fmt.Printf("// Block 2 is the null block.\n")
+	fmt.Printf("var %sValues = [%d]uint16 {", name, nv)
+	printValueBlock(0, t, 0)
+	printValueBlock(1, t, 64)
+	printValueBlock(2, newNode(), 0)
+	for i := 3; i < len(index.valueBlocks); i++ {
+		printValueBlock(i, index.valueBlocks[i], 0x80)
+	}
+	fmt.Print("\n}\n\n")
+
+	ls := len(index.sparseBlocks)
+	fmt.Printf("// %sSparseOffset: %d entries, %d bytes\n", name, ls, ls*2)
+	fmt.Printf("var %sSparseOffset = %#v\n\n", name, index.sparseOffset[1:])
+
+	ns := index.sparseCount
+	fmt.Printf("// %sSparseValues: %d entries, %d bytes\n", name, ns, ns*4)
+	fmt.Printf("var %sSparseValues = [%d]valueRange {", name, ns)
+	for i, n := range index.sparseBlocks {
+		printSparseBlock(i, n)
+	}
+	fmt.Print("\n}\n\n")
+
+	cutoff := len(index.valueBlocks) - blockOffset
+	ni := len(index.lookupBlocks) * blockSize
+	fmt.Printf("// %sLookup: %d bytes\n", name, ni)
+	fmt.Printf("// Block 0 is the null block.\n")
+	fmt.Printf("var %sLookup = [%d]uint8 {", name, ni)
+	printLookupBlock(0, newNode(), 0, cutoff)
+	printLookupBlock(1, newNode(), 0, cutoff)
+	printLookupBlock(2, newNode(), 0, cutoff)
+	printLookupBlock(3, t, 0xC0, cutoff)
+	for i := 4; i < len(index.lookupBlocks); i++ {
+		printLookupBlock(i, index.lookupBlocks[i], 0x80, cutoff)
+	}
+	fmt.Print("\n}\n\n")
+	fmt.Printf("var %sTrie = trie{ %sLookup[:], %sValues[:], %sSparseValues[:], %sSparseOffset[:], %d}\n\n",
+		name, name, name, name, name, cutoff)
+	return nv*2 + ns*4 + ni + ls*2
+}

Godeps/_workspace/src/code.google.com/p/snappy-go/snappy/decode.go

@@ -0,0 +1,124 @@
+// Copyright 2011 The Snappy-Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package snappy
+
+import (
+	"encoding/binary"
+	"errors"
+)
+
+// ErrCorrupt reports that the input is invalid.
+var ErrCorrupt = errors.New("snappy: corrupt input")
+
+// DecodedLen returns the length of the decoded block.
+func DecodedLen(src []byte) (int, error) {
+	v, _, err := decodedLen(src)
+	return v, err
+}
+
+// decodedLen returns the length of the decoded block and the number of bytes
+// that the length header occupied.
+func decodedLen(src []byte) (blockLen, headerLen int, err error) {
+	v, n := binary.Uvarint(src)
+	if n == 0 {
+		return 0, 0, ErrCorrupt
+	}
+	if uint64(int(v)) != v {
+		return 0, 0, errors.New("snappy: decoded block is too large")
+	}
+	return int(v), n, nil
+}
+
+// Decode returns the decoded form of src. The returned slice may be a sub-
+// slice of dst if dst was large enough to hold the entire decoded block.
+// Otherwise, a newly allocated slice will be returned.
+// It is valid to pass a nil dst.
+func Decode(dst, src []byte) ([]byte, error) {
+	dLen, s, err := decodedLen(src)
+	if err != nil {
+		return nil, err
+	}
+	if len(dst) < dLen {
+		dst = make([]byte, dLen)
+	}
+
+	var d, offset, length int
+	for s < len(src) {
+		switch src[s] & 0x03 {
+		case tagLiteral:
+			x := uint(src[s] >> 2)
+			switch {
+			case x < 60:
+				s += 1
+			case x == 60:
+				s += 2
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-1])
+			case x == 61:
+				s += 3
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-2]) | uint(src[s-1])<<8
+			case x == 62:
+				s += 4
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-3]) | uint(src[s-2])<<8 | uint(src[s-1])<<16
+			case x == 63:
+				s += 5
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-4]) | uint(src[s-3])<<8 | uint(src[s-2])<<16 | uint(src[s-1])<<24
+			}
+			length = int(x + 1)
+			if length <= 0 {
+				return nil, errors.New("snappy: unsupported literal length")
+			}
+			if length > len(dst)-d || length > len(src)-s {
+				return nil, ErrCorrupt
+			}
+			copy(dst[d:], src[s:s+length])
+			d += length
+			s += length
+			continue
+
+		case tagCopy1:
+			s += 2
+			if s > len(src) {
+				return nil, ErrCorrupt
+			}
+			length = 4 + int(src[s-2])>>2&0x7
+			offset = int(src[s-2])&0xe0<<3 | int(src[s-1])
+
+		case tagCopy2:
+			s += 3
+			if s > len(src) {
+				return nil, ErrCorrupt
+			}
+			length = 1 + int(src[s-3])>>2
+			offset = int(src[s-2]) | int(src[s-1])<<8
+
+		case tagCopy4:
+			return nil, errors.New("snappy: unsupported COPY_4 tag")
+		}
+
+		end := d + length
+		if offset > d || end > len(dst) {
+			return nil, ErrCorrupt
+		}
+		for ; d < end; d++ {
+			dst[d] = dst[d-offset]
+		}
+	}
+	if d != dLen {
+		return nil, ErrCorrupt
+	}
+	return dst[:d], nil
+}

Godeps/_workspace/src/code.google.com/p/snappy-go/snappy/encode.go

@@ -6,7 +6,6 @@ package snappy
 
 import (
 	"encoding/binary"
-	"io"
 )
 
 // We limit how far copy back-references can go, the same as the C++ code.
@@ -79,7 +78,7 @@ func emitCopy(dst []byte, offset, length int) int {
 // slice of dst if dst was large enough to hold the entire encoded block.
 // Otherwise, a newly allocated slice will be returned.
 // It is valid to pass a nil dst.
-func Encode(dst, src []byte) []byte {
+func Encode(dst, src []byte) ([]byte, error) {
 	if n := MaxEncodedLen(len(src)); len(dst) < n {
 		dst = make([]byte, n)
 	}
@@ -92,7 +91,7 @@ func Encode(dst, src []byte) []byte {
 		if len(src) != 0 {
 			d += emitLiteral(dst[d:], src)
 		}
-		return dst[:d]
+		return dst[:d], nil
 	}
 
 	// Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive.
@@ -145,7 +144,7 @@ func Encode(dst, src []byte) []byte {
 	if lit != len(src) {
 		d += emitLiteral(dst[d:], src[lit:])
 	}
-	return dst[:d]
+	return dst[:d], nil
 }
 
 // MaxEncodedLen returns the maximum length of a snappy block, given its
@@ -173,82 +172,3 @@ func MaxEncodedLen(srcLen int) int {
 	// This last factor dominates the blowup, so the final estimate is:
 	return 32 + srcLen + srcLen/6
 }
-
-// NewWriter returns a new Writer that compresses to w, using the framing
-// format described at
-// https://github.com/google/snappy/blob/master/framing_format.txt
-func NewWriter(w io.Writer) *Writer {
-	return &Writer{
-		w:   w,
-		enc: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)),
-	}
-}
-
-// Writer is an io.Writer than can write Snappy-compressed bytes.
-type Writer struct {
-	w           io.Writer
-	err         error
-	enc         []byte
-	buf         [checksumSize + chunkHeaderSize]byte
-	wroteHeader bool
-}
-
-// Reset discards the writer's state and switches the Snappy writer to write to
-// w. This permits reusing a Writer rather than allocating a new one.
-func (w *Writer) Reset(writer io.Writer) {
-	w.w = writer
-	w.err = nil
-	w.wroteHeader = false
-}
-
-// Write satisfies the io.Writer interface.
-func (w *Writer) Write(p []byte) (n int, errRet error) {
-	if w.err != nil {
-		return 0, w.err
-	}
-	if !w.wroteHeader {
-		copy(w.enc, magicChunk)
-		if _, err := w.w.Write(w.enc[:len(magicChunk)]); err != nil {
-			w.err = err
-			return n, err
-		}
-		w.wroteHeader = true
-	}
-	for len(p) > 0 {
-		var uncompressed []byte
-		if len(p) > maxUncompressedChunkLen {
-			uncompressed, p = p[:maxUncompressedChunkLen], p[maxUncompressedChunkLen:]
-		} else {
-			uncompressed, p = p, nil
-		}
-		checksum := crc(uncompressed)
-
-		// Compress the buffer, discarding the result if the improvement
-		// isn't at least 12.5%.
-		chunkType := uint8(chunkTypeCompressedData)
-		chunkBody := Encode(w.enc, uncompressed)
-		if len(chunkBody) >= len(uncompressed)-len(uncompressed)/8 {
-			chunkType, chunkBody = chunkTypeUncompressedData, uncompressed
-		}
-
-		chunkLen := 4 + len(chunkBody)
-		w.buf[0] = chunkType
-		w.buf[1] = uint8(chunkLen >> 0)
-		w.buf[2] = uint8(chunkLen >> 8)
-		w.buf[3] = uint8(chunkLen >> 16)
-		w.buf[4] = uint8(checksum >> 0)
-		w.buf[5] = uint8(checksum >> 8)
-		w.buf[6] = uint8(checksum >> 16)
-		w.buf[7] = uint8(checksum >> 24)
-		if _, err := w.w.Write(w.buf[:]); err != nil {
-			w.err = err
-			return n, err
-		}
-		if _, err := w.w.Write(chunkBody); err != nil {
-			w.err = err
-			return n, err
-		}
-		n += len(uncompressed)
-	}
-	return n, nil
-}

Godeps/_workspace/src/code.google.com/p/snappy-go/snappy/snappy.go

@@ -5,13 +5,9 @@
 // Package snappy implements the snappy block-based compression format.
 // It aims for very high speeds and reasonable compression.
 //
-// The C++ snappy implementation is at https://github.com/google/snappy
+// The C++ snappy implementation is at http://code.google.com/p/snappy/
 package snappy
 
-import (
-	"hash/crc32"
-)
-
 /*
 Each encoded block begins with the varint-encoded length of the decoded data,
 followed by a sequence of chunks. Chunks begin and end on byte boundaries. The
@@ -40,29 +36,3 @@ const (
 	tagCopy2   = 0x02
 	tagCopy4   = 0x03
 )
-
-const (
-	checksumSize    = 4
-	chunkHeaderSize = 4
-	magicChunk      = "\xff\x06\x00\x00" + magicBody
-	magicBody       = "sNaPpY"
-	// https://github.com/google/snappy/blob/master/framing_format.txt says
-	// that "the uncompressed data in a chunk must be no longer than 65536 bytes".
-	maxUncompressedChunkLen = 65536
-)
-
-const (
-	chunkTypeCompressedData   = 0x00
-	chunkTypeUncompressedData = 0x01
-	chunkTypePadding          = 0xfe
-	chunkTypeStreamIdentifier = 0xff
-)
-
-var crcTable = crc32.MakeTable(crc32.Castagnoli)
-
-// crc implements the checksum specified in section 3 of
-// https://github.com/google/snappy/blob/master/framing_format.txt
-func crc(b []byte) uint32 {
-	c := crc32.Update(0, crcTable, b)
-	return uint32(c>>15|c<<17) + 0xa282ead8
-}

Godeps/_workspace/src/code.google.com/p/snappy-go/snappy/snappy_test.go

@@ -18,13 +18,14 @@ import (
 	"testing"
 )
 
-var (
-	download = flag.Bool("download", false, "If true, download any missing files before running benchmarks")
-	testdata = flag.String("testdata", "testdata", "Directory containing the test data")
-)
+var download = flag.Bool("download", false, "If true, download any missing files before running benchmarks")
 
 func roundtrip(b, ebuf, dbuf []byte) error {
-	d, err := Decode(dbuf, Encode(ebuf, b))
+	e, err := Encode(ebuf, b)
+	if err != nil {
+		return fmt.Errorf("encoding error: %v", err)
+	}
+	d, err := Decode(dbuf, e)
 	if err != nil {
 		return fmt.Errorf("decoding error: %v", err)
 	}
@@ -54,11 +55,11 @@ func TestSmallCopy(t *testing.T) {
 }
 
 func TestSmallRand(t *testing.T) {
-	rng := rand.New(rand.NewSource(27354294))
+	rand.Seed(27354294)
 	for n := 1; n < 20000; n += 23 {
 		b := make([]byte, n)
-		for i := range b {
-			b[i] = uint8(rng.Uint32())
+		for i, _ := range b {
+			b[i] = uint8(rand.Uint32())
 		}
 		if err := roundtrip(b, nil, nil); err != nil {
 			t.Fatal(err)
@@ -69,7 +70,7 @@ func TestSmallRand(t *testing.T) {
 func TestSmallRegular(t *testing.T) {
 	for n := 1; n < 20000; n += 23 {
 		b := make([]byte, n)
-		for i := range b {
+		for i, _ := range b {
 			b[i] = uint8(i%10 + 'a')
 		}
 		if err := roundtrip(b, nil, nil); err != nil {
@@ -78,142 +79,11 @@ func TestSmallRegular(t *testing.T) {
 	}
 }
 
-func TestInvalidVarint(t *testing.T) {
-	data := []byte("\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x00")
-	if _, err := DecodedLen(data); err != ErrCorrupt {
-		t.Errorf("DecodedLen: got %v, want ErrCorrupt", err)
-	}
-	if _, err := Decode(nil, data); err != ErrCorrupt {
-		t.Errorf("Decode: got %v, want ErrCorrupt", err)
-	}
-
-	// The encoded varint overflows 32 bits
-	data = []byte("\xff\xff\xff\xff\xff\x00")
-
-	if _, err := DecodedLen(data); err != ErrCorrupt {
-		t.Errorf("DecodedLen: got %v, want ErrCorrupt", err)
-	}
-	if _, err := Decode(nil, data); err != ErrCorrupt {
-		t.Errorf("Decode: got %v, want ErrCorrupt", err)
-	}
-}
-
-func cmp(a, b []byte) error {
-	if len(a) != len(b) {
-		return fmt.Errorf("got %d bytes, want %d", len(a), len(b))
-	}
-	for i := range a {
-		if a[i] != b[i] {
-			return fmt.Errorf("byte #%d: got 0x%02x, want 0x%02x", i, a[i], b[i])
-		}
-	}
-	return nil
-}
-
-func TestFramingFormat(t *testing.T) {
-	// src is comprised of alternating 1e5-sized sequences of random
-	// (incompressible) bytes and repeated (compressible) bytes. 1e5 was chosen
-	// because it is larger than maxUncompressedChunkLen (64k).
-	src := make([]byte, 1e6)
-	rng := rand.New(rand.NewSource(1))
-	for i := 0; i < 10; i++ {
-		if i%2 == 0 {
-			for j := 0; j < 1e5; j++ {
-				src[1e5*i+j] = uint8(rng.Intn(256))
-			}
-		} else {
-			for j := 0; j < 1e5; j++ {
-				src[1e5*i+j] = uint8(i)
-			}
-		}
-	}
-
-	buf := new(bytes.Buffer)
-	if _, err := NewWriter(buf).Write(src); err != nil {
-		t.Fatalf("Write: encoding: %v", err)
-	}
-	dst, err := ioutil.ReadAll(NewReader(buf))
-	if err != nil {
-		t.Fatalf("ReadAll: decoding: %v", err)
-	}
-	if err := cmp(dst, src); err != nil {
-		t.Fatal(err)
-	}
-}
-
-func TestReaderReset(t *testing.T) {
-	gold := bytes.Repeat([]byte("All that is gold does not glitter,\n"), 10000)
-	buf := new(bytes.Buffer)
-	if _, err := NewWriter(buf).Write(gold); err != nil {
-		t.Fatalf("Write: %v", err)
-	}
-	encoded, invalid, partial := buf.String(), "invalid", "partial"
-	r := NewReader(nil)
-	for i, s := range []string{encoded, invalid, partial, encoded, partial, invalid, encoded, encoded} {
-		if s == partial {
-			r.Reset(strings.NewReader(encoded))
-			if _, err := r.Read(make([]byte, 101)); err != nil {
-				t.Errorf("#%d: %v", i, err)
-				continue
-			}
-			continue
-		}
-		r.Reset(strings.NewReader(s))
-		got, err := ioutil.ReadAll(r)
-		switch s {
-		case encoded:
-			if err != nil {
-				t.Errorf("#%d: %v", i, err)
-				continue
-			}
-			if err := cmp(got, gold); err != nil {
-				t.Errorf("#%d: %v", i, err)
-				continue
-			}
-		case invalid:
-			if err == nil {
-				t.Errorf("#%d: got nil error, want non-nil", i)
-				continue
-			}
-		}
-	}
-}
-
-func TestWriterReset(t *testing.T) {
-	gold := bytes.Repeat([]byte("Not all those who wander are lost;\n"), 10000)
-	var gots, wants [][]byte
-	const n = 20
-	w, failed := NewWriter(nil), false
-	for i := 0; i <= n; i++ {
-		buf := new(bytes.Buffer)
-		w.Reset(buf)
-		want := gold[:len(gold)*i/n]
-		if _, err := w.Write(want); err != nil {
-			t.Errorf("#%d: Write: %v", i, err)
-			failed = true
-			continue
-		}
-		got, err := ioutil.ReadAll(NewReader(buf))
-		if err != nil {
-			t.Errorf("#%d: ReadAll: %v", i, err)
-			failed = true
-			continue
-		}
-		gots = append(gots, got)
-		wants = append(wants, want)
-	}
-	if failed {
-		return
-	}
-	for i := range gots {
-		if err := cmp(gots[i], wants[i]); err != nil {
-			t.Errorf("#%d: %v", i, err)
-		}
-	}
-}
-
 func benchDecode(b *testing.B, src []byte) {
-	encoded := Encode(nil, src)
+	encoded, err := Encode(nil, src)
+	if err != nil {
+		b.Fatal(err)
+	}
 	// Bandwidth is in amount of uncompressed data.
 	b.SetBytes(int64(len(src)))
 	b.ResetTimer()
@@ -232,10 +102,10 @@ func benchEncode(b *testing.B, src []byte) {
 	}
 }
 
-func readFile(b testing.TB, filename string) []byte {
+func readFile(b *testing.B, filename string) []byte {
 	src, err := ioutil.ReadFile(filename)
 	if err != nil {
-		b.Skipf("skipping benchmark: %v", err)
+		b.Fatalf("failed reading %s: %s", filename, err)
 	}
 	if len(src) == 0 {
 		b.Fatalf("%s has zero length", filename)
@@ -274,7 +144,7 @@ func BenchmarkWordsEncode1e5(b *testing.B) { benchWords(b, 1e5, false) }
 func BenchmarkWordsEncode1e6(b *testing.B) { benchWords(b, 1e6, false) }
 
 // testFiles' values are copied directly from
-// https://raw.githubusercontent.com/google/snappy/master/snappy_unittest.cc
+// https://code.google.com/p/snappy/source/browse/trunk/snappy_unittest.cc.
 // The label field is unused in snappy-go.
 var testFiles = []struct {
 	label    string
@@ -282,36 +152,29 @@ var testFiles = []struct {
 }{
 	{"html", "html"},
 	{"urls", "urls.10K"},
-	{"jpg", "fireworks.jpeg"},
-	{"jpg_200", "fireworks.jpeg"},
-	{"pdf", "paper-100k.pdf"},
+	{"jpg", "house.jpg"},
+	{"pdf", "mapreduce-osdi-1.pdf"},
 	{"html4", "html_x_4"},
+	{"cp", "cp.html"},
+	{"c", "fields.c"},
+	{"lsp", "grammar.lsp"},
+	{"xls", "kennedy.xls"},
 	{"txt1", "alice29.txt"},
 	{"txt2", "asyoulik.txt"},
 	{"txt3", "lcet10.txt"},
 	{"txt4", "plrabn12.txt"},
+	{"bin", "ptt5"},
+	{"sum", "sum"},
+	{"man", "xargs.1"},
 	{"pb", "geo.protodata"},
 	{"gaviota", "kppkn.gtb"},
 }
 
 // The test data files are present at this canonical URL.
-const baseURL = "https://raw.githubusercontent.com/google/snappy/master/testdata/"
-
-func downloadTestdata(b *testing.B, basename string) (errRet error) {
-	filename := filepath.Join(*testdata, basename)
-	if stat, err := os.Stat(filename); err == nil && stat.Size() != 0 {
-		return nil
-	}
-
-	if !*download {
-		b.Skipf("test data not found; skipping benchmark without the -download flag")
-	}
-	// Download the official snappy C++ implementation reference test data
-	// files for benchmarking.
-	if err := os.Mkdir(*testdata, 0777); err != nil && !os.IsExist(err) {
-		return fmt.Errorf("failed to create testdata: %s", err)
-	}
+const baseURL = "https://snappy.googlecode.com/svn/trunk/testdata/"
 
+func downloadTestdata(basename string) (errRet error) {
+	filename := filepath.Join("testdata", basename)
 	f, err := os.Create(filename)
 	if err != nil {
 		return fmt.Errorf("failed to create %s: %s", filename, err)
@@ -322,27 +185,36 @@ func downloadTestdata(b *testing.B, basename string) (errRet error) {
 			os.Remove(filename)
 		}
 	}()
-	url := baseURL + basename
-	resp, err := http.Get(url)
+	resp, err := http.Get(baseURL + basename)
 	if err != nil {
-		return fmt.Errorf("failed to download %s: %s", url, err)
+		return fmt.Errorf("failed to download %s: %s", baseURL+basename, err)
 	}
 	defer resp.Body.Close()
-	if s := resp.StatusCode; s != http.StatusOK {
-		return fmt.Errorf("downloading %s: HTTP status code %d (%s)", url, s, http.StatusText(s))
-	}
 	_, err = io.Copy(f, resp.Body)
 	if err != nil {
-		return fmt.Errorf("failed to download %s to %s: %s", url, filename, err)
+		return fmt.Errorf("failed to write %s: %s", filename, err)
 	}
 	return nil
 }
 
 func benchFile(b *testing.B, n int, decode bool) {
-	if err := downloadTestdata(b, testFiles[n].filename); err != nil {
-		b.Fatalf("failed to download testdata: %s", err)
+	filename := filepath.Join("testdata", testFiles[n].filename)
+	if stat, err := os.Stat(filename); err != nil || stat.Size() == 0 {
+		if !*download {
+			b.Fatal("test data not found; skipping benchmark without the -download flag")
+		}
+		// Download the official snappy C++ implementation reference test data
+		// files for benchmarking.
+		if err := os.Mkdir("testdata", 0777); err != nil && !os.IsExist(err) {
+			b.Fatalf("failed to create testdata: %s", err)
+		}
+		for _, tf := range testFiles {
+			if err := downloadTestdata(tf.filename); err != nil {
+				b.Fatalf("failed to download testdata: %s", err)
+			}
+		}
 	}
-	data := readFile(b, filepath.Join(*testdata, testFiles[n].filename))
+	data := readFile(b, filename)
 	if decode {
 		benchDecode(b, data)
 	} else {
@@ -363,6 +235,12 @@ func Benchmark_UFlat8(b *testing.B)  { benchFile(b, 8, true) }
 func Benchmark_UFlat9(b *testing.B)  { benchFile(b, 9, true) }
 func Benchmark_UFlat10(b *testing.B) { benchFile(b, 10, true) }
 func Benchmark_UFlat11(b *testing.B) { benchFile(b, 11, true) }
+func Benchmark_UFlat12(b *testing.B) { benchFile(b, 12, true) }
+func Benchmark_UFlat13(b *testing.B) { benchFile(b, 13, true) }
+func Benchmark_UFlat14(b *testing.B) { benchFile(b, 14, true) }
+func Benchmark_UFlat15(b *testing.B) { benchFile(b, 15, true) }
+func Benchmark_UFlat16(b *testing.B) { benchFile(b, 16, true) }
+func Benchmark_UFlat17(b *testing.B) { benchFile(b, 17, true) }
 func Benchmark_ZFlat0(b *testing.B)  { benchFile(b, 0, false) }
 func Benchmark_ZFlat1(b *testing.B)  { benchFile(b, 1, false) }
 func Benchmark_ZFlat2(b *testing.B)  { benchFile(b, 2, false) }
@@ -375,3 +253,9 @@ func Benchmark_ZFlat8(b *testing.B)  { benchFile(b, 8, false) }
 func Benchmark_ZFlat9(b *testing.B)  { benchFile(b, 9, false) }
 func Benchmark_ZFlat10(b *testing.B) { benchFile(b, 10, false) }
 func Benchmark_ZFlat11(b *testing.B) { benchFile(b, 11, false) }
+func Benchmark_ZFlat12(b *testing.B) { benchFile(b, 12, false) }
+func Benchmark_ZFlat13(b *testing.B) { benchFile(b, 13, false) }
+func Benchmark_ZFlat14(b *testing.B) { benchFile(b, 14, false) }
+func Benchmark_ZFlat15(b *testing.B) { benchFile(b, 15, false) }
+func Benchmark_ZFlat16(b *testing.B) { benchFile(b, 16, false) }
+func Benchmark_ZFlat17(b *testing.B) { benchFile(b, 17, false) }

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/.travis.yml

@@ -4,6 +4,4 @@ go:
   - 1.1
   - 1.2
   - 1.3
-  - 1.4
-  - 1.5
   - tip

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/README.md

@@ -4,7 +4,7 @@ go-lz4
 go-lz4 is port of LZ4 lossless compression algorithm to Go. The original C code
 is located at:
 
-https://github.com/Cyan4973/lz4
+https://code.google.com/p/lz4/
 
 Status
 ------

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/fuzz.go

@@ -1,23 +0,0 @@
-// +build gofuzz
-
-package lz4
-
-import "encoding/binary"
-
-func Fuzz(data []byte) int {
-
-	if len(data) < 4 {
-		return 0
-	}
-
-	ln := binary.LittleEndian.Uint32(data)
-	if ln > (1 << 21) {
-		return 0
-	}
-
-	if _, err := Decode(nil, data); err != nil {
-		return 0
-	}
-
-	return 1
-}

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/lz4-example/main.go

@@ -72,18 +72,10 @@ func main() {
 
 	if *decompress {
 		data, _ = ioutil.ReadAll(input)
-		data, err = lz4.Decode(nil, data)
-		if err != nil {
-			fmt.Println("Failed to decode:", err)
-			return
-		}
+		data, _ = lz4.Decode(nil, data)
 	} else {
 		data, _ = ioutil.ReadAll(input)
-		data, err = lz4.Encode(nil, data)
-		if err != nil {
-			fmt.Println("Failed to encode:", err)
-			return
-		}
+		data, _ = lz4.Encode(nil, data)
 	}
 
 	err = ioutil.WriteFile(args[1], data, 0644)

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/reader.go

@@ -1,199 +1,194 @@
-/*
- * Copyright 2011-2012 Branimir Karadzic. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without modification,
- * are permitted provided that the following conditions are met:
- *
- *    1. Redistributions of source code must retain the above copyright notice, this
- *       list of conditions and the following disclaimer.
- *
- *    2. Redistributions in binary form must reproduce the above copyright notice,
- *       this list of conditions and the following disclaimer in the documentation
- *       and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
- * SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
- * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-package lz4
-
-import (
-	"encoding/binary"
-	"errors"
-	"io"
-)
-
-var (
-	// ErrCorrupt indicates the input was corrupt
-	ErrCorrupt = errors.New("corrupt input")
-)
-
-const (
-	mlBits  = 4
-	mlMask  = (1 << mlBits) - 1
-	runBits = 8 - mlBits
-	runMask = (1 << runBits) - 1
-)
-
-type decoder struct {
-	src  []byte
-	dst  []byte
-	spos uint32
-	dpos uint32
-	ref  uint32
-}
-
-func (d *decoder) readByte() (uint8, error) {
-	if int(d.spos) == len(d.src) {
-		return 0, io.EOF
-	}
-	b := d.src[d.spos]
-	d.spos++
-	return b, nil
-}
-
-func (d *decoder) getLen() (uint32, error) {
-
-	length := uint32(0)
-	ln, err := d.readByte()
-	if err != nil {
-		return 0, ErrCorrupt
-	}
-	for ln == 255 {
-		length += 255
-		ln, err = d.readByte()
-		if err != nil {
-			return 0, ErrCorrupt
-		}
-	}
-	length += uint32(ln)
-
-	return length, nil
-}
-
-func (d *decoder) cp(length, decr uint32) {
-
-	if int(d.ref+length) < int(d.dpos) {
-		copy(d.dst[d.dpos:], d.dst[d.ref:d.ref+length])
-	} else {
-		for ii := uint32(0); ii < length; ii++ {
-			d.dst[d.dpos+ii] = d.dst[d.ref+ii]
-		}
-	}
-	d.dpos += length
-	d.ref += length - decr
-}
-
-func (d *decoder) finish(err error) error {
-	if err == io.EOF {
-		return nil
-	}
-
-	return err
-}
-
-// Decode returns the decoded form of src.  The returned slice may be a
-// subslice of dst if it was large enough to hold the entire decoded block.
-func Decode(dst, src []byte) ([]byte, error) {
-
-	if len(src) < 4 {
-		return nil, ErrCorrupt
-	}
-
-	uncompressedLen := binary.LittleEndian.Uint32(src)
-
-	if uncompressedLen == 0 {
-		return nil, nil
-	}
-
-	if uncompressedLen > MaxInputSize {
-		return nil, ErrTooLarge
-	}
-
-	if dst == nil || len(dst) < int(uncompressedLen) {
-		dst = make([]byte, uncompressedLen)
-	}
-
-	d := decoder{src: src, dst: dst[:uncompressedLen], spos: 4}
-
-	decr := []uint32{0, 3, 2, 3}
-
-	for {
-		code, err := d.readByte()
-		if err != nil {
-			return d.dst, d.finish(err)
-		}
-
-		length := uint32(code >> mlBits)
-		if length == runMask {
-			ln, err := d.getLen()
-			if err != nil {
-				return nil, ErrCorrupt
-			}
-			length += ln
-		}
-
-		if int(d.spos+length) > len(d.src) || int(d.dpos+length) > len(d.dst) {
-			return nil, ErrCorrupt
-		}
-
-		for ii := uint32(0); ii < length; ii++ {
-			d.dst[d.dpos+ii] = d.src[d.spos+ii]
-		}
-
-		d.spos += length
-		d.dpos += length
-
-		if int(d.spos) == len(d.src) {
-			return d.dst, nil
-		}
-
-		if int(d.spos+2) >= len(d.src) {
-			return nil, ErrCorrupt
-		}
-
-		back := uint32(d.src[d.spos]) | uint32(d.src[d.spos+1])<<8
-
-		if back > d.dpos {
-			return nil, ErrCorrupt
-		}
-
-		d.spos += 2
-		d.ref = d.dpos - back
-
-		length = uint32(code & mlMask)
-		if length == mlMask {
-			ln, err := d.getLen()
-			if err != nil {
-				return nil, ErrCorrupt
-			}
-			length += ln
-		}
-
-		literal := d.dpos - d.ref
-
-		if literal < 4 {
-			if int(d.dpos+4) > len(d.dst) {
-				return nil, ErrCorrupt
-			}
-
-			d.cp(4, decr[literal])
-		} else {
-			length += 4
-		}
-
-		if d.dpos+length > uncompressedLen {
-			return nil, ErrCorrupt
-		}
-
-		d.cp(length, 0)
-	}
-}
+/*
+ * Copyright 2011-2012 Branimir Karadzic. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice, this
+ *       list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above copyright notice,
+ *       this list of conditions and the following disclaimer in the documentation
+ *       and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package lz4
+
+import (
+	"encoding/binary"
+	"errors"
+	"io"
+)
+
+var (
+	// ErrCorrupt indicates the input was corrupt
+	ErrCorrupt = errors.New("corrupt input")
+)
+
+const (
+	mlBits  = 4
+	mlMask  = (1 << mlBits) - 1
+	runBits = 8 - mlBits
+	runMask = (1 << runBits) - 1
+)
+
+type decoder struct {
+	src  []byte
+	dst  []byte
+	spos uint32
+	dpos uint32
+	ref  uint32
+}
+
+func (d *decoder) readByte() (uint8, error) {
+	if int(d.spos) == len(d.src) {
+		return 0, io.EOF
+	}
+	b := d.src[d.spos]
+	d.spos++
+	return b, nil
+}
+
+func (d *decoder) getLen() (uint32, error) {
+
+	length := uint32(0)
+	ln, err := d.readByte()
+	if err != nil {
+		return 0, ErrCorrupt
+	}
+	for ln == 255 {
+		length += 255
+		ln, err = d.readByte()
+		if err != nil {
+			return 0, ErrCorrupt
+		}
+	}
+	length += uint32(ln)
+
+	return length, nil
+}
+
+func (d *decoder) cp(length, decr uint32) {
+
+	if int(d.ref+length) < int(d.dpos) {
+		copy(d.dst[d.dpos:], d.dst[d.ref:d.ref+length])
+	} else {
+		for ii := uint32(0); ii < length; ii++ {
+			d.dst[d.dpos+ii] = d.dst[d.ref+ii]
+		}
+	}
+	d.dpos += length
+	d.ref += length - decr
+}
+
+func (d *decoder) finish(err error) error {
+	if err == io.EOF {
+		return nil
+	}
+
+	return err
+}
+
+// Decode returns the decoded form of src.  The returned slice may be a
+// subslice of dst if it was large enough to hold the entire decoded block.
+func Decode(dst, src []byte) ([]byte, error) {
+
+	if len(src) < 4 {
+		return nil, ErrCorrupt
+	}
+
+	uncompressedLen := binary.LittleEndian.Uint32(src)
+
+	if uncompressedLen == 0 {
+		return nil, nil
+	}
+
+	if uncompressedLen > MaxInputSize {
+		return nil, ErrTooLarge
+	}
+
+	if dst == nil || len(dst) < int(uncompressedLen) {
+		dst = make([]byte, uncompressedLen)
+	}
+
+	d := decoder{src: src, dst: dst[:uncompressedLen], spos: 4}
+
+	decr := []uint32{0, 3, 2, 3}
+
+	for {
+		code, err := d.readByte()
+		if err != nil {
+			return d.dst, d.finish(err)
+		}
+
+		length := uint32(code >> mlBits)
+		if length == runMask {
+			ln, err := d.getLen()
+			if err != nil {
+				return nil, ErrCorrupt
+			}
+			length += ln
+		}
+
+		if int(d.spos+length) > len(d.src) {
+			return nil, ErrCorrupt
+		}
+
+		for ii := uint32(0); ii < length; ii++ {
+			d.dst[d.dpos+ii] = d.src[d.spos+ii]
+		}
+
+		d.spos += length
+		d.dpos += length
+
+		if int(d.spos) == len(d.src) {
+			return d.dst, nil
+		}
+
+		if int(d.spos+2) >= len(d.src) {
+			return nil, ErrCorrupt
+		}
+
+		back := uint32(d.src[d.spos]) | uint32(d.src[d.spos+1])<<8
+
+		if back > d.dpos {
+			return nil, ErrCorrupt
+		}
+
+		d.spos += 2
+		d.ref = d.dpos - back
+
+		length = uint32(code & mlMask)
+		if length == mlMask {
+			ln, err := d.getLen()
+			if err != nil {
+				return nil, ErrCorrupt
+			}
+			length += ln
+		}
+
+		literal := d.dpos - d.ref
+		if literal < 4 {
+			d.cp(4, decr[literal])
+		} else {
+			length += 4
+		}
+
+		if d.dpos+length > uncompressedLen {
+			return nil, ErrCorrupt
+		}
+
+		d.cp(length, 0)
+	}
+}

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/writer.go

@@ -1,190 +1,188 @@
-/*
- * Copyright 2011-2012 Branimir Karadzic. All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without modification,
- * are permitted provided that the following conditions are met:
- *
- *    1. Redistributions of source code must retain the above copyright notice, this
- *       list of conditions and the following disclaimer.
- *
- *    2. Redistributions in binary form must reproduce the above copyright notice,
- *       this list of conditions and the following disclaimer in the documentation
- *       and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
- * SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
- * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
- * THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-package lz4
-
-import (
-	"encoding/binary"
-	"errors"
-)
-
-const (
-	minMatch              = 4
-	hashLog               = 17
-	hashTableSize         = 1 << hashLog
-	hashShift             = (minMatch * 8) - hashLog
-	incompressible uint32 = 128
-	uninitHash            = 0x88888888
-
-	// MaxInputSize is the largest buffer than can be compressed in a single block
-	MaxInputSize = 0x7E000000
-)
-
-var (
-	// ErrTooLarge indicates the input buffer was too large
-	ErrTooLarge = errors.New("input too large")
-)
-
-type encoder struct {
-	src       []byte
-	dst       []byte
-	hashTable []uint32
-	pos       uint32
-	anchor    uint32
-	dpos      uint32
-}
-
-// CompressBound returns the maximum length of a lz4 block, given it's uncompressed length
-func CompressBound(isize int) int {
-	if isize > MaxInputSize {
-		return 0
-	}
-	return isize + ((isize) / 255) + 16 + 4
-}
-
-func (e *encoder) writeLiterals(length, mlLen, pos uint32) {
-
-	ln := length
-
-	var code byte
-	if ln > runMask-1 {
-		code = runMask
-	} else {
-		code = byte(ln)
-	}
-
-	if mlLen > mlMask-1 {
-		e.dst[e.dpos] = (code << mlBits) + byte(mlMask)
-	} else {
-		e.dst[e.dpos] = (code << mlBits) + byte(mlLen)
-	}
-	e.dpos++
-
-	if code == runMask {
-		ln -= runMask
-		for ; ln > 254; ln -= 255 {
-			e.dst[e.dpos] = 255
-			e.dpos++
-		}
-
-		e.dst[e.dpos] = byte(ln)
-		e.dpos++
-	}
-
-	for ii := uint32(0); ii < length; ii++ {
-		e.dst[e.dpos+ii] = e.src[pos+ii]
-	}
-
-	e.dpos += length
-}
-
-// Encode returns the encoded form of src.  The returned array may be a
-// sub-slice of dst if it was large enough to hold the entire output.
-func Encode(dst, src []byte) ([]byte, error) {
-
-	if len(src) >= MaxInputSize {
-		return nil, ErrTooLarge
-	}
-
-	if n := CompressBound(len(src)); len(dst) < n {
-		dst = make([]byte, n)
-	}
-
-	e := encoder{src: src, dst: dst, hashTable: make([]uint32, hashTableSize)}
-
-	binary.LittleEndian.PutUint32(dst, uint32(len(src)))
-	e.dpos = 4
-
-	var (
-		step  uint32 = 1
-		limit        = incompressible
-	)
-
-	for {
-		if int(e.pos)+12 >= len(e.src) {
-			e.writeLiterals(uint32(len(e.src))-e.anchor, 0, e.anchor)
-			return e.dst[:e.dpos], nil
-		}
-
-		sequence := uint32(e.src[e.pos+3])<<24 | uint32(e.src[e.pos+2])<<16 | uint32(e.src[e.pos+1])<<8 | uint32(e.src[e.pos+0])
-
-		hash := (sequence * 2654435761) >> hashShift
-		ref := e.hashTable[hash] + uninitHash
-		e.hashTable[hash] = e.pos - uninitHash
-
-		if ((e.pos-ref)>>16) != 0 || uint32(e.src[ref+3])<<24|uint32(e.src[ref+2])<<16|uint32(e.src[ref+1])<<8|uint32(e.src[ref+0]) != sequence {
-			if e.pos-e.anchor > limit {
-				limit <<= 1
-				step += 1 + (step >> 2)
-			}
-			e.pos += step
-			continue
-		}
-
-		if step > 1 {
-			e.hashTable[hash] = ref - uninitHash
-			e.pos -= step - 1
-			step = 1
-			continue
-		}
-		limit = incompressible
-
-		ln := e.pos - e.anchor
-		back := e.pos - ref
-
-		anchor := e.anchor
-
-		e.pos += minMatch
-		ref += minMatch
-		e.anchor = e.pos
-
-		for int(e.pos) < len(e.src)-5 && e.src[e.pos] == e.src[ref] {
-			e.pos++
-			ref++
-		}
-
-		mlLen := e.pos - e.anchor
-
-		e.writeLiterals(ln, mlLen, anchor)
-		e.dst[e.dpos] = uint8(back)
-		e.dst[e.dpos+1] = uint8(back >> 8)
-		e.dpos += 2
-
-		if mlLen > mlMask-1 {
-			mlLen -= mlMask
-			for mlLen > 254 {
-				mlLen -= 255
-
-				e.dst[e.dpos] = 255
-				e.dpos++
-			}
-
-			e.dst[e.dpos] = byte(mlLen)
-			e.dpos++
-		}
-
-		e.anchor = e.pos
-	}
-}
+/*
+ * Copyright 2011-2012 Branimir Karadzic. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *    1. Redistributions of source code must retain the above copyright notice, this
+ *       list of conditions and the following disclaimer.
+ *
+ *    2. Redistributions in binary form must reproduce the above copyright notice,
+ *       this list of conditions and the following disclaimer in the documentation
+ *       and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT
+ * SHALL COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+ * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package lz4
+
+import "encoding/binary"
+import "errors"
+
+const (
+	minMatch              = 4
+	hashLog               = 17
+	hashTableSize         = 1 << hashLog
+	hashShift             = (minMatch * 8) - hashLog
+	incompressible uint32 = 128
+	uninitHash            = 0x88888888
+
+	// MaxInputSize is the largest buffer than can be compressed in a single block
+	MaxInputSize = 0x7E000000
+)
+
+var (
+	// ErrTooLarge indicates the input buffer was too large
+	ErrTooLarge = errors.New("input too large")
+)
+
+type encoder struct {
+	src       []byte
+	dst       []byte
+	hashTable []uint32
+	pos       uint32
+	anchor    uint32
+	dpos      uint32
+}
+
+// CompressBound returns the maximum length of a lz4 block, given it's uncompressed length
+func CompressBound(isize int) int {
+	if isize > MaxInputSize {
+		return 0
+	}
+	return isize + ((isize) / 255) + 16 + 4
+}
+
+func (e *encoder) writeLiterals(length, mlLen, pos uint32) {
+
+	ln := length
+
+	var code byte
+	if ln > runMask-1 {
+		code = runMask
+	} else {
+		code = byte(ln)
+	}
+
+	if mlLen > mlMask-1 {
+		e.dst[e.dpos] = (code << mlBits) + byte(mlMask)
+	} else {
+		e.dst[e.dpos] = (code << mlBits) + byte(mlLen)
+	}
+	e.dpos++
+
+	if code == runMask {
+		ln -= runMask
+		for ; ln > 254; ln -= 255 {
+			e.dst[e.dpos] = 255
+			e.dpos++
+		}
+
+		e.dst[e.dpos] = byte(ln)
+		e.dpos++
+	}
+
+	for ii := uint32(0); ii < length; ii++ {
+		e.dst[e.dpos+ii] = e.src[pos+ii]
+	}
+
+	e.dpos += length
+}
+
+// Encode returns the encoded form of src.  The returned array may be a
+// sub-slice of dst if it was large enough to hold the entire output.
+func Encode(dst, src []byte) ([]byte, error) {
+
+	if len(src) >= MaxInputSize {
+		return nil, ErrTooLarge
+	}
+
+	if n := CompressBound(len(src)); len(dst) < n {
+		dst = make([]byte, n)
+	}
+
+	e := encoder{src: src, dst: dst, hashTable: make([]uint32, hashTableSize)}
+
+	binary.LittleEndian.PutUint32(dst, uint32(len(src)))
+	e.dpos = 4
+
+	var (
+		step  uint32 = 1
+		limit        = incompressible
+	)
+
+	for {
+		if int(e.pos)+4 >= len(e.src) {
+			e.writeLiterals(uint32(len(e.src))-e.anchor, 0, e.anchor)
+			return e.dst[:e.dpos], nil
+		}
+
+		sequence := uint32(e.src[e.pos+3])<<24 | uint32(e.src[e.pos+2])<<16 | uint32(e.src[e.pos+1])<<8 | uint32(e.src[e.pos+0])
+
+		hash := (sequence * 2654435761) >> hashShift
+		ref := e.hashTable[hash] + uninitHash
+		e.hashTable[hash] = e.pos - uninitHash
+
+		if ((e.pos-ref)>>16) != 0 || uint32(e.src[ref+3])<<24|uint32(e.src[ref+2])<<16|uint32(e.src[ref+1])<<8|uint32(e.src[ref+0]) != sequence {
+			if e.pos-e.anchor > limit {
+				limit <<= 1
+				step += 1 + (step >> 2)
+			}
+			e.pos += step
+			continue
+		}
+
+		if step > 1 {
+			e.hashTable[hash] = ref - uninitHash
+			e.pos -= step - 1
+			step = 1
+			continue
+		}
+		limit = incompressible
+
+		ln := e.pos - e.anchor
+		back := e.pos - ref
+
+		anchor := e.anchor
+
+		e.pos += minMatch
+		ref += minMatch
+		e.anchor = e.pos
+
+		for int(e.pos) < len(e.src) && e.src[e.pos] == e.src[ref] {
+			e.pos++
+			ref++
+		}
+
+		mlLen := e.pos - e.anchor
+
+		e.writeLiterals(ln, mlLen, anchor)
+		e.dst[e.dpos] = uint8(back)
+		e.dst[e.dpos+1] = uint8(back >> 8)
+		e.dpos += 2
+
+		if mlLen > mlMask-1 {
+			mlLen -= mlMask
+			for mlLen > 254 {
+				mlLen -= 255
+
+				e.dst[e.dpos] = 255
+				e.dpos++
+			}
+
+			e.dst[e.dpos] = byte(mlLen)
+			e.dpos++
+		}
+
+		e.anchor = e.pos
+	}
+}

Godeps/_workspace/src/github.com/calmh/du/LICENSE

@@ -1,24 +0,0 @@
-This is free and unencumbered software released into the public domain.
-
-Anyone is free to copy, modify, publish, use, compile, sell, or
-distribute this software, either in source code form or as a compiled
-binary, for any purpose, commercial or non-commercial, and by any
-means.
-
-In jurisdictions that recognize copyright laws, the author or authors
-of this software dedicate any and all copyright interest in the
-software to the public domain. We make this dedication for the benefit
-of the public at large and to the detriment of our heirs and
-successors. We intend this dedication to be an overt act of
-relinquishment in perpetuity of all present and future rights to this
-software under copyright law.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
-OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-OTHER DEALINGS IN THE SOFTWARE.
-
-For more information, please refer to <http://unlicense.org>

Godeps/_workspace/src/github.com/calmh/du/README.md

@@ -1,14 +0,0 @@
-du
-==
-
-Get total and available disk space on a given volume.
-
-Documentation
--------------
-
-http://godoc.org/github.com/calmh/du
-
-License
--------
-
-Public Domain

Godeps/_workspace/src/github.com/calmh/du/cmd/du/main.go

@@ -1,21 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"log"
-	"os"
-
-	"github.com/calmh/du"
-)
-
-var KB = int64(1024)
-
-func main() {
-	usage, err := du.Get(os.Args[1])
-	if err != nil {
-		log.Fatal(err)
-	}
-	fmt.Println("Free:", usage.FreeBytes/(KB*KB), "MiB")
-	fmt.Println("Available:", usage.AvailBytes/(KB*KB), "MiB")
-	fmt.Println("Size:", usage.TotalBytes/(KB*KB), "MiB")
-}

Godeps/_workspace/src/github.com/calmh/du/diskusage.go

@@ -1,8 +0,0 @@
-package du
-
-// Usage holds information about total and available storage on a volume.
-type Usage struct {
-	TotalBytes int64 // Size of volume
-	FreeBytes  int64 // Unused size
-	AvailBytes int64 // Available to a non-privileged user
-}

Godeps/_workspace/src/github.com/calmh/du/diskusage_posix.go

@@ -1,24 +0,0 @@
-// +build !windows,!netbsd,!openbsd,!solaris
-
-package du
-
-import (
-	"path/filepath"
-	"syscall"
-)
-
-// Get returns the Usage of a given path, or an error if usage data is
-// unavailable.
-func Get(path string) (Usage, error) {
-	var stat syscall.Statfs_t
-	err := syscall.Statfs(filepath.Clean(path), &stat)
-	if err != nil {
-		return Usage{}, err
-	}
-	u := Usage{
-		FreeBytes:  int64(stat.Bfree) * int64(stat.Bsize),
-		TotalBytes: int64(stat.Blocks) * int64(stat.Bsize),
-		AvailBytes: int64(stat.Bavail) * int64(stat.Bsize),
-	}
-	return u, nil
-}

Godeps/_workspace/src/github.com/calmh/du/diskusage_unsupported.go

@@ -1,13 +0,0 @@
-// +build netbsd openbsd solaris
-
-package du
-
-import "errors"
-
-var ErrUnsupported = errors.New("unsupported platform")
-
-// Get returns the Usage of a given path, or an error if usage data is
-// unavailable.
-func Get(path string) (Usage, error) {
-	return Usage{}, ErrUnsupported
-}

Godeps/_workspace/src/github.com/calmh/du/diskusage_windows.go

@@ -1,27 +0,0 @@
-package du
-
-import (
-	"syscall"
-	"unsafe"
-)
-
-// Get returns the Usage of a given path, or an error if usage data is
-// unavailable.
-func Get(path string) (Usage, error) {
-	h := syscall.MustLoadDLL("kernel32.dll")
-	c := h.MustFindProc("GetDiskFreeSpaceExW")
-
-	var u Usage
-
-	ret, _, err := c.Call(
-		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
-		uintptr(unsafe.Pointer(&u.FreeBytes)),
-		uintptr(unsafe.Pointer(&u.TotalBytes)),
-		uintptr(unsafe.Pointer(&u.AvailBytes)))
-
-	if ret == 0 {
-		return u, err
-	}
-
-	return u, nil
-}

Godeps/_workspace/src/github.com/calmh/luhn/LICENSE

@@ -1,19 +0,0 @@
-Copyright (C) 2014 Jakob Borg
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-- The above copyright notice and this permission notice shall be included in
-  all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

Godeps/_workspace/src/github.com/calmh/xdr/.travis.yml

@@ -4,7 +4,7 @@ go:
 
 install:
 - export PATH=$PATH:$HOME/gopath/bin
-- go get golang.org/x/tools/cover
+- go get code.google.com/p/go.tools/cmd/cover
 - go get github.com/mattn/goveralls
 
 script:

Godeps/_workspace/src/github.com/calmh/xdr/README.md

@@ -1,7 +1,7 @@
 xdr
 ===
 
-[![Build Status](https://img.shields.io/circleci/project/calmh/xdr.svg?style=flat-square)](https://circleci.com/gh/calmh/xdr)
+[![Build Status](https://img.shields.io/travis/calmh/xdr.svg?style=flat)](https://travis-ci.org/calmh/xdr)
 [![Coverage Status](https://img.shields.io/coveralls/calmh/xdr.svg?style=flat)](https://coveralls.io/r/calmh/xdr?branch=master)
 [![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat)](http://godoc.org/github.com/calmh/xdr)
 [![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://opensource.org/licenses/MIT)

Godeps/_workspace/src/github.com/calmh/xdr/bench_test.go

@@ -33,15 +33,11 @@ var s = XDRBenchStruct{
 	S0:  "Hello World! String one.",
 	S1:  "Hello World! String two.",
 }
-var e []byte
-
-func init() {
-	e, _ = s.MarshalXDR()
-}
+var e = s.MarshalXDR()
 
 func BenchmarkThisMarshal(b *testing.B) {
 	for i := 0; i < b.N; i++ {
-		res, _ = s.MarshalXDR()
+		res = s.MarshalXDR()
 	}
 }
 
@@ -67,7 +63,7 @@ func BenchmarkThisEncode(b *testing.B) {
 func BenchmarkThisEncoder(b *testing.B) {
 	w := xdr.NewWriter(ioutil.Discard)
 	for i := 0; i < b.N; i++ {
-		_, err := s.EncodeXDRInto(w)
+		_, err := s.encodeXDR(w)
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -108,7 +104,7 @@ func BenchmarkThisDecoder(b *testing.B) {
 	r := xdr.NewReader(rr)
 	var t XDRBenchStruct
 	for i := 0; i < b.N; i++ {
-		err := t.DecodeXDRFrom(r)
+		err := t.decodeXDR(r)
 		if err != nil {
 			b.Fatal(err)
 		}

Godeps/_workspace/src/github.com/calmh/xdr/bench_xdr_test.go

@@ -26,9 +26,7 @@ XDRBenchStruct Structure:
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |            0x0000             |              I3               |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                        uint8 Structure                        \
-/                                                               /
+|                             uint8                             |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |                         Length of Bs0                         |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -71,40 +69,32 @@ struct XDRBenchStruct {
 
 func (o XDRBenchStruct) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.EncodeXDRInto(xw)
+	return o.encodeXDR(xw)
 }
 
-func (o XDRBenchStruct) MarshalXDR() ([]byte, error) {
+func (o XDRBenchStruct) MarshalXDR() []byte {
 	return o.AppendXDR(make([]byte, 0, 128))
 }
 
-func (o XDRBenchStruct) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o XDRBenchStruct) AppendXDR(bs []byte) ([]byte, error) {
+func (o XDRBenchStruct) AppendXDR(bs []byte) []byte {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.EncodeXDRInto(xw)
-	return []byte(aw), err
+	o.encodeXDR(xw)
+	return []byte(aw)
 }
 
-func (o XDRBenchStruct) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+func (o XDRBenchStruct) encodeXDR(xw *xdr.Writer) (int, error) {
 	xw.WriteUint64(o.I1)
 	xw.WriteUint32(o.I2)
 	xw.WriteUint16(o.I3)
 	xw.WriteUint8(o.I4)
-	if l := len(o.Bs0); l > 128 {
-		return xw.Tot(), xdr.ElementSizeExceeded("Bs0", l, 128)
+	if len(o.Bs0) > 128 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
 	}
 	xw.WriteBytes(o.Bs0)
 	xw.WriteBytes(o.Bs1)
-	if l := len(o.S0); l > 128 {
-		return xw.Tot(), xdr.ElementSizeExceeded("S0", l, 128)
+	if len(o.S0) > 128 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
 	}
 	xw.WriteString(o.S0)
 	xw.WriteString(o.S1)
@@ -113,16 +103,16 @@ func (o XDRBenchStruct) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 
 func (o *XDRBenchStruct) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.DecodeXDRFrom(xr)
+	return o.decodeXDR(xr)
 }
 
 func (o *XDRBenchStruct) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.DecodeXDRFrom(xr)
+	return o.decodeXDR(xr)
 }
 
-func (o *XDRBenchStruct) DecodeXDRFrom(xr *xdr.Reader) error {
+func (o *XDRBenchStruct) decodeXDR(xr *xdr.Reader) error {
 	o.I1 = xr.ReadUint64()
 	o.I2 = xr.ReadUint32()
 	o.I3 = xr.ReadUint16()
@@ -157,45 +147,37 @@ struct repeatReader {
 
 func (o repeatReader) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.EncodeXDRInto(xw)
+	return o.encodeXDR(xw)
 }
 
-func (o repeatReader) MarshalXDR() ([]byte, error) {
+func (o repeatReader) MarshalXDR() []byte {
 	return o.AppendXDR(make([]byte, 0, 128))
 }
 
-func (o repeatReader) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o repeatReader) AppendXDR(bs []byte) ([]byte, error) {
+func (o repeatReader) AppendXDR(bs []byte) []byte {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.EncodeXDRInto(xw)
-	return []byte(aw), err
+	o.encodeXDR(xw)
+	return []byte(aw)
 }
 
-func (o repeatReader) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+func (o repeatReader) encodeXDR(xw *xdr.Writer) (int, error) {
 	xw.WriteBytes(o.data)
 	return xw.Tot(), xw.Error()
 }
 
 func (o *repeatReader) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.DecodeXDRFrom(xr)
+	return o.decodeXDR(xr)
 }
 
 func (o *repeatReader) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.DecodeXDRFrom(xr)
+	return o.decodeXDR(xr)
 }
 
-func (o *repeatReader) DecodeXDRFrom(xr *xdr.Reader) error {
+func (o *repeatReader) decodeXDR(xr *xdr.Reader) error {
 	o.data = xr.ReadBytes()
 	return xr.Error()
 }

Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go

@@ -11,8 +11,6 @@ import (
 	"go/format"
 	"go/parser"
 	"go/token"
-	"io"
-	"log"
 	"os"
 	"regexp"
 	"strconv"
@@ -28,7 +26,6 @@ type fieldInfo struct {
 	Encoder   string // the encoder name, i.e. "Uint64" for Read/WriteUint64
 	Convert   string // what to convert to when encoding, i.e. "uint64"
 	Max       int    // max size for slices and strings
-	Submax    int    // max size for strings inside slices
 }
 
 type structInfo struct {
@@ -53,50 +50,42 @@ import (
 var encodeTpl = template.Must(template.New("encoder").Parse(`
 func (o {{.TypeName}}) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.EncodeXDRInto(xw)
+	return o.encodeXDR(xw)
 }//+n
 
-func (o {{.TypeName}}) MarshalXDR() ([]byte, error) {
+func (o {{.TypeName}}) MarshalXDR() []byte {
 	return o.AppendXDR(make([]byte, 0, 128))
 }//+n
 
-func (o {{.TypeName}}) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}//+n
-
-func (o {{.TypeName}}) AppendXDR(bs []byte) ([]byte, error) {
+func (o {{.TypeName}}) AppendXDR(bs []byte) []byte {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.EncodeXDRInto(xw)
-	return []byte(aw), err
+	o.encodeXDR(xw)
+	return []byte(aw)
 }//+n
 
-func (o {{.TypeName}}) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
 	{{range $fieldInfo := .Fields}}
 		{{if not $fieldInfo.IsSlice}}
 			{{if ne $fieldInfo.Convert ""}}
 				xw.Write{{$fieldInfo.Encoder}}({{$fieldInfo.Convert}}(o.{{$fieldInfo.Name}}))
 			{{else if $fieldInfo.IsBasic}}
 				{{if ge $fieldInfo.Max 1}}
-					if l := len(o.{{$fieldInfo.Name}}); l > {{$fieldInfo.Max}} {
-						return xw.Tot(), xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", l, {{$fieldInfo.Max}})
+					if len(o.{{$fieldInfo.Name}}) > {{$fieldInfo.Max}} {
+						return xw.Tot(), xdr.ErrElementSizeExceeded
 					}
 				{{end}}
 				xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}})
 			{{else}}
-				_, err := o.{{$fieldInfo.Name}}.EncodeXDRInto(xw)
+				_, err := o.{{$fieldInfo.Name}}.encodeXDR(xw)
 				if err != nil {
 					return xw.Tot(), err
 				}
 			{{end}}
 		{{else}}
 			{{if ge $fieldInfo.Max 1}}
-				if l := len(o.{{$fieldInfo.Name}}); l > {{$fieldInfo.Max}} {
-					return xw.Tot(), xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", l, {{$fieldInfo.Max}})
+				if len(o.{{$fieldInfo.Name}}) > {{$fieldInfo.Max}} {
+					return xw.Tot(), xdr.ErrElementSizeExceeded
 				}
 			{{end}}
 			xw.WriteUint32(uint32(len(o.{{$fieldInfo.Name}})))
@@ -106,7 +95,7 @@ func (o {{.TypeName}}) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 			{{else if $fieldInfo.IsBasic}}
 				xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}}[i])
 			{{else}}
-				_, err := o.{{$fieldInfo.Name}}[i].EncodeXDRInto(xw)
+				_, err := o.{{$fieldInfo.Name}}[i].encodeXDR(xw)
 				if err != nil {
 					return xw.Tot(), err
 				}
@@ -119,16 +108,16 @@ func (o {{.TypeName}}) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 
 func (o *{{.TypeName}}) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.DecodeXDRFrom(xr)
+	return o.decodeXDR(xr)
 }//+n
 
 func (o *{{.TypeName}}) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.DecodeXDRFrom(xr)
+	return o.decodeXDR(xr)
 }//+n
 
-func (o *{{.TypeName}}) DecodeXDRFrom(xr *xdr.Reader) error {
+func (o *{{.TypeName}}) decodeXDR(xr *xdr.Reader) error {
 	{{range $fieldInfo := .Fields}}
 		{{if not $fieldInfo.IsSlice}}
 			{{if ne $fieldInfo.Convert ""}}
@@ -140,16 +129,13 @@ func (o *{{.TypeName}}) DecodeXDRFrom(xr *xdr.Reader) error {
 					o.{{$fieldInfo.Name}} = xr.Read{{$fieldInfo.Encoder}}()
 				{{end}}
 			{{else}}
-				(&o.{{$fieldInfo.Name}}).DecodeXDRFrom(xr)
+				(&o.{{$fieldInfo.Name}}).decodeXDR(xr)
 			{{end}}
 		{{else}}
 			_{{$fieldInfo.Name}}Size := int(xr.ReadUint32())
-			if _{{$fieldInfo.Name}}Size < 0 {
-				return xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", _{{$fieldInfo.Name}}Size, {{$fieldInfo.Max}})
-			}
 			{{if ge $fieldInfo.Max 1}}
 				if _{{$fieldInfo.Name}}Size > {{$fieldInfo.Max}} {
-					return xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", _{{$fieldInfo.Name}}Size, {{$fieldInfo.Max}})
+					return xdr.ErrElementSizeExceeded
 				}
 			{{end}}
 			o.{{$fieldInfo.Name}} = make([]{{$fieldInfo.FieldType}}, _{{$fieldInfo.Name}}Size)
@@ -157,13 +143,9 @@ func (o *{{.TypeName}}) DecodeXDRFrom(xr *xdr.Reader) error {
 				{{if ne $fieldInfo.Convert ""}}
 					o.{{$fieldInfo.Name}}[i] = {{$fieldInfo.FieldType}}(xr.Read{{$fieldInfo.Encoder}}())
 				{{else if $fieldInfo.IsBasic}}
-					{{if ge $fieldInfo.Submax 1}}
-						o.{{$fieldInfo.Name}}[i] = xr.Read{{$fieldInfo.Encoder}}Max({{$fieldInfo.Submax}})
-					{{else}}
-						o.{{$fieldInfo.Name}}[i] = xr.Read{{$fieldInfo.Encoder}}()
-					{{end}}
+					o.{{$fieldInfo.Name}}[i] = xr.Read{{$fieldInfo.Encoder}}()
 				{{else}}
-					(&o.{{$fieldInfo.Name}}[i]).DecodeXDRFrom(xr)
+					(&o.{{$fieldInfo.Name}}[i]).decodeXDR(xr)
 				{{end}}
 			}
 		{{end}}
@@ -171,7 +153,7 @@ func (o *{{.TypeName}}) DecodeXDRFrom(xr *xdr.Reader) error {
 	return xr.Error()
 }`))
 
-var maxRe = regexp.MustCompile(`(?:\Wmax:)(\d+)(?:\s*,\s*(\d+))?`)
+var maxRe = regexp.MustCompile(`\Wmax:(\d+)`)
 
 type typeSet struct {
 	Type    string
@@ -203,15 +185,11 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 		}
 
 		fn := sf.Names[0].Name
-		var max1, max2 int
+		var max = 0
 		if sf.Comment != nil {
 			c := sf.Comment.List[0].Text
-			m := maxRe.FindStringSubmatch(c)
-			if len(m) >= 2 {
-				max1, _ = strconv.Atoi(m[1])
-			}
-			if len(m) >= 3 {
-				max2, _ = strconv.Atoi(m[2])
+			if m := maxRe.FindStringSubmatch(c); m != nil {
+				max, _ = strconv.Atoi(m[1])
 			}
 			if strings.Contains(c, "noencode") {
 				continue
@@ -229,16 +207,14 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 					FieldType: tn,
 					Encoder:   enc.Encoder,
 					Convert:   enc.Type,
-					Max:       max1,
-					Submax:    max2,
+					Max:       max,
 				}
 			} else {
 				f = fieldInfo{
 					Name:      fn,
 					IsBasic:   false,
 					FieldType: tn,
-					Max:       max1,
-					Submax:    max2,
+					Max:       max,
 				}
 			}
 
@@ -256,8 +232,7 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 					FieldType: tn,
 					Encoder:   enc.Encoder,
 					Convert:   enc.Type,
-					Max:       max1,
-					Submax:    max2,
+					Max:       max,
 				}
 			} else if enc, ok := xdrEncoders[tn]; ok {
 				f = fieldInfo{
@@ -267,26 +242,17 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 					FieldType: tn,
 					Encoder:   enc.Encoder,
 					Convert:   enc.Type,
-					Max:       max1,
-					Submax:    max2,
+					Max:       max,
 				}
 			} else {
 				f = fieldInfo{
 					Name:      fn,
+					IsBasic:   false,
 					IsSlice:   true,
 					FieldType: tn,
-					Max:       max1,
-					Submax:    max2,
+					Max:       max,
 				}
 			}
-
-		case *ast.SelectorExpr:
-			f = fieldInfo{
-				Name:      fn,
-				FieldType: ft.Sel.Name,
-				Max:       max1,
-				Submax:    max2,
-			}
 		}
 
 		fs = append(fs, f)
@@ -295,7 +261,7 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 	return fs
 }
 
-func generateCode(output io.Writer, s structInfo) {
+func generateCode(s structInfo) {
 	name := s.Name
 	fs := s.Fields
 
@@ -312,7 +278,7 @@ func generateCode(output io.Writer, s structInfo) {
 	if err != nil {
 		panic(err)
 	}
-	fmt.Fprintln(output, string(bs))
+	fmt.Println(string(bs))
 }
 
 func uncamelize(s string) string {
@@ -321,71 +287,69 @@ func uncamelize(s string) string {
 	})
 }
 
-func generateDiagram(output io.Writer, s structInfo) {
+func generateDiagram(s structInfo) {
 	sn := s.Name
 	fs := s.Fields
 
-	fmt.Fprintln(output, sn+" Structure:")
-	fmt.Fprintln(output)
-	fmt.Fprintln(output, " 0                   1                   2                   3")
-	fmt.Fprintln(output, " 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1")
+	fmt.Println(sn + " Structure:")
+	fmt.Println()
+	fmt.Println(" 0                   1                   2                   3")
+	fmt.Println(" 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1")
 	line := "+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+"
-	fmt.Fprintln(output, line)
+	fmt.Println(line)
 
 	for _, f := range fs {
 		tn := f.FieldType
+		sl := f.IsSlice
 		name := uncamelize(f.Name)
 
-		if f.IsSlice {
-			fmt.Fprintf(output, "| %s |\n", center("Number of "+name, 61))
-			fmt.Fprintln(output, line)
+		if sl {
+			fmt.Printf("| %s |\n", center("Number of "+name, 61))
+			fmt.Println(line)
 		}
 		switch tn {
 		case "bool":
-			fmt.Fprintf(output, "| %s |V|\n", center(name+" (V=0 or 1)", 59))
-			fmt.Fprintln(output, line)
-		case "int16", "uint16":
-			fmt.Fprintf(output, "| %s | %s |\n", center("0x0000", 29), center(name, 29))
-			fmt.Fprintln(output, line)
-		case "int32", "uint32":
-			fmt.Fprintf(output, "| %s |\n", center(name, 61))
-			fmt.Fprintln(output, line)
+			fmt.Printf("| %s |V|\n", center(name+" (V=0 or 1)", 59))
+			fmt.Println(line)
+		case "uint16":
+			fmt.Printf("| %s | %s |\n", center("0x0000", 29), center(name, 29))
+			fmt.Println(line)
+		case "uint32":
+			fmt.Printf("| %s |\n", center(name, 61))
+			fmt.Println(line)
 		case "int64", "uint64":
-			fmt.Fprintf(output, "| %-61s |\n", "")
-			fmt.Fprintf(output, "+ %s +\n", center(name+" (64 bits)", 61))
-			fmt.Fprintf(output, "| %-61s |\n", "")
-			fmt.Fprintln(output, line)
+			fmt.Printf("| %-61s |\n", "")
+			fmt.Printf("+ %s +\n", center(name+" (64 bits)", 61))
+			fmt.Printf("| %-61s |\n", "")
+			fmt.Println(line)
 		case "string", "byte": // XXX We assume slice of byte!
-			fmt.Fprintf(output, "| %s |\n", center("Length of "+name, 61))
-			fmt.Fprintln(output, line)
-			fmt.Fprintf(output, "/ %61s /\n", "")
-			fmt.Fprintf(output, "\\ %s \\\n", center(name+" (variable length)", 61))
-			fmt.Fprintf(output, "/ %61s /\n", "")
-			fmt.Fprintln(output, line)
+			fmt.Printf("| %s |\n", center("Length of "+name, 61))
+			fmt.Println(line)
+			fmt.Printf("/ %61s /\n", "")
+			fmt.Printf("\\ %s \\\n", center(name+" (variable length)", 61))
+			fmt.Printf("/ %61s /\n", "")
+			fmt.Println(line)
 		default:
-			if f.IsSlice {
+			if sl {
 				tn = "Zero or more " + tn + " Structures"
-				fmt.Fprintf(output, "/ %s /\n", center("", 61))
-				fmt.Fprintf(output, "\\ %s \\\n", center(tn, 61))
-				fmt.Fprintf(output, "/ %s /\n", center("", 61))
+				fmt.Printf("/ %s /\n", center("", 61))
+				fmt.Printf("\\ %s \\\n", center(tn, 61))
+				fmt.Printf("/ %s /\n", center("", 61))
 			} else {
-				tn = tn + " Structure"
-				fmt.Fprintf(output, "/ %s /\n", center("", 61))
-				fmt.Fprintf(output, "\\ %s \\\n", center(tn, 61))
-				fmt.Fprintf(output, "/ %s /\n", center("", 61))
+				fmt.Printf("| %s |\n", center(tn, 61))
 			}
-			fmt.Fprintln(output, line)
+			fmt.Println(line)
 		}
 	}
-	fmt.Fprintln(output)
-	fmt.Fprintln(output)
+	fmt.Println()
+	fmt.Println()
 }
 
-func generateXdr(output io.Writer, s structInfo) {
+func generateXdr(s structInfo) {
 	sn := s.Name
 	fs := s.Fields
 
-	fmt.Fprintf(output, "struct %s {\n", sn)
+	fmt.Printf("struct %s {\n", sn)
 
 	for _, f := range fs {
 		tn := f.FieldType
@@ -400,24 +364,22 @@ func generateXdr(output io.Writer, s structInfo) {
 		}
 
 		switch tn {
-		case "int16", "int32":
-			fmt.Fprintf(output, "\tint %s%s;\n", fn, suf)
 		case "uint16", "uint32":
-			fmt.Fprintf(output, "\tunsigned int %s%s;\n", fn, suf)
+			fmt.Printf("\tunsigned int %s%s;\n", fn, suf)
 		case "int64":
-			fmt.Fprintf(output, "\thyper %s%s;\n", fn, suf)
+			fmt.Printf("\thyper %s%s;\n", fn, suf)
 		case "uint64":
-			fmt.Fprintf(output, "\tunsigned hyper %s%s;\n", fn, suf)
+			fmt.Printf("\tunsigned hyper %s%s;\n", fn, suf)
 		case "string":
-			fmt.Fprintf(output, "\tstring %s<%s>;\n", fn, l)
+			fmt.Printf("\tstring %s<%s>;\n", fn, l)
 		case "byte":
-			fmt.Fprintf(output, "\topaque %s<%s>;\n", fn, l)
+			fmt.Printf("\topaque %s<%s>;\n", fn, l)
 		default:
-			fmt.Fprintf(output, "\t%s %s%s;\n", tn, fn, suf)
+			fmt.Printf("\t%s %s%s;\n", tn, fn, suf)
 		}
 	}
-	fmt.Fprintln(output, "}")
-	fmt.Fprintln(output)
+	fmt.Println("}")
+	fmt.Println()
 }
 
 func center(s string, w int) string {
@@ -448,35 +410,25 @@ func inspector(structs *[]structInfo) func(ast.Node) bool {
 }
 
 func main() {
-	outputFile := flag.String("o", "", "Output file, blank for stdout")
 	flag.Parse()
 	fname := flag.Arg(0)
 
 	fset := token.NewFileSet()
 	f, err := parser.ParseFile(fset, fname, nil, parser.ParseComments)
 	if err != nil {
-		log.Fatal(err)
+		panic(err)
 	}
 
 	var structs []structInfo
 	i := inspector(&structs)
 	ast.Inspect(f, i)
 
-	var output io.Writer = os.Stdout
-	if *outputFile != "" {
-		fd, err := os.Create(*outputFile)
-		if err != nil {
-			log.Fatal(err)
-		}
-		output = fd
-	}
-
-	headerTpl.Execute(output, map[string]string{"Package": f.Name.Name})
+	headerTpl.Execute(os.Stdout, map[string]string{"Package": f.Name.Name})
 	for _, s := range structs {
-		fmt.Fprintf(output, "\n/*\n\n")
-		generateDiagram(output, s)
-		generateXdr(output, s)
-		fmt.Fprintf(output, "*/\n")
-		generateCode(output, s)
+		fmt.Printf("\n/*\n\n")
+		generateDiagram(s)
+		generateXdr(s)
+		fmt.Printf("*/\n")
+		generateCode(s)
 	}
 }

Godeps/_workspace/src/github.com/calmh/xdr/encdec_test.go

@@ -24,19 +24,18 @@ type TestStruct struct {
 	UI32 uint32
 	I64  int64
 	UI64 uint64
-	BS   []byte // max:1024
-	S    string // max:1024
+	BS   []byte
+	S    string
 	C    Opaque
-	SS   []string // max:1024
 }
 
 type Opaque [32]byte
 
-func (u *Opaque) EncodeXDRInto(w *xdr.Writer) (int, error) {
+func (u *Opaque) encodeXDR(w *xdr.Writer) (int, error) {
 	return w.WriteRaw(u[:])
 }
 
-func (u *Opaque) DecodeXDRFrom(r *xdr.Reader) (int, error) {
+func (u *Opaque) decodeXDR(r *xdr.Reader) (int, error) {
 	return r.ReadRaw(u[:])
 }
 
@@ -50,12 +49,9 @@ func (Opaque) Generate(rand *rand.Rand, size int) reflect.Value {
 
 func TestEncDec(t *testing.T) {
 	fn := func(t0 TestStruct) bool {
-		bs, err := t0.MarshalXDR()
-		if err != nil {
-			t.Fatal(err)
-		}
+		bs := t0.MarshalXDR()
 		var t1 TestStruct
-		err = t1.UnmarshalXDR(bs)
+		err := t1.UnmarshalXDR(bs)
 		if err != nil {
 			t.Fatal(err)
 		}

Godeps/_workspace/src/github.com/calmh/xdr/encdec_xdr_test.go

@@ -18,23 +18,17 @@ TestStruct Structure:
  0                   1                   2                   3
  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                         int Structure                         \
-/                                                               /
+|                              int                              |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                        int8 Structure                         \
-/                                                               /
+|                             int8                              |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                        uint8 Structure                        \
-/                                                               /
+|                             uint8                             |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|            0x0000             |              I16              |
+|                             int16                             |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |            0x0000             |             UI16              |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                              I32                              |
+|                             int32                             |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |                             UI32                              |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -58,17 +52,7 @@ TestStruct Structure:
 \                      S (variable length)                      \
 /                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                       Opaque Structure                        \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                         Number of SS                          |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                         Length of SS                          |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                     SS (variable length)                      \
-/                                                               /
+|                            Opaque                             |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 
@@ -76,45 +60,36 @@ struct TestStruct {
 	int I;
 	int8 I8;
 	uint8 UI8;
-	int I16;
+	int16 I16;
 	unsigned int UI16;
-	int I32;
+	int32 I32;
 	unsigned int UI32;
 	hyper I64;
 	unsigned hyper UI64;
-	opaque BS<1024>;
-	string S<1024>;
+	opaque BS<>;
+	string S<>;
 	Opaque C;
-	string SS<1024>;
 }
 
 */
 
 func (o TestStruct) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.EncodeXDRInto(xw)
+	return o.encodeXDR(xw)
 }
 
-func (o TestStruct) MarshalXDR() ([]byte, error) {
+func (o TestStruct) MarshalXDR() []byte {
 	return o.AppendXDR(make([]byte, 0, 128))
 }
 
-func (o TestStruct) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o TestStruct) AppendXDR(bs []byte) ([]byte, error) {
+func (o TestStruct) AppendXDR(bs []byte) []byte {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.EncodeXDRInto(xw)
-	return []byte(aw), err
+	o.encodeXDR(xw)
+	return []byte(aw)
 }
 
-func (o TestStruct) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+func (o TestStruct) encodeXDR(xw *xdr.Writer) (int, error) {
 	xw.WriteUint64(uint64(o.I))
 	xw.WriteUint8(uint8(o.I8))
 	xw.WriteUint8(o.UI8)
@@ -124,40 +99,27 @@ func (o TestStruct) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	xw.WriteUint32(o.UI32)
 	xw.WriteUint64(uint64(o.I64))
 	xw.WriteUint64(o.UI64)
-	if l := len(o.BS); l > 1024 {
-		return xw.Tot(), xdr.ElementSizeExceeded("BS", l, 1024)
-	}
 	xw.WriteBytes(o.BS)
-	if l := len(o.S); l > 1024 {
-		return xw.Tot(), xdr.ElementSizeExceeded("S", l, 1024)
-	}
 	xw.WriteString(o.S)
-	_, err := o.C.EncodeXDRInto(xw)
+	_, err := o.C.encodeXDR(xw)
 	if err != nil {
 		return xw.Tot(), err
 	}
-	if l := len(o.SS); l > 1024 {
-		return xw.Tot(), xdr.ElementSizeExceeded("SS", l, 1024)
-	}
-	xw.WriteUint32(uint32(len(o.SS)))
-	for i := range o.SS {
-		xw.WriteString(o.SS[i])
-	}
 	return xw.Tot(), xw.Error()
 }
 
 func (o *TestStruct) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.DecodeXDRFrom(xr)
+	return o.decodeXDR(xr)
 }
 
 func (o *TestStruct) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.DecodeXDRFrom(xr)
+	return o.decodeXDR(xr)
 }
 
-func (o *TestStruct) DecodeXDRFrom(xr *xdr.Reader) error {
+func (o *TestStruct) decodeXDR(xr *xdr.Reader) error {
 	o.I = int(xr.ReadUint64())
 	o.I8 = int8(xr.ReadUint8())
 	o.UI8 = xr.ReadUint8()
@@ -167,19 +129,8 @@ func (o *TestStruct) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.UI32 = xr.ReadUint32()
 	o.I64 = int64(xr.ReadUint64())
 	o.UI64 = xr.ReadUint64()
-	o.BS = xr.ReadBytesMax(1024)
-	o.S = xr.ReadStringMax(1024)
-	(&o.C).DecodeXDRFrom(xr)
-	_SSSize := int(xr.ReadUint32())
-	if _SSSize < 0 {
-		return xdr.ElementSizeExceeded("SS", _SSSize, 1024)
-	}
-	if _SSSize > 1024 {
-		return xdr.ElementSizeExceeded("SS", _SSSize, 1024)
-	}
-	o.SS = make([]string, _SSSize)
-	for i := range o.SS {
-		o.SS[i] = xr.ReadString()
-	}
+	o.BS = xr.ReadBytes()
+	o.S = xr.ReadString()
+	(&o.C).decodeXDR(xr)
 	return xr.Error()
 }

Godeps/_workspace/src/github.com/calmh/xdr/reader.go

@@ -5,12 +5,14 @@
 package xdr
 
 import (
-	"fmt"
+	"errors"
 	"io"
 	"reflect"
 	"unsafe"
 )
 
+var ErrElementSizeExceeded = errors.New("element size exceeded")
+
 type Reader struct {
 	r   io.Reader
 	err error
@@ -68,9 +70,8 @@ func (r *Reader) ReadBytesMaxInto(max int, dst []byte) []byte {
 	if r.err != nil {
 		return nil
 	}
-	if l < 0 || max > 0 && l > max {
-		// l may be negative on 32 bit builds
-		r.err = ElementSizeExceeded("bytes field", l, max)
+	if max > 0 && l > max {
+		r.err = ErrElementSizeExceeded
 		return nil
 	}
 
@@ -155,17 +156,9 @@ func (e XDRError) Error() string {
 	return "xdr " + e.op + ": " + e.err.Error()
 }
 
-func (e XDRError) IsEOF() bool {
-	return e.err == io.EOF
-}
-
 func (r *Reader) Error() error {
 	if r.err == nil {
 		return nil
 	}
 	return XDRError{"read", r.err}
 }
-
-func ElementSizeExceeded(field string, size, limit int) error {
-	return fmt.Errorf("%s exceeds size limit; %d > %d", field, size, limit)
-}

Godeps/_workspace/src/github.com/calmh/xdr/xdr_test.go

@@ -5,7 +5,6 @@ package xdr
 
 import (
 	"bytes"
-	"strings"
 	"testing"
 	"testing/quick"
 )
@@ -61,7 +60,7 @@ func TestReadBytesMaxInto(t *testing.T) {
 				if read := len(bs); read != tot {
 					t.Errorf("Incorrect read bytes, wrote=%d, buf=%d, max=%d, read=%d", tot, tot+diff, max, read)
 				}
-			} else if !strings.Contains(r.err.Error(), "exceeds size") {
+			} else if r.err != ErrElementSizeExceeded {
 				t.Errorf("Unexpected non-ErrElementSizeExceeded error for wrote=%d, max=%d: %v", tot, max, r.err)
 			}
 		}
@@ -85,7 +84,7 @@ func TestReadStringMax(t *testing.T) {
 				if read != tot {
 					t.Errorf("Incorrect read bytes, wrote=%d, max=%d, read=%d", tot, max, read)
 				}
-			} else if !strings.Contains(r.err.Error(), "exceeds size") {
+			} else if r.err != ErrElementSizeExceeded {
 				t.Errorf("Unexpected non-ErrElementSizeExceeded error for wrote=%d, max=%d, read=%d: %v", tot, max, read, r.err)
 			}
 		}

Godeps/_workspace/src/github.com/golang/snappy/AUTHORS

@@ -1,14 +0,0 @@
-# This is the official list of Snappy-Go authors for copyright purposes.
-# This file is distinct from the CONTRIBUTORS files.
-# See the latter for an explanation.
-
-# Names should be added to this file as
-#	Name or Organization <email address>
-# The email address is not required for organizations.
-
-# Please keep the list sorted.
-
-Damian Gryski <dgryski@gmail.com>
-Google Inc.
-Jan Mercl <0xjnml@gmail.com>
-Sebastien Binet <seb.binet@gmail.com>

Godeps/_workspace/src/github.com/golang/snappy/CONTRIBUTORS

@@ -1,36 +0,0 @@
-# This is the official list of people who can contribute
-# (and typically have contributed) code to the Snappy-Go repository.
-# The AUTHORS file lists the copyright holders; this file
-# lists people.  For example, Google employees are listed here
-# but not in AUTHORS, because Google holds the copyright.
-#
-# The submission process automatically checks to make sure
-# that people submitting code are listed in this file (by email address).
-#
-# Names should be added to this file only after verifying that
-# the individual or the individual's organization has agreed to
-# the appropriate Contributor License Agreement, found here:
-#
-#     http://code.google.com/legal/individual-cla-v1.0.html
-#     http://code.google.com/legal/corporate-cla-v1.0.html
-#
-# The agreement for individuals can be filled out on the web.
-#
-# When adding J Random Contributor's name to this file,
-# either J's name or J's organization's name should be
-# added to the AUTHORS file, depending on whether the
-# individual or corporate CLA was used.
-
-# Names should be added to this file like so:
-#     Name <email address>
-
-# Please keep the list sorted.
-
-Damian Gryski <dgryski@gmail.com>
-Jan Mercl <0xjnml@gmail.com>
-Kai Backman <kaib@golang.org>
-Marc-Antoine Ruel <maruel@chromium.org>
-Nigel Tao <nigeltao@golang.org>
-Rob Pike <r@golang.org>
-Russ Cox <rsc@golang.org>
-Sebastien Binet <seb.binet@gmail.com>

Godeps/_workspace/src/github.com/golang/snappy/LICENSE

@@ -1,27 +0,0 @@
-Copyright (c) 2011 The Snappy-Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Godeps/_workspace/src/github.com/golang/snappy/README

@@ -1,7 +0,0 @@
-The Snappy compression format in the Go programming language.
-
-To download and install from source:
-$ go get github.com/golang/snappy
-
-Unless otherwise noted, the Snappy-Go source files are distributed
-under the BSD-style license found in the LICENSE file.

Godeps/_workspace/src/github.com/golang/snappy/decode.go

@@ -1,294 +0,0 @@
-// Copyright 2011 The Snappy-Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package snappy
-
-import (
-	"encoding/binary"
-	"errors"
-	"io"
-)
-
-var (
-	// ErrCorrupt reports that the input is invalid.
-	ErrCorrupt = errors.New("snappy: corrupt input")
-	// ErrTooLarge reports that the uncompressed length is too large.
-	ErrTooLarge = errors.New("snappy: decoded block is too large")
-	// ErrUnsupported reports that the input isn't supported.
-	ErrUnsupported = errors.New("snappy: unsupported input")
-)
-
-// DecodedLen returns the length of the decoded block.
-func DecodedLen(src []byte) (int, error) {
-	v, _, err := decodedLen(src)
-	return v, err
-}
-
-// decodedLen returns the length of the decoded block and the number of bytes
-// that the length header occupied.
-func decodedLen(src []byte) (blockLen, headerLen int, err error) {
-	v, n := binary.Uvarint(src)
-	if n <= 0 || v > 0xffffffff {
-		return 0, 0, ErrCorrupt
-	}
-
-	const wordSize = 32 << (^uint(0) >> 32 & 1)
-	if wordSize == 32 && v > 0x7fffffff {
-		return 0, 0, ErrTooLarge
-	}
-	return int(v), n, nil
-}
-
-// Decode returns the decoded form of src. The returned slice may be a sub-
-// slice of dst if dst was large enough to hold the entire decoded block.
-// Otherwise, a newly allocated slice will be returned.
-// It is valid to pass a nil dst.
-func Decode(dst, src []byte) ([]byte, error) {
-	dLen, s, err := decodedLen(src)
-	if err != nil {
-		return nil, err
-	}
-	if len(dst) < dLen {
-		dst = make([]byte, dLen)
-	}
-
-	var d, offset, length int
-	for s < len(src) {
-		switch src[s] & 0x03 {
-		case tagLiteral:
-			x := uint(src[s] >> 2)
-			switch {
-			case x < 60:
-				s++
-			case x == 60:
-				s += 2
-				if s > len(src) {
-					return nil, ErrCorrupt
-				}
-				x = uint(src[s-1])
-			case x == 61:
-				s += 3
-				if s > len(src) {
-					return nil, ErrCorrupt
-				}
-				x = uint(src[s-2]) | uint(src[s-1])<<8
-			case x == 62:
-				s += 4
-				if s > len(src) {
-					return nil, ErrCorrupt
-				}
-				x = uint(src[s-3]) | uint(src[s-2])<<8 | uint(src[s-1])<<16
-			case x == 63:
-				s += 5
-				if s > len(src) {
-					return nil, ErrCorrupt
-				}
-				x = uint(src[s-4]) | uint(src[s-3])<<8 | uint(src[s-2])<<16 | uint(src[s-1])<<24
-			}
-			length = int(x + 1)
-			if length <= 0 {
-				return nil, errors.New("snappy: unsupported literal length")
-			}
-			if length > len(dst)-d || length > len(src)-s {
-				return nil, ErrCorrupt
-			}
-			copy(dst[d:], src[s:s+length])
-			d += length
-			s += length
-			continue
-
-		case tagCopy1:
-			s += 2
-			if s > len(src) {
-				return nil, ErrCorrupt
-			}
-			length = 4 + int(src[s-2])>>2&0x7
-			offset = int(src[s-2])&0xe0<<3 | int(src[s-1])
-
-		case tagCopy2:
-			s += 3
-			if s > len(src) {
-				return nil, ErrCorrupt
-			}
-			length = 1 + int(src[s-3])>>2
-			offset = int(src[s-2]) | int(src[s-1])<<8
-
-		case tagCopy4:
-			return nil, errors.New("snappy: unsupported COPY_4 tag")
-		}
-
-		end := d + length
-		if offset > d || end > len(dst) {
-			return nil, ErrCorrupt
-		}
-		for ; d < end; d++ {
-			dst[d] = dst[d-offset]
-		}
-	}
-	if d != dLen {
-		return nil, ErrCorrupt
-	}
-	return dst[:d], nil
-}
-
-// NewReader returns a new Reader that decompresses from r, using the framing
-// format described at
-// https://github.com/google/snappy/blob/master/framing_format.txt
-func NewReader(r io.Reader) *Reader {
-	return &Reader{
-		r:       r,
-		decoded: make([]byte, maxUncompressedChunkLen),
-		buf:     make([]byte, MaxEncodedLen(maxUncompressedChunkLen)+checksumSize),
-	}
-}
-
-// Reader is an io.Reader than can read Snappy-compressed bytes.
-type Reader struct {
-	r       io.Reader
-	err     error
-	decoded []byte
-	buf     []byte
-	// decoded[i:j] contains decoded bytes that have not yet been passed on.
-	i, j       int
-	readHeader bool
-}
-
-// Reset discards any buffered data, resets all state, and switches the Snappy
-// reader to read from r. This permits reusing a Reader rather than allocating
-// a new one.
-func (r *Reader) Reset(reader io.Reader) {
-	r.r = reader
-	r.err = nil
-	r.i = 0
-	r.j = 0
-	r.readHeader = false
-}
-
-func (r *Reader) readFull(p []byte) (ok bool) {
-	if _, r.err = io.ReadFull(r.r, p); r.err != nil {
-		if r.err == io.ErrUnexpectedEOF {
-			r.err = ErrCorrupt
-		}
-		return false
-	}
-	return true
-}
-
-// Read satisfies the io.Reader interface.
-func (r *Reader) Read(p []byte) (int, error) {
-	if r.err != nil {
-		return 0, r.err
-	}
-	for {
-		if r.i < r.j {
-			n := copy(p, r.decoded[r.i:r.j])
-			r.i += n
-			return n, nil
-		}
-		if !r.readFull(r.buf[:4]) {
-			return 0, r.err
-		}
-		chunkType := r.buf[0]
-		if !r.readHeader {
-			if chunkType != chunkTypeStreamIdentifier {
-				r.err = ErrCorrupt
-				return 0, r.err
-			}
-			r.readHeader = true
-		}
-		chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16
-		if chunkLen > len(r.buf) {
-			r.err = ErrUnsupported
-			return 0, r.err
-		}
-
-		// The chunk types are specified at
-		// https://github.com/google/snappy/blob/master/framing_format.txt
-		switch chunkType {
-		case chunkTypeCompressedData:
-			// Section 4.2. Compressed data (chunk type 0x00).
-			if chunkLen < checksumSize {
-				r.err = ErrCorrupt
-				return 0, r.err
-			}
-			buf := r.buf[:chunkLen]
-			if !r.readFull(buf) {
-				return 0, r.err
-			}
-			checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
-			buf = buf[checksumSize:]
-
-			n, err := DecodedLen(buf)
-			if err != nil {
-				r.err = err
-				return 0, r.err
-			}
-			if n > len(r.decoded) {
-				r.err = ErrCorrupt
-				return 0, r.err
-			}
-			if _, err := Decode(r.decoded, buf); err != nil {
-				r.err = err
-				return 0, r.err
-			}
-			if crc(r.decoded[:n]) != checksum {
-				r.err = ErrCorrupt
-				return 0, r.err
-			}
-			r.i, r.j = 0, n
-			continue
-
-		case chunkTypeUncompressedData:
-			// Section 4.3. Uncompressed data (chunk type 0x01).
-			if chunkLen < checksumSize {
-				r.err = ErrCorrupt
-				return 0, r.err
-			}
-			buf := r.buf[:checksumSize]
-			if !r.readFull(buf) {
-				return 0, r.err
-			}
-			checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
-			// Read directly into r.decoded instead of via r.buf.
-			n := chunkLen - checksumSize
-			if !r.readFull(r.decoded[:n]) {
-				return 0, r.err
-			}
-			if crc(r.decoded[:n]) != checksum {
-				r.err = ErrCorrupt
-				return 0, r.err
-			}
-			r.i, r.j = 0, n
-			continue
-
-		case chunkTypeStreamIdentifier:
-			// Section 4.1. Stream identifier (chunk type 0xff).
-			if chunkLen != len(magicBody) {
-				r.err = ErrCorrupt
-				return 0, r.err
-			}
-			if !r.readFull(r.buf[:len(magicBody)]) {
-				return 0, r.err
-			}
-			for i := 0; i < len(magicBody); i++ {
-				if r.buf[i] != magicBody[i] {
-					r.err = ErrCorrupt
-					return 0, r.err
-				}
-			}
-			continue
-		}
-
-		if chunkType <= 0x7f {
-			// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
-			r.err = ErrUnsupported
-			return 0, r.err
-		}
-		// Section 4.4 Padding (chunk type 0xfe).
-		// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
-		if !r.readFull(r.buf[:chunkLen]) {
-			return 0, r.err
-		}
-	}
-}

Godeps/_workspace/src/github.com/juju/ratelimit/LICENSE

@@ -1,9 +1,3 @@
-All files in this repository are licensed as follows. If you contribute
-to this repository, it is assumed that you license your contribution
-under the same license unless you state otherwise.
-
-All files Copyright (C) 2015 Canonical Ltd. unless otherwise specified in the file.
-
 This software is licensed under the LGPLv3, included below.
 
 As a special exception to the GNU Lesser General Public License version 3

Godeps/_workspace/src/github.com/juju/ratelimit/README.md

@@ -20,7 +20,7 @@ token in the bucket represents one byte.
 ```go
 func Writer(w io.Writer, bucket *Bucket) io.Writer
 ```
-Writer returns a writer that is rate limited by the given token bucket. Each
+Writer returns a reader that is rate limited by the given token bucket. Each
 token in the bucket represents one byte.
 
 #### type Bucket

Godeps/_workspace/src/github.com/juju/ratelimit/ratelimit.go

@@ -2,8 +2,7 @@
 // Licensed under the LGPLv3 with static-linking exception.
 // See LICENCE file for details.
 
-// The ratelimit package provides an efficient token bucket implementation
-// that can be used to limit the rate of arbitrary things.
+// The ratelimit package provides an efficient token bucket implementation.
 // See http://en.wikipedia.org/wiki/Token_bucket.
 package ratelimit
 
@@ -11,7 +10,6 @@ import (
 	"strconv"
 	"sync"
 	"time"
-	"math"
 )
 
 // Bucket represents a token bucket that fills at a predetermined rate.
@@ -57,7 +55,7 @@ func NewBucketWithRate(rate float64, capacity int64) *Bucket {
 			continue
 		}
 		tb := NewBucketWithQuantum(fillInterval, capacity, quantum)
-		if diff := math.Abs(tb.Rate() - rate); diff/rate <= rateMargin {
+		if diff := abs(tb.Rate() - rate); diff/rate <= rateMargin {
 			return tb
 		}
 	}
@@ -219,3 +217,10 @@ func (tb *Bucket) adjust(now time.Time) (currentTick int64) {
 	tb.availTick = currentTick
 	return
 }
+
+func abs(f float64) float64 {
+	if f < 0 {
+		return -f
+	}
+	return f
+}

Godeps/_workspace/src/github.com/kardianos/osext/LICENSE

@@ -1,27 +0,0 @@
-Copyright (c) 2012 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Godeps/_workspace/src/github.com/kardianos/osext/README.md

@@ -1,16 +0,0 @@
-### Extensions to the "os" package.
-
-## Find the current Executable and ExecutableFolder.
-
-There is sometimes utility in finding the current executable file
-that is running. This can be used for upgrading the current executable
-or finding resources located relative to the executable file. Both
-working directory and the os.Args[0] value are arbitrary and cannot
-be relied on; os.Args[0] can be "faked".
-
-Multi-platform and supports:
- * Linux
- * OS X
- * Windows
- * Plan 9
- * BSDs.

Godeps/_workspace/src/github.com/kardianos/osext/osext_test.go

@@ -1,203 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build darwin linux freebsd netbsd windows
-
-package osext
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"runtime"
-	"testing"
-)
-
-const (
-	executableEnvVar = "OSTEST_OUTPUT_EXECUTABLE"
-
-	executableEnvValueMatch  = "match"
-	executableEnvValueDelete = "delete"
-)
-
-func TestPrintExecutable(t *testing.T) {
-	ef, err := Executable()
-	if err != nil {
-		t.Fatalf("Executable failed: %v", err)
-	}
-	t.Log("Executable:", ef)
-}
-func TestPrintExecutableFolder(t *testing.T) {
-	ef, err := ExecutableFolder()
-	if err != nil {
-		t.Fatalf("ExecutableFolder failed: %v", err)
-	}
-	t.Log("Executable Folder:", ef)
-}
-func TestExecutableFolder(t *testing.T) {
-	ef, err := ExecutableFolder()
-	if err != nil {
-		t.Fatalf("ExecutableFolder failed: %v", err)
-	}
-	if ef[len(ef)-1] == filepath.Separator {
-		t.Fatal("ExecutableFolder ends with a trailing slash.")
-	}
-}
-func TestExecutableMatch(t *testing.T) {
-	ep, err := Executable()
-	if err != nil {
-		t.Fatalf("Executable failed: %v", err)
-	}
-
-	// fullpath to be of the form "dir/prog".
-	dir := filepath.Dir(filepath.Dir(ep))
-	fullpath, err := filepath.Rel(dir, ep)
-	if err != nil {
-		t.Fatalf("filepath.Rel: %v", err)
-	}
-	// Make child start with a relative program path.
-	// Alter argv[0] for child to verify getting real path without argv[0].
-	cmd := &exec.Cmd{
-		Dir:  dir,
-		Path: fullpath,
-		Env:  []string{fmt.Sprintf("%s=%s", executableEnvVar, executableEnvValueMatch)},
-	}
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Fatalf("exec(self) failed: %v", err)
-	}
-	outs := string(out)
-	if !filepath.IsAbs(outs) {
-		t.Fatalf("Child returned %q, want an absolute path", out)
-	}
-	if !sameFile(outs, ep) {
-		t.Fatalf("Child returned %q, not the same file as %q", out, ep)
-	}
-}
-
-func TestExecutableDelete(t *testing.T) {
-	if runtime.GOOS != "linux" {
-		t.Skip()
-	}
-	fpath, err := Executable()
-	if err != nil {
-		t.Fatalf("Executable failed: %v", err)
-	}
-
-	r, w := io.Pipe()
-	stderrBuff := &bytes.Buffer{}
-	stdoutBuff := &bytes.Buffer{}
-	cmd := &exec.Cmd{
-		Path:   fpath,
-		Env:    []string{fmt.Sprintf("%s=%s", executableEnvVar, executableEnvValueDelete)},
-		Stdin:  r,
-		Stderr: stderrBuff,
-		Stdout: stdoutBuff,
-	}
-	err = cmd.Start()
-	if err != nil {
-		t.Fatalf("exec(self) start failed: %v", err)
-	}
-
-	tempPath := fpath + "_copy"
-	_ = os.Remove(tempPath)
-
-	err = copyFile(tempPath, fpath)
-	if err != nil {
-		t.Fatalf("copy file failed: %v", err)
-	}
-	err = os.Remove(fpath)
-	if err != nil {
-		t.Fatalf("remove running test file failed: %v", err)
-	}
-	err = os.Rename(tempPath, fpath)
-	if err != nil {
-		t.Fatalf("rename copy to previous name failed: %v", err)
-	}
-
-	w.Write([]byte{0})
-	w.Close()
-
-	err = cmd.Wait()
-	if err != nil {
-		t.Fatalf("exec wait failed: %v", err)
-	}
-
-	childPath := stderrBuff.String()
-	if !filepath.IsAbs(childPath) {
-		t.Fatalf("Child returned %q, want an absolute path", childPath)
-	}
-	if !sameFile(childPath, fpath) {
-		t.Fatalf("Child returned %q, not the same file as %q", childPath, fpath)
-	}
-}
-
-func sameFile(fn1, fn2 string) bool {
-	fi1, err := os.Stat(fn1)
-	if err != nil {
-		return false
-	}
-	fi2, err := os.Stat(fn2)
-	if err != nil {
-		return false
-	}
-	return os.SameFile(fi1, fi2)
-}
-func copyFile(dest, src string) error {
-	df, err := os.Create(dest)
-	if err != nil {
-		return err
-	}
-	defer df.Close()
-
-	sf, err := os.Open(src)
-	if err != nil {
-		return err
-	}
-	defer sf.Close()
-
-	_, err = io.Copy(df, sf)
-	return err
-}
-
-func TestMain(m *testing.M) {
-	env := os.Getenv(executableEnvVar)
-	switch env {
-	case "":
-		os.Exit(m.Run())
-	case executableEnvValueMatch:
-		// First chdir to another path.
-		dir := "/"
-		if runtime.GOOS == "windows" {
-			dir = filepath.VolumeName(".")
-		}
-		os.Chdir(dir)
-		if ep, err := Executable(); err != nil {
-			fmt.Fprint(os.Stderr, "ERROR: ", err)
-		} else {
-			fmt.Fprint(os.Stderr, ep)
-		}
-	case executableEnvValueDelete:
-		bb := make([]byte, 1)
-		var err error
-		n, err := os.Stdin.Read(bb)
-		if err != nil {
-			fmt.Fprint(os.Stderr, "ERROR: ", err)
-			os.Exit(2)
-		}
-		if n != 1 {
-			fmt.Fprint(os.Stderr, "ERROR: n != 1, n == ", n)
-			os.Exit(2)
-		}
-		if ep, err := Executable(); err != nil {
-			fmt.Fprint(os.Stderr, "ERROR: ", err)
-		} else {
-			fmt.Fprint(os.Stderr, ep)
-		}
-	}
-	os.Exit(0)
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch.go

@@ -8,84 +8,65 @@ package leveldb
 
 import (
 	"encoding/binary"
-	"fmt"
+	"errors"
 
-	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/memdb"
 )
 
-type ErrBatchCorrupted struct {
-	Reason string
-}
-
-func (e *ErrBatchCorrupted) Error() string {
-	return fmt.Sprintf("leveldb: batch corrupted: %s", e.Reason)
-}
-
-func newErrBatchCorrupted(reason string) error {
-	return errors.NewErrCorrupted(nil, &ErrBatchCorrupted{reason})
-}
-
-const (
-	batchHdrLen  = 8 + 4
-	batchGrowRec = 3000
+var (
+	errBatchTooShort  = errors.New("leveldb: batch is too short")
+	errBatchBadRecord = errors.New("leveldb: bad record in batch")
 )
 
-type BatchReplay interface {
-	Put(key, value []byte)
-	Delete(key []byte)
+const kBatchHdrLen = 8 + 4
+
+type batchReplay interface {
+	put(key, value []byte, seq uint64)
+	delete(key []byte, seq uint64)
 }
 
 // Batch is a write batch.
 type Batch struct {
-	data       []byte
+	buf        []byte
 	rLen, bLen int
 	seq        uint64
 	sync       bool
 }
 
 func (b *Batch) grow(n int) {
-	off := len(b.data)
+	off := len(b.buf)
 	if off == 0 {
-		off = batchHdrLen
-		if b.data != nil {
-			b.data = b.data[:off]
-		}
+		// include headers
+		off = kBatchHdrLen
+		n += off
 	}
-	if cap(b.data)-off < n {
-		if b.data == nil {
-			b.data = make([]byte, off, off+n)
-		} else {
-			odata := b.data
-			div := 1
-			if b.rLen > batchGrowRec {
-				div = b.rLen / batchGrowRec
-			}
-			b.data = make([]byte, off, off+n+(off-batchHdrLen)/div)
-			copy(b.data, odata)
-		}
+	if cap(b.buf)-off >= n {
+		return
 	}
+	buf := make([]byte, 2*cap(b.buf)+n)
+	copy(buf, b.buf)
+	b.buf = buf[:off]
 }
 
-func (b *Batch) appendRec(kt kType, key, value []byte) {
+func (b *Batch) appendRec(t vType, key, value []byte) {
 	n := 1 + binary.MaxVarintLen32 + len(key)
-	if kt == ktVal {
+	if t == tVal {
 		n += binary.MaxVarintLen32 + len(value)
 	}
 	b.grow(n)
-	off := len(b.data)
-	data := b.data[:off+n]
-	data[off] = byte(kt)
+	off := len(b.buf)
+	buf := b.buf[:off+n]
+	buf[off] = byte(t)
 	off += 1
-	off += binary.PutUvarint(data[off:], uint64(len(key)))
-	copy(data[off:], key)
+	off += binary.PutUvarint(buf[off:], uint64(len(key)))
+	copy(buf[off:], key)
 	off += len(key)
-	if kt == ktVal {
-		off += binary.PutUvarint(data[off:], uint64(len(value)))
-		copy(data[off:], value)
+	if t == tVal {
+		off += binary.PutUvarint(buf[off:], uint64(len(value)))
+		copy(buf[off:], value)
 		off += len(value)
 	}
-	b.data = data[:off]
+	b.buf = buf[:off]
 	b.rLen++
 	//  Include 8-byte ikey header
 	b.bLen += len(key) + len(value) + 8
@@ -94,51 +75,18 @@ func (b *Batch) appendRec(kt kType, key, value []byte) {
 // Put appends 'put operation' of the given key/value pair to the batch.
 // It is safe to modify the contents of the argument after Put returns.
 func (b *Batch) Put(key, value []byte) {
-	b.appendRec(ktVal, key, value)
+	b.appendRec(tVal, key, value)
 }
 
 // Delete appends 'delete operation' of the given key to the batch.
 // It is safe to modify the contents of the argument after Delete returns.
 func (b *Batch) Delete(key []byte) {
-	b.appendRec(ktDel, key, nil)
-}
-
-// Dump dumps batch contents. The returned slice can be loaded into the
-// batch using Load method.
-// The returned slice is not its own copy, so the contents should not be
-// modified.
-func (b *Batch) Dump() []byte {
-	return b.encode()
-}
-
-// Load loads given slice into the batch. Previous contents of the batch
-// will be discarded.
-// The given slice will not be copied and will be used as batch buffer, so
-// it is not safe to modify the contents of the slice.
-func (b *Batch) Load(data []byte) error {
-	return b.decode(0, data)
-}
-
-// Replay replays batch contents.
-func (b *Batch) Replay(r BatchReplay) error {
-	return b.decodeRec(func(i int, kt kType, key, value []byte) {
-		switch kt {
-		case ktVal:
-			r.Put(key, value)
-		case ktDel:
-			r.Delete(key)
-		}
-	})
-}
-
-// Len returns number of records in the batch.
-func (b *Batch) Len() int {
-	return b.rLen
+	b.appendRec(tDel, key, nil)
 }
 
 // Reset resets the batch.
 func (b *Batch) Reset() {
-	b.data = b.data[:0]
+	b.buf = nil
 	b.seq = 0
 	b.rLen = 0
 	b.bLen = 0
@@ -149,10 +97,24 @@ func (b *Batch) init(sync bool) {
 	b.sync = sync
 }
 
+func (b *Batch) put(key, value []byte, seq uint64) {
+	if b.rLen == 0 {
+		b.seq = seq
+	}
+	b.Put(key, value)
+}
+
+func (b *Batch) delete(key []byte, seq uint64) {
+	if b.rLen == 0 {
+		b.seq = seq
+	}
+	b.Delete(key)
+}
+
 func (b *Batch) append(p *Batch) {
 	if p.rLen > 0 {
-		b.grow(len(p.data) - batchHdrLen)
-		b.data = append(b.data, p.data[batchHdrLen:]...)
+		b.grow(len(p.buf) - kBatchHdrLen)
+		b.buf = append(b.buf, p.buf[kBatchHdrLen:]...)
 		b.rLen += p.rLen
 	}
 	if p.sync {
@@ -160,93 +122,95 @@ func (b *Batch) append(p *Batch) {
 	}
 }
 
-// size returns sums of key/value pair length plus 8-bytes ikey.
+func (b *Batch) len() int {
+	return b.rLen
+}
+
 func (b *Batch) size() int {
 	return b.bLen
 }
 
 func (b *Batch) encode() []byte {
 	b.grow(0)
-	binary.LittleEndian.PutUint64(b.data, b.seq)
-	binary.LittleEndian.PutUint32(b.data[8:], uint32(b.rLen))
+	binary.LittleEndian.PutUint64(b.buf, b.seq)
+	binary.LittleEndian.PutUint32(b.buf[8:], uint32(b.rLen))
 
-	return b.data
+	return b.buf
 }
 
-func (b *Batch) decode(prevSeq uint64, data []byte) error {
-	if len(data) < batchHdrLen {
-		return newErrBatchCorrupted("too short")
+func (b *Batch) decode(buf []byte) error {
+	if len(buf) < kBatchHdrLen {
+		return errBatchTooShort
 	}
 
-	b.seq = binary.LittleEndian.Uint64(data)
-	if b.seq < prevSeq {
-		return newErrBatchCorrupted("invalid sequence number")
-	}
-	b.rLen = int(binary.LittleEndian.Uint32(data[8:]))
-	if b.rLen < 0 {
-		return newErrBatchCorrupted("invalid records length")
-	}
+	b.seq = binary.LittleEndian.Uint64(buf)
+	b.rLen = int(binary.LittleEndian.Uint32(buf[8:]))
 	// No need to be precise at this point, it won't be used anyway
-	b.bLen = len(data) - batchHdrLen
-	b.data = data
+	b.bLen = len(buf) - kBatchHdrLen
+	b.buf = buf
 
 	return nil
 }
 
-func (b *Batch) decodeRec(f func(i int, kt kType, key, value []byte)) (err error) {
-	off := batchHdrLen
+func (b *Batch) decodeRec(f func(i int, t vType, key, value []byte)) error {
+	off := kBatchHdrLen
 	for i := 0; i < b.rLen; i++ {
-		if off >= len(b.data) {
-			return newErrBatchCorrupted("invalid records length")
+		if off >= len(b.buf) {
+			return errors.New("leveldb: invalid batch record length")
 		}
 
-		kt := kType(b.data[off])
-		if kt > ktVal {
-			return newErrBatchCorrupted("bad record: invalid type")
+		t := vType(b.buf[off])
+		if t > tVal {
+			return errors.New("leveldb: invalid batch record type in batch")
 		}
 		off += 1
 
-		x, n := binary.Uvarint(b.data[off:])
+		x, n := binary.Uvarint(b.buf[off:])
 		off += n
-		if n <= 0 || off+int(x) > len(b.data) {
-			return newErrBatchCorrupted("bad record: invalid key length")
+		if n <= 0 || off+int(x) > len(b.buf) {
+			return errBatchBadRecord
 		}
-		key := b.data[off : off+int(x)]
+		key := b.buf[off : off+int(x)]
 		off += int(x)
+
 		var value []byte
-		if kt == ktVal {
-			x, n := binary.Uvarint(b.data[off:])
+		if t == tVal {
+			x, n := binary.Uvarint(b.buf[off:])
 			off += n
-			if n <= 0 || off+int(x) > len(b.data) {
-				return newErrBatchCorrupted("bad record: invalid value length")
+			if n <= 0 || off+int(x) > len(b.buf) {
+				return errBatchBadRecord
 			}
-			value = b.data[off : off+int(x)]
+			value = b.buf[off : off+int(x)]
 			off += int(x)
 		}
 
-		f(i, kt, key, value)
+		f(i, t, key, value)
 	}
 
 	return nil
 }
 
+func (b *Batch) replay(to batchReplay) error {
+	return b.decodeRec(func(i int, t vType, key, value []byte) {
+		switch t {
+		case tVal:
+			to.put(key, value, b.seq+uint64(i))
+		case tDel:
+			to.delete(key, b.seq+uint64(i))
+		}
+	})
+}
+
 func (b *Batch) memReplay(to *memdb.DB) error {
-	return b.decodeRec(func(i int, kt kType, key, value []byte) {
-		ikey := newIkey(key, b.seq+uint64(i), kt)
+	return b.decodeRec(func(i int, t vType, key, value []byte) {
+		ikey := newIKey(key, b.seq+uint64(i), t)
 		to.Put(ikey, value)
 	})
 }
 
-func (b *Batch) memDecodeAndReplay(prevSeq uint64, data []byte, to *memdb.DB) error {
-	if err := b.decode(prevSeq, data); err != nil {
-		return err
-	}
-	return b.memReplay(to)
-}
-
 func (b *Batch) revertMemReplay(to *memdb.DB) error {
-	return b.decodeRec(func(i int, kt kType, key, value []byte) {
-		ikey := newIkey(key, b.seq+uint64(i), kt)
+	return b.decodeRec(func(i int, t vType, key, value []byte) {
+		ikey := newIKey(key, b.seq+uint64(i), t)
 		to.Delete(ikey)
 	})
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch_test.go

@@ -15,7 +15,7 @@ import (
 )
 
 type tbRec struct {
-	kt         kType
+	t          vType
 	key, value []byte
 }
 
@@ -23,39 +23,39 @@ type testBatch struct {
 	rec []*tbRec
 }
 
-func (p *testBatch) Put(key, value []byte) {
-	p.rec = append(p.rec, &tbRec{ktVal, key, value})
+func (p *testBatch) put(key, value []byte, seq uint64) {
+	p.rec = append(p.rec, &tbRec{tVal, key, value})
 }
 
-func (p *testBatch) Delete(key []byte) {
-	p.rec = append(p.rec, &tbRec{ktDel, key, nil})
+func (p *testBatch) delete(key []byte, seq uint64) {
+	p.rec = append(p.rec, &tbRec{tDel, key, nil})
 }
 
 func compareBatch(t *testing.T, b1, b2 *Batch) {
 	if b1.seq != b2.seq {
 		t.Errorf("invalid seq number want %d, got %d", b1.seq, b2.seq)
 	}
-	if b1.Len() != b2.Len() {
-		t.Fatalf("invalid record length want %d, got %d", b1.Len(), b2.Len())
+	if b1.len() != b2.len() {
+		t.Fatalf("invalid record length want %d, got %d", b1.len(), b2.len())
 	}
 	p1, p2 := new(testBatch), new(testBatch)
-	err := b1.Replay(p1)
+	err := b1.replay(p1)
 	if err != nil {
 		t.Fatal("error when replaying batch 1: ", err)
 	}
-	err = b2.Replay(p2)
+	err = b2.replay(p2)
 	if err != nil {
 		t.Fatal("error when replaying batch 2: ", err)
 	}
 	for i := range p1.rec {
 		r1, r2 := p1.rec[i], p2.rec[i]
-		if r1.kt != r2.kt {
-			t.Errorf("invalid type on record '%d' want %d, got %d", i, r1.kt, r2.kt)
+		if r1.t != r2.t {
+			t.Errorf("invalid type on record '%d' want %d, got %d", i, r1.t, r2.t)
 		}
 		if !bytes.Equal(r1.key, r2.key) {
 			t.Errorf("invalid key on record '%d' want %s, got %s", i, string(r1.key), string(r2.key))
 		}
-		if r1.kt == ktVal {
+		if r1.t == tVal {
 			if !bytes.Equal(r1.value, r2.value) {
 				t.Errorf("invalid value on record '%d' want %s, got %s", i, string(r1.value), string(r2.value))
 			}
@@ -75,7 +75,7 @@ func TestBatch_EncodeDecode(t *testing.T) {
 	b1.Delete([]byte("k"))
 	buf := b1.encode()
 	b2 := new(Batch)
-	err := b2.decode(0, buf)
+	err := b2.decode(buf)
 	if err != nil {
 		t.Error("error when decoding batch: ", err)
 	}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/bench2_test.go

@@ -1,30 +0,0 @@
-// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-// +build !go1.2
-
-package cache
-
-import (
-	"math/rand"
-	"testing"
-)
-
-func BenchmarkLRUCache(b *testing.B) {
-	c := NewCache(NewLRU(10000))
-
-	b.SetParallelism(10)
-	b.RunParallel(func(pb *testing.PB) {
-		r := rand.New(rand.NewSource(time.Now().UnixNano()))
-
-		for pb.Next() {
-			key := uint64(r.Intn(1000000))
-			c.Get(0, key, func() (int, Value) {
-				return 1, key
-			}).Release()
-		}
-	})
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache.go

@@ -8,669 +8,151 @@
 package cache
 
 import (
-	"sync"
 	"sync/atomic"
-	"unsafe"
-
-	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
-// Cacher provides interface to implements a caching functionality.
-// An implementation must be goroutine-safe.
-type Cacher interface {
-	// Capacity returns cache capacity.
-	Capacity() int
+// SetFunc is the function that will be called by Namespace.Get to create
+// a cache object, if charge is less than one than the cache object will
+// not be registered to cache tree, if value is nil then the cache object
+// will not be created.
+type SetFunc func() (charge int, value interface{})
 
-	// SetCapacity sets cache capacity.
+// DelFin is the function that will be called as the result of a delete operation.
+// Exist == true is indication that the object is exist, and pending == true is
+// indication of deletion already happen but haven't done yet (wait for all handles
+// to be released). And exist == false means the object doesn't exist.
+type DelFin func(exist, pending bool)
+
+// PurgeFin is the function that will be called as the result of a purge operation.
+type PurgeFin func(ns, key uint64)
+
+// Cache is a cache tree. A cache instance must be goroutine-safe.
+type Cache interface {
+	// SetCapacity sets cache tree capacity.
 	SetCapacity(capacity int)
 
-	// Promote promotes the 'cache node'.
-	Promote(n *Node)
+	// Capacity returns cache tree capacity.
+	Capacity() int
 
-	// Ban evicts the 'cache node' and prevent subsequent 'promote'.
-	Ban(n *Node)
+	// Used returns used cache tree capacity.
+	Used() int
 
-	// Evict evicts the 'cache node'.
-	Evict(n *Node)
+	// Size returns entire alive cache objects size.
+	Size() int
 
-	// EvictNS evicts 'cache node' with the given namespace.
-	EvictNS(ns uint64)
+	// NumObjects returns number of alive objects.
+	NumObjects() int
 
-	// EvictAll evicts all 'cache node'.
-	EvictAll()
+	// GetNamespace gets cache namespace with the given id.
+	// GetNamespace is never return nil.
+	GetNamespace(id uint64) Namespace
 
-	// Close closes the 'cache tree'
-	Close() error
+	// PurgeNamespace purges cache namespace with the given id from this cache tree.
+	// Also read Namespace.Purge.
+	PurgeNamespace(id uint64, fin PurgeFin)
+
+	// ZapNamespace detaches cache namespace with the given id from this cache tree.
+	// Also read Namespace.Zap.
+	ZapNamespace(id uint64)
+
+	// Purge purges all cache namespace from this cache tree.
+	// This is behave the same as calling Namespace.Purge method on all cache namespace.
+	Purge(fin PurgeFin)
+
+	// Zap detaches all cache namespace from this cache tree.
+	// This is behave the same as calling Namespace.Zap method on all cache namespace.
+	Zap()
 }
 
-// Value is a 'cacheable object'. It may implements util.Releaser, if
-// so the the Release method will be called once object is released.
-type Value interface{}
+// Namespace is a cache namespace. A namespace instance must be goroutine-safe.
+type Namespace interface {
+	// Get gets cache object with the given key.
+	// If cache object is not found and setf is not nil, Get will atomically creates
+	// the cache object by calling setf. Otherwise Get will returns nil.
+	//
+	// The returned cache handle should be released after use by calling Release
+	// method.
+	Get(key uint64, setf SetFunc) Handle
 
-type CacheGetter struct {
-	Cache *Cache
-	NS    uint64
+	// Delete removes cache object with the given key from cache tree.
+	// A deleted cache object will be released as soon as all of its handles have
+	// been released.
+	// Delete only happen once, subsequent delete will consider cache object doesn't
+	// exist, even if the cache object ins't released yet.
+	//
+	// If not nil, fin will be called if the cache object doesn't exist or when
+	// finally be released.
+	//
+	// Delete returns true if such cache object exist and never been deleted.
+	Delete(key uint64, fin DelFin) bool
+
+	// Purge removes all cache objects within this namespace from cache tree.
+	// This is the same as doing delete on all cache objects.
+	//
+	// If not nil, fin will be called on all cache objects when its finally be
+	// released.
+	Purge(fin PurgeFin)
+
+	// Zap detaches namespace from cache tree and release all its cache objects.
+	// A zapped namespace can never be filled again.
+	// Calling Get on zapped namespace will always return nil.
+	Zap()
 }
 
-func (g *CacheGetter) Get(key uint64, setFunc func() (size int, value Value)) *Handle {
-	return g.Cache.Get(g.NS, key, setFunc)
-}
+// Handle is a cache handle.
+type Handle interface {
+	// Release releases this cache handle. This method can be safely called mutiple
+	// times.
+	Release()
 
-// The hash tables implementation is based on:
-// "Dynamic-Sized Nonblocking Hash Tables", by Yujie Liu, Kunlong Zhang, and Michael Spear. ACM Symposium on Principles of Distributed Computing, Jul 2014.
+	// Value returns value of this cache handle.
+	// Value will returns nil after this cache handle have be released.
+	Value() interface{}
+}
 
 const (
-	mInitialSize           = 1 << 4
-	mOverflowThreshold     = 1 << 5
-	mOverflowGrowThreshold = 1 << 7
+	DelNotExist = iota
+	DelExist
+	DelPendig
 )
 
-type mBucket struct {
-	mu     sync.Mutex
-	node   []*Node
-	frozen bool
+// Namespace state.
+type nsState int
+
+const (
+	nsEffective nsState = iota
+	nsZapped
+)
+
+// Node state.
+type nodeState int
+
+const (
+	nodeEffective nodeState = iota
+	nodeEvicted
+	nodeDeleted
+)
+
+// Fake handle.
+type fakeHandle struct {
+	value interface{}
+	fin   func()
+	once  uint32
 }
 
-func (b *mBucket) freeze() []*Node {
-	b.mu.Lock()
-	defer b.mu.Unlock()
-	if !b.frozen {
-		b.frozen = true
-	}
-	return b.node
-}
-
-func (b *mBucket) get(r *Cache, h *mNode, hash uint32, ns, key uint64, noset bool) (done, added bool, n *Node) {
-	b.mu.Lock()
-
-	if b.frozen {
-		b.mu.Unlock()
-		return
-	}
-
-	// Scan the node.
-	for _, n := range b.node {
-		if n.hash == hash && n.ns == ns && n.key == key {
-			atomic.AddInt32(&n.ref, 1)
-			b.mu.Unlock()
-			return true, false, n
-		}
-	}
-
-	// Get only.
-	if noset {
-		b.mu.Unlock()
-		return true, false, nil
-	}
-
-	// Create node.
-	n = &Node{
-		r:    r,
-		hash: hash,
-		ns:   ns,
-		key:  key,
-		ref:  1,
-	}
-	// Add node to bucket.
-	b.node = append(b.node, n)
-	bLen := len(b.node)
-	b.mu.Unlock()
-
-	// Update counter.
-	grow := atomic.AddInt32(&r.nodes, 1) >= h.growThreshold
-	if bLen > mOverflowThreshold {
-		grow = grow || atomic.AddInt32(&h.overflow, 1) >= mOverflowGrowThreshold
-	}
-
-	// Grow.
-	if grow && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) {
-		nhLen := len(h.buckets) << 1
-		nh := &mNode{
-			buckets:         make([]unsafe.Pointer, nhLen),
-			mask:            uint32(nhLen) - 1,
-			pred:            unsafe.Pointer(h),
-			growThreshold:   int32(nhLen * mOverflowThreshold),
-			shrinkThreshold: int32(nhLen >> 1),
-		}
-		ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh))
-		if !ok {
-			panic("BUG: failed swapping head")
-		}
-		go nh.initBuckets()
-	}
-
-	return true, true, n
-}
-
-func (b *mBucket) delete(r *Cache, h *mNode, hash uint32, ns, key uint64) (done, deleted bool) {
-	b.mu.Lock()
-
-	if b.frozen {
-		b.mu.Unlock()
-		return
-	}
-
-	// Scan the node.
-	var (
-		n    *Node
-		bLen int
-	)
-	for i := range b.node {
-		n = b.node[i]
-		if n.ns == ns && n.key == key {
-			if atomic.LoadInt32(&n.ref) == 0 {
-				deleted = true
-
-				// Call releaser.
-				if n.value != nil {
-					if r, ok := n.value.(util.Releaser); ok {
-						r.Release()
-					}
-					n.value = nil
-				}
-
-				// Remove node from bucket.
-				b.node = append(b.node[:i], b.node[i+1:]...)
-				bLen = len(b.node)
-			}
-			break
-		}
-	}
-	b.mu.Unlock()
-
-	if deleted {
-		// Call OnDel.
-		for _, f := range n.onDel {
-			f()
-		}
-
-		// Update counter.
-		atomic.AddInt32(&r.size, int32(n.size)*-1)
-		shrink := atomic.AddInt32(&r.nodes, -1) < h.shrinkThreshold
-		if bLen >= mOverflowThreshold {
-			atomic.AddInt32(&h.overflow, -1)
-		}
-
-		// Shrink.
-		if shrink && len(h.buckets) > mInitialSize && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) {
-			nhLen := len(h.buckets) >> 1
-			nh := &mNode{
-				buckets:         make([]unsafe.Pointer, nhLen),
-				mask:            uint32(nhLen) - 1,
-				pred:            unsafe.Pointer(h),
-				growThreshold:   int32(nhLen * mOverflowThreshold),
-				shrinkThreshold: int32(nhLen >> 1),
-			}
-			ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh))
-			if !ok {
-				panic("BUG: failed swapping head")
-			}
-			go nh.initBuckets()
-		}
-	}
-
-	return true, deleted
-}
-
-type mNode struct {
-	buckets         []unsafe.Pointer // []*mBucket
-	mask            uint32
-	pred            unsafe.Pointer // *mNode
-	resizeInProgess int32
-
-	overflow        int32
-	growThreshold   int32
-	shrinkThreshold int32
-}
-
-func (n *mNode) initBucket(i uint32) *mBucket {
-	if b := (*mBucket)(atomic.LoadPointer(&n.buckets[i])); b != nil {
-		return b
-	}
-
-	p := (*mNode)(atomic.LoadPointer(&n.pred))
-	if p != nil {
-		var node []*Node
-		if n.mask > p.mask {
-			// Grow.
-			pb := (*mBucket)(atomic.LoadPointer(&p.buckets[i&p.mask]))
-			if pb == nil {
-				pb = p.initBucket(i & p.mask)
-			}
-			m := pb.freeze()
-			// Split nodes.
-			for _, x := range m {
-				if x.hash&n.mask == i {
-					node = append(node, x)
-				}
-			}
-		} else {
-			// Shrink.
-			pb0 := (*mBucket)(atomic.LoadPointer(&p.buckets[i]))
-			if pb0 == nil {
-				pb0 = p.initBucket(i)
-			}
-			pb1 := (*mBucket)(atomic.LoadPointer(&p.buckets[i+uint32(len(n.buckets))]))
-			if pb1 == nil {
-				pb1 = p.initBucket(i + uint32(len(n.buckets)))
-			}
-			m0 := pb0.freeze()
-			m1 := pb1.freeze()
-			// Merge nodes.
-			node = make([]*Node, 0, len(m0)+len(m1))
-			node = append(node, m0...)
-			node = append(node, m1...)
-		}
-		b := &mBucket{node: node}
-		if atomic.CompareAndSwapPointer(&n.buckets[i], nil, unsafe.Pointer(b)) {
-			if len(node) > mOverflowThreshold {
-				atomic.AddInt32(&n.overflow, int32(len(node)-mOverflowThreshold))
-			}
-			return b
-		}
-	}
-
-	return (*mBucket)(atomic.LoadPointer(&n.buckets[i]))
-}
-
-func (n *mNode) initBuckets() {
-	for i := range n.buckets {
-		n.initBucket(uint32(i))
-	}
-	atomic.StorePointer(&n.pred, nil)
-}
-
-// Cache is a 'cache map'.
-type Cache struct {
-	mu     sync.RWMutex
-	mHead  unsafe.Pointer // *mNode
-	nodes  int32
-	size   int32
-	cacher Cacher
-	closed bool
-}
-
-// NewCache creates a new 'cache map'. The cacher is optional and
-// may be nil.
-func NewCache(cacher Cacher) *Cache {
-	h := &mNode{
-		buckets:         make([]unsafe.Pointer, mInitialSize),
-		mask:            mInitialSize - 1,
-		growThreshold:   int32(mInitialSize * mOverflowThreshold),
-		shrinkThreshold: 0,
-	}
-	for i := range h.buckets {
-		h.buckets[i] = unsafe.Pointer(&mBucket{})
-	}
-	r := &Cache{
-		mHead:  unsafe.Pointer(h),
-		cacher: cacher,
-	}
-	return r
-}
-
-func (r *Cache) getBucket(hash uint32) (*mNode, *mBucket) {
-	h := (*mNode)(atomic.LoadPointer(&r.mHead))
-	i := hash & h.mask
-	b := (*mBucket)(atomic.LoadPointer(&h.buckets[i]))
-	if b == nil {
-		b = h.initBucket(i)
-	}
-	return h, b
-}
-
-func (r *Cache) delete(n *Node) bool {
-	for {
-		h, b := r.getBucket(n.hash)
-		done, deleted := b.delete(r, h, n.hash, n.ns, n.key)
-		if done {
-			return deleted
-		}
-	}
-	return false
-}
-
-// Nodes returns number of 'cache node' in the map.
-func (r *Cache) Nodes() int {
-	return int(atomic.LoadInt32(&r.nodes))
-}
-
-// Size returns sums of 'cache node' size in the map.
-func (r *Cache) Size() int {
-	return int(atomic.LoadInt32(&r.size))
-}
-
-// Capacity returns cache capacity.
-func (r *Cache) Capacity() int {
-	if r.cacher == nil {
-		return 0
-	}
-	return r.cacher.Capacity()
-}
-
-// SetCapacity sets cache capacity.
-func (r *Cache) SetCapacity(capacity int) {
-	if r.cacher != nil {
-		r.cacher.SetCapacity(capacity)
-	}
-}
-
-// Get gets 'cache node' with the given namespace and key.
-// If cache node is not found and setFunc is not nil, Get will atomically creates
-// the 'cache node' by calling setFunc. Otherwise Get will returns nil.
-//
-// The returned 'cache handle' should be released after use by calling Release
-// method.
-func (r *Cache) Get(ns, key uint64, setFunc func() (size int, value Value)) *Handle {
-	r.mu.RLock()
-	defer r.mu.RUnlock()
-	if r.closed {
-		return nil
-	}
-
-	hash := murmur32(ns, key, 0xf00)
-	for {
-		h, b := r.getBucket(hash)
-		done, _, n := b.get(r, h, hash, ns, key, setFunc == nil)
-		if done {
-			if n != nil {
-				n.mu.Lock()
-				if n.value == nil {
-					if setFunc == nil {
-						n.mu.Unlock()
-						n.unref()
-						return nil
-					}
-
-					n.size, n.value = setFunc()
-					if n.value == nil {
-						n.size = 0
-						n.mu.Unlock()
-						n.unref()
-						return nil
-					}
-					atomic.AddInt32(&r.size, int32(n.size))
-				}
-				n.mu.Unlock()
-				if r.cacher != nil {
-					r.cacher.Promote(n)
-				}
-				return &Handle{unsafe.Pointer(n)}
-			}
-
-			break
-		}
+func (h *fakeHandle) Value() interface{} {
+	if atomic.LoadUint32(&h.once) == 0 {
+		return h.value
 	}
 	return nil
 }
 
-// Delete removes and ban 'cache node' with the given namespace and key.
-// A banned 'cache node' will never inserted into the 'cache tree'. Ban
-// only attributed to the particular 'cache node', so when a 'cache node'
-// is recreated it will not be banned.
-//
-// If onDel is not nil, then it will be executed if such 'cache node'
-// doesn't exist or once the 'cache node' is released.
-//
-// Delete return true is such 'cache node' exist.
-func (r *Cache) Delete(ns, key uint64, onDel func()) bool {
-	r.mu.RLock()
-	defer r.mu.RUnlock()
-	if r.closed {
-		return false
-	}
-
-	hash := murmur32(ns, key, 0xf00)
-	for {
-		h, b := r.getBucket(hash)
-		done, _, n := b.get(r, h, hash, ns, key, true)
-		if done {
-			if n != nil {
-				if onDel != nil {
-					n.mu.Lock()
-					n.onDel = append(n.onDel, onDel)
-					n.mu.Unlock()
-				}
-				if r.cacher != nil {
-					r.cacher.Ban(n)
-				}
-				n.unref()
-				return true
-			}
-
-			break
-		}
-	}
-
-	if onDel != nil {
-		onDel()
-	}
-
-	return false
-}
-
-// Evict evicts 'cache node' with the given namespace and key. This will
-// simply call Cacher.Evict.
-//
-// Evict return true is such 'cache node' exist.
-func (r *Cache) Evict(ns, key uint64) bool {
-	r.mu.RLock()
-	defer r.mu.RUnlock()
-	if r.closed {
-		return false
-	}
-
-	hash := murmur32(ns, key, 0xf00)
-	for {
-		h, b := r.getBucket(hash)
-		done, _, n := b.get(r, h, hash, ns, key, true)
-		if done {
-			if n != nil {
-				if r.cacher != nil {
-					r.cacher.Evict(n)
-				}
-				n.unref()
-				return true
-			}
-
-			break
-		}
-	}
-
-	return false
-}
-
-// EvictNS evicts 'cache node' with the given namespace. This will
-// simply call Cacher.EvictNS.
-func (r *Cache) EvictNS(ns uint64) {
-	r.mu.RLock()
-	defer r.mu.RUnlock()
-	if r.closed {
+func (h *fakeHandle) Release() {
+	if !atomic.CompareAndSwapUint32(&h.once, 0, 1) {
 		return
 	}
-
-	if r.cacher != nil {
-		r.cacher.EvictNS(ns)
+	if h.fin != nil {
+		h.fin()
+		h.fin = nil
 	}
 }
-
-// EvictAll evicts all 'cache node'. This will simply call Cacher.EvictAll.
-func (r *Cache) EvictAll() {
-	r.mu.RLock()
-	defer r.mu.RUnlock()
-	if r.closed {
-		return
-	}
-
-	if r.cacher != nil {
-		r.cacher.EvictAll()
-	}
-}
-
-// Close closes the 'cache map' and releases all 'cache node'.
-func (r *Cache) Close() error {
-	r.mu.Lock()
-	if !r.closed {
-		r.closed = true
-
-		if r.cacher != nil {
-			if err := r.cacher.Close(); err != nil {
-				return err
-			}
-		}
-
-		h := (*mNode)(r.mHead)
-		h.initBuckets()
-
-		for i := range h.buckets {
-			b := (*mBucket)(h.buckets[i])
-			for _, n := range b.node {
-				// Call releaser.
-				if n.value != nil {
-					if r, ok := n.value.(util.Releaser); ok {
-						r.Release()
-					}
-					n.value = nil
-				}
-
-				// Call OnDel.
-				for _, f := range n.onDel {
-					f()
-				}
-			}
-		}
-	}
-	r.mu.Unlock()
-	return nil
-}
-
-// Node is a 'cache node'.
-type Node struct {
-	r *Cache
-
-	hash    uint32
-	ns, key uint64
-
-	mu    sync.Mutex
-	size  int
-	value Value
-
-	ref   int32
-	onDel []func()
-
-	CacheData unsafe.Pointer
-}
-
-// NS returns this 'cache node' namespace.
-func (n *Node) NS() uint64 {
-	return n.ns
-}
-
-// Key returns this 'cache node' key.
-func (n *Node) Key() uint64 {
-	return n.key
-}
-
-// Size returns this 'cache node' size.
-func (n *Node) Size() int {
-	return n.size
-}
-
-// Value returns this 'cache node' value.
-func (n *Node) Value() Value {
-	return n.value
-}
-
-// Ref returns this 'cache node' ref counter.
-func (n *Node) Ref() int32 {
-	return atomic.LoadInt32(&n.ref)
-}
-
-// GetHandle returns an handle for this 'cache node'.
-func (n *Node) GetHandle() *Handle {
-	if atomic.AddInt32(&n.ref, 1) <= 1 {
-		panic("BUG: Node.GetHandle on zero ref")
-	}
-	return &Handle{unsafe.Pointer(n)}
-}
-
-func (n *Node) unref() {
-	if atomic.AddInt32(&n.ref, -1) == 0 {
-		n.r.delete(n)
-	}
-}
-
-func (n *Node) unrefLocked() {
-	if atomic.AddInt32(&n.ref, -1) == 0 {
-		n.r.mu.RLock()
-		if !n.r.closed {
-			n.r.delete(n)
-		}
-		n.r.mu.RUnlock()
-	}
-}
-
-type Handle struct {
-	n unsafe.Pointer // *Node
-}
-
-func (h *Handle) Value() Value {
-	n := (*Node)(atomic.LoadPointer(&h.n))
-	if n != nil {
-		return n.value
-	}
-	return nil
-}
-
-func (h *Handle) Release() {
-	nPtr := atomic.LoadPointer(&h.n)
-	if nPtr != nil && atomic.CompareAndSwapPointer(&h.n, nPtr, nil) {
-		n := (*Node)(nPtr)
-		n.unrefLocked()
-	}
-}
-
-func murmur32(ns, key uint64, seed uint32) uint32 {
-	const (
-		m = uint32(0x5bd1e995)
-		r = 24
-	)
-
-	k1 := uint32(ns >> 32)
-	k2 := uint32(ns)
-	k3 := uint32(key >> 32)
-	k4 := uint32(key)
-
-	k1 *= m
-	k1 ^= k1 >> r
-	k1 *= m
-
-	k2 *= m
-	k2 ^= k2 >> r
-	k2 *= m
-
-	k3 *= m
-	k3 ^= k3 >> r
-	k3 *= m
-
-	k4 *= m
-	k4 ^= k4 >> r
-	k4 *= m
-
-	h := seed
-
-	h *= m
-	h ^= k1
-	h *= m
-	h ^= k2
-	h *= m
-	h ^= k3
-	h *= m
-	h ^= k4
-
-	h ^= h >> 13
-	h *= m
-	h ^= h >> 15
-
-	return h
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go

@@ -7,32 +7,19 @@
 package cache
 
 import (
+	"fmt"
 	"math/rand"
 	"runtime"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
-	"unsafe"
 )
 
-type int32o int32
-
-func (o *int32o) acquire() {
-	if atomic.AddInt32((*int32)(o), 1) != 1 {
-		panic("BUG: invalid ref")
-	}
-}
-
-func (o *int32o) Release() {
-	if atomic.AddInt32((*int32)(o), -1) != 0 {
-		panic("BUG: invalid ref")
-	}
-}
-
 type releaserFunc struct {
 	fn    func()
-	value Value
+	value interface{}
 }
 
 func (r releaserFunc) Release() {
@@ -41,8 +28,8 @@ func (r releaserFunc) Release() {
 	}
 }
 
-func set(c *Cache, ns, key uint64, value Value, charge int, relf func()) *Handle {
-	return c.Get(ns, key, func() (int, Value) {
+func set(ns Namespace, key uint64, value interface{}, charge int, relf func()) Handle {
+	return ns.Get(key, func() (int, interface{}) {
 		if relf != nil {
 			return charge, releaserFunc{relf, value}
 		} else {
@@ -51,246 +38,7 @@ func set(c *Cache, ns, key uint64, value Value, charge int, relf func()) *Handle
 	})
 }
 
-func TestCacheMap(t *testing.T) {
-	runtime.GOMAXPROCS(runtime.NumCPU())
-
-	nsx := []struct {
-		nobjects, nhandles, concurrent, repeat int
-	}{
-		{10000, 400, 50, 3},
-		{100000, 1000, 100, 10},
-	}
-
-	var (
-		objects [][]int32o
-		handles [][]unsafe.Pointer
-	)
-
-	for _, x := range nsx {
-		objects = append(objects, make([]int32o, x.nobjects))
-		handles = append(handles, make([]unsafe.Pointer, x.nhandles))
-	}
-
-	c := NewCache(nil)
-
-	wg := new(sync.WaitGroup)
-	var done int32
-
-	for ns, x := range nsx {
-		for i := 0; i < x.concurrent; i++ {
-			wg.Add(1)
-			go func(ns, i, repeat int, objects []int32o, handles []unsafe.Pointer) {
-				defer wg.Done()
-				r := rand.New(rand.NewSource(time.Now().UnixNano()))
-
-				for j := len(objects) * repeat; j >= 0; j-- {
-					key := uint64(r.Intn(len(objects)))
-					h := c.Get(uint64(ns), key, func() (int, Value) {
-						o := &objects[key]
-						o.acquire()
-						return 1, o
-					})
-					if v := h.Value().(*int32o); v != &objects[key] {
-						t.Fatalf("#%d invalid value: want=%p got=%p", ns, &objects[key], v)
-					}
-					if objects[key] != 1 {
-						t.Fatalf("#%d invalid object %d: %d", ns, key, objects[key])
-					}
-					if !atomic.CompareAndSwapPointer(&handles[r.Intn(len(handles))], nil, unsafe.Pointer(h)) {
-						h.Release()
-					}
-				}
-			}(ns, i, x.repeat, objects[ns], handles[ns])
-		}
-
-		go func(handles []unsafe.Pointer) {
-			r := rand.New(rand.NewSource(time.Now().UnixNano()))
-
-			for atomic.LoadInt32(&done) == 0 {
-				i := r.Intn(len(handles))
-				h := (*Handle)(atomic.LoadPointer(&handles[i]))
-				if h != nil && atomic.CompareAndSwapPointer(&handles[i], unsafe.Pointer(h), nil) {
-					h.Release()
-				}
-				time.Sleep(time.Millisecond)
-			}
-		}(handles[ns])
-	}
-
-	go func() {
-		handles := make([]*Handle, 100000)
-		for atomic.LoadInt32(&done) == 0 {
-			for i := range handles {
-				handles[i] = c.Get(999999999, uint64(i), func() (int, Value) {
-					return 1, 1
-				})
-			}
-			for _, h := range handles {
-				h.Release()
-			}
-		}
-	}()
-
-	wg.Wait()
-
-	atomic.StoreInt32(&done, 1)
-
-	for _, handles0 := range handles {
-		for i := range handles0 {
-			h := (*Handle)(atomic.LoadPointer(&handles0[i]))
-			if h != nil && atomic.CompareAndSwapPointer(&handles0[i], unsafe.Pointer(h), nil) {
-				h.Release()
-			}
-		}
-	}
-
-	for ns, objects0 := range objects {
-		for i, o := range objects0 {
-			if o != 0 {
-				t.Fatalf("invalid object #%d.%d: ref=%d", ns, i, o)
-			}
-		}
-	}
-}
-
-func TestCacheMap_NodesAndSize(t *testing.T) {
-	c := NewCache(nil)
-	if c.Nodes() != 0 {
-		t.Errorf("invalid nodes counter: want=%d got=%d", 0, c.Nodes())
-	}
-	if c.Size() != 0 {
-		t.Errorf("invalid size counter: want=%d got=%d", 0, c.Size())
-	}
-	set(c, 0, 1, 1, 1, nil)
-	set(c, 0, 2, 2, 2, nil)
-	set(c, 1, 1, 3, 3, nil)
-	set(c, 2, 1, 4, 1, nil)
-	if c.Nodes() != 4 {
-		t.Errorf("invalid nodes counter: want=%d got=%d", 4, c.Nodes())
-	}
-	if c.Size() != 7 {
-		t.Errorf("invalid size counter: want=%d got=%d", 4, c.Size())
-	}
-}
-
-func TestLRUCache_Capacity(t *testing.T) {
-	c := NewCache(NewLRU(10))
-	if c.Capacity() != 10 {
-		t.Errorf("invalid capacity: want=%d got=%d", 10, c.Capacity())
-	}
-	set(c, 0, 1, 1, 1, nil).Release()
-	set(c, 0, 2, 2, 2, nil).Release()
-	set(c, 1, 1, 3, 3, nil).Release()
-	set(c, 2, 1, 4, 1, nil).Release()
-	set(c, 2, 2, 5, 1, nil).Release()
-	set(c, 2, 3, 6, 1, nil).Release()
-	set(c, 2, 4, 7, 1, nil).Release()
-	set(c, 2, 5, 8, 1, nil).Release()
-	if c.Nodes() != 7 {
-		t.Errorf("invalid nodes counter: want=%d got=%d", 7, c.Nodes())
-	}
-	if c.Size() != 10 {
-		t.Errorf("invalid size counter: want=%d got=%d", 10, c.Size())
-	}
-	c.SetCapacity(9)
-	if c.Capacity() != 9 {
-		t.Errorf("invalid capacity: want=%d got=%d", 9, c.Capacity())
-	}
-	if c.Nodes() != 6 {
-		t.Errorf("invalid nodes counter: want=%d got=%d", 6, c.Nodes())
-	}
-	if c.Size() != 8 {
-		t.Errorf("invalid size counter: want=%d got=%d", 8, c.Size())
-	}
-}
-
-func TestCacheMap_NilValue(t *testing.T) {
-	c := NewCache(NewLRU(10))
-	h := c.Get(0, 0, func() (size int, value Value) {
-		return 1, nil
-	})
-	if h != nil {
-		t.Error("cache handle is non-nil")
-	}
-	if c.Nodes() != 0 {
-		t.Errorf("invalid nodes counter: want=%d got=%d", 0, c.Nodes())
-	}
-	if c.Size() != 0 {
-		t.Errorf("invalid size counter: want=%d got=%d", 0, c.Size())
-	}
-}
-
-func TestLRUCache_GetLatency(t *testing.T) {
-	runtime.GOMAXPROCS(runtime.NumCPU())
-
-	const (
-		concurrentSet = 30
-		concurrentGet = 3
-		duration      = 3 * time.Second
-		delay         = 3 * time.Millisecond
-		maxkey        = 100000
-	)
-
-	var (
-		set, getHit, getAll        int32
-		getMaxLatency, getDuration int64
-	)
-
-	c := NewCache(NewLRU(5000))
-	wg := &sync.WaitGroup{}
-	until := time.Now().Add(duration)
-	for i := 0; i < concurrentSet; i++ {
-		wg.Add(1)
-		go func(i int) {
-			defer wg.Done()
-			r := rand.New(rand.NewSource(time.Now().UnixNano()))
-			for time.Now().Before(until) {
-				c.Get(0, uint64(r.Intn(maxkey)), func() (int, Value) {
-					time.Sleep(delay)
-					atomic.AddInt32(&set, 1)
-					return 1, 1
-				}).Release()
-			}
-		}(i)
-	}
-	for i := 0; i < concurrentGet; i++ {
-		wg.Add(1)
-		go func(i int) {
-			defer wg.Done()
-			r := rand.New(rand.NewSource(time.Now().UnixNano()))
-			for {
-				mark := time.Now()
-				if mark.Before(until) {
-					h := c.Get(0, uint64(r.Intn(maxkey)), nil)
-					latency := int64(time.Now().Sub(mark))
-					m := atomic.LoadInt64(&getMaxLatency)
-					if latency > m {
-						atomic.CompareAndSwapInt64(&getMaxLatency, m, latency)
-					}
-					atomic.AddInt64(&getDuration, latency)
-					if h != nil {
-						atomic.AddInt32(&getHit, 1)
-						h.Release()
-					}
-					atomic.AddInt32(&getAll, 1)
-				} else {
-					break
-				}
-			}
-		}(i)
-	}
-
-	wg.Wait()
-	getAvglatency := time.Duration(getDuration) / time.Duration(getAll)
-	t.Logf("set=%d getHit=%d getAll=%d getMaxLatency=%v getAvgLatency=%v",
-		set, getHit, getAll, time.Duration(getMaxLatency), getAvglatency)
-
-	if getAvglatency > delay/3 {
-		t.Errorf("get avg latency > %v: got=%v", delay/3, getAvglatency)
-	}
-}
-
-func TestLRUCache_HitMiss(t *testing.T) {
+func TestCache_HitMiss(t *testing.T) {
 	cases := []struct {
 		key   uint64
 		value string
@@ -308,13 +56,14 @@ func TestLRUCache_HitMiss(t *testing.T) {
 	}
 
 	setfin := 0
-	c := NewCache(NewLRU(1000))
+	c := NewLRUCache(1000)
+	ns := c.GetNamespace(0)
 	for i, x := range cases {
-		set(c, 0, x.key, x.value, len(x.value), func() {
+		set(ns, x.key, x.value, len(x.value), func() {
 			setfin++
 		}).Release()
 		for j, y := range cases {
-			h := c.Get(0, y.key, nil)
+			h := ns.Get(y.key, nil)
 			if j <= i {
 				// should hit
 				if h == nil {
@@ -338,7 +87,7 @@ func TestLRUCache_HitMiss(t *testing.T) {
 
 	for i, x := range cases {
 		finalizerOk := false
-		c.Delete(0, x.key, func() {
+		ns.Delete(x.key, func(exist, pending bool) {
 			finalizerOk = true
 		})
 
@@ -347,7 +96,7 @@ func TestLRUCache_HitMiss(t *testing.T) {
 		}
 
 		for j, y := range cases {
-			h := c.Get(0, y.key, nil)
+			h := ns.Get(y.key, nil)
 			if j > i {
 				// should hit
 				if h == nil {
@@ -375,19 +124,20 @@ func TestLRUCache_HitMiss(t *testing.T) {
 }
 
 func TestLRUCache_Eviction(t *testing.T) {
-	c := NewCache(NewLRU(12))
-	o1 := set(c, 0, 1, 1, 1, nil)
-	set(c, 0, 2, 2, 1, nil).Release()
-	set(c, 0, 3, 3, 1, nil).Release()
-	set(c, 0, 4, 4, 1, nil).Release()
-	set(c, 0, 5, 5, 1, nil).Release()
-	if h := c.Get(0, 2, nil); h != nil { // 1,3,4,5,2
+	c := NewLRUCache(12)
+	ns := c.GetNamespace(0)
+	o1 := set(ns, 1, 1, 1, nil)
+	set(ns, 2, 2, 1, nil).Release()
+	set(ns, 3, 3, 1, nil).Release()
+	set(ns, 4, 4, 1, nil).Release()
+	set(ns, 5, 5, 1, nil).Release()
+	if h := ns.Get(2, nil); h != nil { // 1,3,4,5,2
 		h.Release()
 	}
-	set(c, 0, 9, 9, 10, nil).Release() // 5,2,9
+	set(ns, 9, 9, 10, nil).Release() // 5,2,9
 
 	for _, key := range []uint64{9, 2, 5, 1} {
-		h := c.Get(0, key, nil)
+		h := ns.Get(key, nil)
 		if h == nil {
 			t.Errorf("miss for key '%d'", key)
 		} else {
@@ -399,7 +149,7 @@ func TestLRUCache_Eviction(t *testing.T) {
 	}
 	o1.Release()
 	for _, key := range []uint64{1, 2, 5} {
-		h := c.Get(0, key, nil)
+		h := ns.Get(key, nil)
 		if h == nil {
 			t.Errorf("miss for key '%d'", key)
 		} else {
@@ -410,7 +160,7 @@ func TestLRUCache_Eviction(t *testing.T) {
 		}
 	}
 	for _, key := range []uint64{3, 4, 9} {
-		h := c.Get(0, key, nil)
+		h := ns.Get(key, nil)
 		if h != nil {
 			t.Errorf("hit for key '%d'", key)
 			if x := h.Value().(int); x != int(key) {
@@ -421,134 +171,377 @@ func TestLRUCache_Eviction(t *testing.T) {
 	}
 }
 
-func TestLRUCache_Evict(t *testing.T) {
-	c := NewCache(NewLRU(6))
-	set(c, 0, 1, 1, 1, nil).Release()
-	set(c, 0, 2, 2, 1, nil).Release()
-	set(c, 1, 1, 4, 1, nil).Release()
-	set(c, 1, 2, 5, 1, nil).Release()
-	set(c, 2, 1, 6, 1, nil).Release()
-	set(c, 2, 2, 7, 1, nil).Release()
-
-	for ns := 0; ns < 3; ns++ {
-		for key := 1; key < 3; key++ {
-			if h := c.Get(uint64(ns), uint64(key), nil); h != nil {
-				h.Release()
+func TestLRUCache_SetGet(t *testing.T) {
+	c := NewLRUCache(13)
+	ns := c.GetNamespace(0)
+	for i := 0; i < 200; i++ {
+		n := uint64(rand.Intn(99999) % 20)
+		set(ns, n, n, 1, nil).Release()
+		if h := ns.Get(n, nil); h != nil {
+			if h.Value() == nil {
+				t.Errorf("key '%d' contains nil value", n)
 			} else {
-				t.Errorf("Cache.Get on #%d.%d return nil", ns, key)
+				if x := h.Value().(uint64); x != n {
+					t.Errorf("invalid value for key '%d' want '%d', got '%d'", n, n, x)
+				}
 			}
-		}
-	}
-
-	if ok := c.Evict(0, 1); !ok {
-		t.Error("first Cache.Evict on #0.1 return false")
-	}
-	if ok := c.Evict(0, 1); ok {
-		t.Error("second Cache.Evict on #0.1 return true")
-	}
-	if h := c.Get(0, 1, nil); h != nil {
-		t.Errorf("Cache.Get on #0.1 return non-nil: %v", h.Value())
-	}
-
-	c.EvictNS(1)
-	if h := c.Get(1, 1, nil); h != nil {
-		t.Errorf("Cache.Get on #1.1 return non-nil: %v", h.Value())
-	}
-	if h := c.Get(1, 2, nil); h != nil {
-		t.Errorf("Cache.Get on #1.2 return non-nil: %v", h.Value())
-	}
-
-	c.EvictAll()
-	for ns := 0; ns < 3; ns++ {
-		for key := 1; key < 3; key++ {
-			if h := c.Get(uint64(ns), uint64(key), nil); h != nil {
-				t.Errorf("Cache.Get on #%d.%d return non-nil: %v", ns, key, h.Value())
-			}
-		}
-	}
-}
-
-func TestLRUCache_Delete(t *testing.T) {
-	delFuncCalled := 0
-	delFunc := func() {
-		delFuncCalled++
-	}
-
-	c := NewCache(NewLRU(2))
-	set(c, 0, 1, 1, 1, nil).Release()
-	set(c, 0, 2, 2, 1, nil).Release()
-
-	if ok := c.Delete(0, 1, delFunc); !ok {
-		t.Error("Cache.Delete on #1 return false")
-	}
-	if h := c.Get(0, 1, nil); h != nil {
-		t.Errorf("Cache.Get on #1 return non-nil: %v", h.Value())
-	}
-	if ok := c.Delete(0, 1, delFunc); ok {
-		t.Error("Cache.Delete on #1 return true")
-	}
-
-	h2 := c.Get(0, 2, nil)
-	if h2 == nil {
-		t.Error("Cache.Get on #2 return nil")
-	}
-	if ok := c.Delete(0, 2, delFunc); !ok {
-		t.Error("(1) Cache.Delete on #2 return false")
-	}
-	if ok := c.Delete(0, 2, delFunc); !ok {
-		t.Error("(2) Cache.Delete on #2 return false")
-	}
-
-	set(c, 0, 3, 3, 1, nil).Release()
-	set(c, 0, 4, 4, 1, nil).Release()
-	c.Get(0, 2, nil).Release()
-
-	for key := 2; key <= 4; key++ {
-		if h := c.Get(0, uint64(key), nil); h != nil {
 			h.Release()
 		} else {
-			t.Errorf("Cache.Get on #%d return nil", key)
+			t.Errorf("key '%d' doesn't exist", n)
+		}
+	}
+}
+
+func TestLRUCache_Purge(t *testing.T) {
+	c := NewLRUCache(3)
+	ns1 := c.GetNamespace(0)
+	o1 := set(ns1, 1, 1, 1, nil)
+	o2 := set(ns1, 2, 2, 1, nil)
+	ns1.Purge(nil)
+	set(ns1, 3, 3, 1, nil).Release()
+	for _, key := range []uint64{1, 2, 3} {
+		h := ns1.Get(key, nil)
+		if h == nil {
+			t.Errorf("miss for key '%d'", key)
+		} else {
+			if x := h.Value().(int); x != int(key) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
+			}
+			h.Release()
+		}
+	}
+	o1.Release()
+	o2.Release()
+	for _, key := range []uint64{1, 2} {
+		h := ns1.Get(key, nil)
+		if h != nil {
+			t.Errorf("hit for key '%d'", key)
+			if x := h.Value().(int); x != int(key) {
+				t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
+			}
+			h.Release()
+		}
+	}
+}
+
+type testingCacheObjectCounter struct {
+	created  uint32
+	released uint32
+}
+
+func (c *testingCacheObjectCounter) createOne() {
+	atomic.AddUint32(&c.created, 1)
+}
+
+func (c *testingCacheObjectCounter) releaseOne() {
+	atomic.AddUint32(&c.released, 1)
+}
+
+type testingCacheObject struct {
+	t   *testing.T
+	cnt *testingCacheObjectCounter
+
+	ns, key uint64
+
+	releaseCalled uint32
+}
+
+func (x *testingCacheObject) Release() {
+	if atomic.CompareAndSwapUint32(&x.releaseCalled, 0, 1) {
+		x.cnt.releaseOne()
+	} else {
+		x.t.Errorf("duplicate setfin NS#%d KEY#%s", x.ns, x.key)
+	}
+}
+
+func TestLRUCache_Finalizer(t *testing.T) {
+	const (
+		capacity   = 100
+		goroutines = 100
+		iterations = 10000
+		keymax     = 8000
+	)
+
+	runtime.GOMAXPROCS(runtime.NumCPU())
+	defer runtime.GOMAXPROCS(1)
+
+	wg := &sync.WaitGroup{}
+	cnt := &testingCacheObjectCounter{}
+
+	c := NewLRUCache(capacity)
+
+	type instance struct {
+		seed       int64
+		rnd        *rand.Rand
+		ns         uint64
+		effective  int32
+		handles    []Handle
+		handlesMap map[uint64]int
+
+		delete          bool
+		purge           bool
+		zap             bool
+		wantDel         int32
+		delfinCalledAll int32
+		delfinCalledEff int32
+		purgefinCalled  int32
+	}
+
+	instanceGet := func(p *instance, ns Namespace, key uint64) {
+		h := ns.Get(key, func() (charge int, value interface{}) {
+			to := &testingCacheObject{
+				t: t, cnt: cnt,
+				ns:  p.ns,
+				key: key,
+			}
+			atomic.AddInt32(&p.effective, 1)
+			cnt.createOne()
+			return 1, releaserFunc{func() {
+				to.Release()
+				atomic.AddInt32(&p.effective, -1)
+			}, to}
+		})
+		p.handles = append(p.handles, h)
+		p.handlesMap[key] = p.handlesMap[key] + 1
+	}
+	instanceRelease := func(p *instance, ns Namespace, i int) {
+		h := p.handles[i]
+		key := h.Value().(releaserFunc).value.(*testingCacheObject).key
+		if n := p.handlesMap[key]; n == 0 {
+			t.Fatal("key ref == 0")
+		} else if n > 1 {
+			p.handlesMap[key] = n - 1
+		} else {
+			delete(p.handlesMap, key)
+		}
+		h.Release()
+		p.handles = append(p.handles[:i], p.handles[i+1:]...)
+		p.handles[len(p.handles) : len(p.handles)+1][0] = nil
+	}
+
+	seeds := make([]int64, goroutines)
+	instances := make([]instance, goroutines)
+	for i := range instances {
+		p := &instances[i]
+		p.handlesMap = make(map[uint64]int)
+		if seeds[i] == 0 {
+			seeds[i] = time.Now().UnixNano()
+		}
+		p.seed = seeds[i]
+		p.rnd = rand.New(rand.NewSource(p.seed))
+		p.ns = uint64(i)
+		p.delete = i%6 == 0
+		p.purge = i%8 == 0
+		p.zap = i%12 == 0 || i%3 == 0
+	}
+
+	seedsStr := make([]string, len(seeds))
+	for i, seed := range seeds {
+		seedsStr[i] = fmt.Sprint(seed)
+	}
+	t.Logf("seeds := []int64{%s}", strings.Join(seedsStr, ", "))
+
+	// Get and release.
+	for i := range instances {
+		p := &instances[i]
+
+		wg.Add(1)
+		go func(p *instance) {
+			defer wg.Done()
+
+			ns := c.GetNamespace(p.ns)
+			for i := 0; i < iterations; i++ {
+				if len(p.handles) == 0 || p.rnd.Int()%2 == 0 {
+					instanceGet(p, ns, uint64(p.rnd.Intn(keymax)))
+				} else {
+					instanceRelease(p, ns, p.rnd.Intn(len(p.handles)))
+				}
+			}
+		}(p)
+	}
+	wg.Wait()
+
+	if used, cap := c.Used(), c.Capacity(); used > cap {
+		t.Errorf("Used > capacity, used=%d cap=%d", used, cap)
+	}
+
+	// Check effective objects.
+	for i := range instances {
+		p := &instances[i]
+		if int(p.effective) < len(p.handlesMap) {
+			t.Errorf("#%d effective objects < acquired handle, eo=%d ah=%d", i, p.effective, len(p.handlesMap))
 		}
 	}
 
-	h2.Release()
-	if h := c.Get(0, 2, nil); h != nil {
-		t.Errorf("Cache.Get on #2 return non-nil: %v", h.Value())
+	if want := int(cnt.created - cnt.released); c.Size() != want {
+		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
 	}
 
-	if delFuncCalled != 4 {
-		t.Errorf("delFunc isn't called 4 times: got=%d", delFuncCalled)
+	// Delete and purge.
+	for i := range instances {
+		p := &instances[i]
+		p.wantDel = p.effective
+
+		wg.Add(1)
+		go func(p *instance) {
+			defer wg.Done()
+
+			ns := c.GetNamespace(p.ns)
+
+			if p.delete {
+				for key := uint64(0); key < keymax; key++ {
+					_, wantExist := p.handlesMap[key]
+					gotExist := ns.Delete(key, func(exist, pending bool) {
+						atomic.AddInt32(&p.delfinCalledAll, 1)
+						if exist {
+							atomic.AddInt32(&p.delfinCalledEff, 1)
+						}
+					})
+					if !gotExist && wantExist {
+						t.Errorf("delete on NS#%d KEY#%d not found", p.ns, key)
+					}
+				}
+
+				var delfinCalled int
+				for key := uint64(0); key < keymax; key++ {
+					func(key uint64) {
+						gotExist := ns.Delete(key, func(exist, pending bool) {
+							if exist && !pending {
+								t.Errorf("delete fin on NS#%d KEY#%d exist and not pending for deletion", p.ns, key)
+							}
+							delfinCalled++
+						})
+						if gotExist {
+							t.Errorf("delete on NS#%d KEY#%d found", p.ns, key)
+						}
+					}(key)
+				}
+				if delfinCalled != keymax {
+					t.Errorf("(2) #%d not all delete fin called, diff=%d", p.ns, keymax-delfinCalled)
+				}
+			}
+
+			if p.purge {
+				ns.Purge(func(ns, key uint64) {
+					atomic.AddInt32(&p.purgefinCalled, 1)
+				})
+			}
+		}(p)
+	}
+	wg.Wait()
+
+	if want := int(cnt.created - cnt.released); c.Size() != want {
+		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
+	}
+
+	// Release.
+	for i := range instances {
+		p := &instances[i]
+
+		if !p.zap {
+			wg.Add(1)
+			go func(p *instance) {
+				defer wg.Done()
+
+				ns := c.GetNamespace(p.ns)
+				for i := len(p.handles) - 1; i >= 0; i-- {
+					instanceRelease(p, ns, i)
+				}
+			}(p)
+		}
+	}
+	wg.Wait()
+
+	if want := int(cnt.created - cnt.released); c.Size() != want {
+		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
+	}
+
+	// Zap.
+	for i := range instances {
+		p := &instances[i]
+
+		if p.zap {
+			wg.Add(1)
+			go func(p *instance) {
+				defer wg.Done()
+
+				ns := c.GetNamespace(p.ns)
+				ns.Zap()
+
+				p.handles = nil
+				p.handlesMap = nil
+			}(p)
+		}
+	}
+	wg.Wait()
+
+	if want := int(cnt.created - cnt.released); c.Size() != want {
+		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
+	}
+
+	if notrel, used := int(cnt.created-cnt.released), c.Used(); notrel != used {
+		t.Errorf("Invalid used value, want=%d got=%d", notrel, used)
+	}
+
+	c.Purge(nil)
+
+	for i := range instances {
+		p := &instances[i]
+
+		if p.delete {
+			if p.delfinCalledAll != keymax {
+				t.Errorf("#%d not all delete fin called, purge=%v zap=%v diff=%d", p.ns, p.purge, p.zap, keymax-p.delfinCalledAll)
+			}
+			if p.delfinCalledEff != p.wantDel {
+				t.Errorf("#%d not all effective delete fin called, diff=%d", p.ns, p.wantDel-p.delfinCalledEff)
+			}
+			if p.purge && p.purgefinCalled > 0 {
+				t.Errorf("#%d some purge fin called, delete=%v zap=%v n=%d", p.ns, p.delete, p.zap, p.purgefinCalled)
+			}
+		} else {
+			if p.purge {
+				if p.purgefinCalled != p.wantDel {
+					t.Errorf("#%d not all purge fin called, delete=%v zap=%v diff=%d", p.ns, p.delete, p.zap, p.wantDel-p.purgefinCalled)
+				}
+			}
+		}
+	}
+
+	if cnt.created != cnt.released {
+		t.Errorf("Some cache object weren't released, created=%d released=%d", cnt.created, cnt.released)
 	}
 }
 
-func TestLRUCache_Close(t *testing.T) {
-	relFuncCalled := 0
-	relFunc := func() {
-		relFuncCalled++
+func BenchmarkLRUCache_SetRelease(b *testing.B) {
+	capacity := b.N / 100
+	if capacity <= 0 {
+		capacity = 10
 	}
-	delFuncCalled := 0
-	delFunc := func() {
-		delFuncCalled++
-	}
-
-	c := NewCache(NewLRU(2))
-	set(c, 0, 1, 1, 1, relFunc).Release()
-	set(c, 0, 2, 2, 1, relFunc).Release()
-
-	h3 := set(c, 0, 3, 3, 1, relFunc)
-	if h3 == nil {
-		t.Error("Cache.Get on #3 return nil")
-	}
-	if ok := c.Delete(0, 3, delFunc); !ok {
-		t.Error("Cache.Delete on #3 return false")
-	}
-
-	c.Close()
-
-	if relFuncCalled != 3 {
-		t.Errorf("relFunc isn't called 3 times: got=%d", relFuncCalled)
-	}
-	if delFuncCalled != 1 {
-		t.Errorf("delFunc isn't called 1 times: got=%d", delFuncCalled)
+	c := NewLRUCache(capacity)
+	ns := c.GetNamespace(0)
+	b.ResetTimer()
+	for i := uint64(0); i < uint64(b.N); i++ {
+		set(ns, i, nil, 1, nil).Release()
+	}
+}
+
+func BenchmarkLRUCache_SetReleaseTwice(b *testing.B) {
+	capacity := b.N / 100
+	if capacity <= 0 {
+		capacity = 10
+	}
+	c := NewLRUCache(capacity)
+	ns := c.GetNamespace(0)
+	b.ResetTimer()
+
+	na := b.N / 2
+	nb := b.N - na
+
+	for i := uint64(0); i < uint64(na); i++ {
+		set(ns, i, nil, 1, nil).Release()
+	}
+
+	for i := uint64(0); i < uint64(nb); i++ {
+		set(ns, i, nil, 1, nil).Release()
 	}
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru.go

@@ -1,195 +0,0 @@
-// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
-// All rights reserved.
-//
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-package cache
-
-import (
-	"sync"
-	"unsafe"
-)
-
-type lruNode struct {
-	n   *Node
-	h   *Handle
-	ban bool
-
-	next, prev *lruNode
-}
-
-func (n *lruNode) insert(at *lruNode) {
-	x := at.next
-	at.next = n
-	n.prev = at
-	n.next = x
-	x.prev = n
-}
-
-func (n *lruNode) remove() {
-	if n.prev != nil {
-		n.prev.next = n.next
-		n.next.prev = n.prev
-		n.prev = nil
-		n.next = nil
-	} else {
-		panic("BUG: removing removed node")
-	}
-}
-
-type lru struct {
-	mu       sync.Mutex
-	capacity int
-	used     int
-	recent   lruNode
-}
-
-func (r *lru) reset() {
-	r.recent.next = &r.recent
-	r.recent.prev = &r.recent
-	r.used = 0
-}
-
-func (r *lru) Capacity() int {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-	return r.capacity
-}
-
-func (r *lru) SetCapacity(capacity int) {
-	var evicted []*lruNode
-
-	r.mu.Lock()
-	r.capacity = capacity
-	for r.used > r.capacity {
-		rn := r.recent.prev
-		if rn == nil {
-			panic("BUG: invalid LRU used or capacity counter")
-		}
-		rn.remove()
-		rn.n.CacheData = nil
-		r.used -= rn.n.Size()
-		evicted = append(evicted, rn)
-	}
-	r.mu.Unlock()
-
-	for _, rn := range evicted {
-		rn.h.Release()
-	}
-}
-
-func (r *lru) Promote(n *Node) {
-	var evicted []*lruNode
-
-	r.mu.Lock()
-	if n.CacheData == nil {
-		if n.Size() <= r.capacity {
-			rn := &lruNode{n: n, h: n.GetHandle()}
-			rn.insert(&r.recent)
-			n.CacheData = unsafe.Pointer(rn)
-			r.used += n.Size()
-
-			for r.used > r.capacity {
-				rn := r.recent.prev
-				if rn == nil {
-					panic("BUG: invalid LRU used or capacity counter")
-				}
-				rn.remove()
-				rn.n.CacheData = nil
-				r.used -= rn.n.Size()
-				evicted = append(evicted, rn)
-			}
-		}
-	} else {
-		rn := (*lruNode)(n.CacheData)
-		if !rn.ban {
-			rn.remove()
-			rn.insert(&r.recent)
-		}
-	}
-	r.mu.Unlock()
-
-	for _, rn := range evicted {
-		rn.h.Release()
-	}
-}
-
-func (r *lru) Ban(n *Node) {
-	r.mu.Lock()
-	if n.CacheData == nil {
-		n.CacheData = unsafe.Pointer(&lruNode{n: n, ban: true})
-	} else {
-		rn := (*lruNode)(n.CacheData)
-		if !rn.ban {
-			rn.remove()
-			rn.ban = true
-			r.used -= rn.n.Size()
-			r.mu.Unlock()
-
-			rn.h.Release()
-			rn.h = nil
-			return
-		}
-	}
-	r.mu.Unlock()
-}
-
-func (r *lru) Evict(n *Node) {
-	r.mu.Lock()
-	rn := (*lruNode)(n.CacheData)
-	if rn == nil || rn.ban {
-		r.mu.Unlock()
-		return
-	}
-	n.CacheData = nil
-	r.mu.Unlock()
-
-	rn.h.Release()
-}
-
-func (r *lru) EvictNS(ns uint64) {
-	var evicted []*lruNode
-
-	r.mu.Lock()
-	for e := r.recent.prev; e != &r.recent; {
-		rn := e
-		e = e.prev
-		if rn.n.NS() == ns {
-			rn.remove()
-			rn.n.CacheData = nil
-			r.used -= rn.n.Size()
-			evicted = append(evicted, rn)
-		}
-	}
-	r.mu.Unlock()
-
-	for _, rn := range evicted {
-		rn.h.Release()
-	}
-}
-
-func (r *lru) EvictAll() {
-	r.mu.Lock()
-	back := r.recent.prev
-	for rn := back; rn != &r.recent; rn = rn.prev {
-		rn.n.CacheData = nil
-	}
-	r.reset()
-	r.mu.Unlock()
-
-	for rn := back; rn != &r.recent; rn = rn.prev {
-		rn.h.Release()
-	}
-}
-
-func (r *lru) Close() error {
-	return nil
-}
-
-// NewLRU create a new LRU-cache.
-func NewLRU(capacity int) Cacher {
-	r := &lru{capacity: capacity}
-	r.reset()
-	return r
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru_cache.go

@@ -0,0 +1,382 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package cache
+
+import (
+	"sync"
+	"sync/atomic"
+
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+// lruCache represent a LRU cache state.
+type lruCache struct {
+	mu                sync.Mutex
+	recent            lruNode
+	table             map[uint64]*lruNs
+	capacity          int
+	used, size, alive int
+}
+
+// NewLRUCache creates a new initialized LRU cache with the given capacity.
+func NewLRUCache(capacity int) Cache {
+	c := &lruCache{
+		table:    make(map[uint64]*lruNs),
+		capacity: capacity,
+	}
+	c.recent.rNext = &c.recent
+	c.recent.rPrev = &c.recent
+	return c
+}
+
+func (c *lruCache) Capacity() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.capacity
+}
+
+func (c *lruCache) Used() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.used
+}
+
+func (c *lruCache) Size() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.size
+}
+
+func (c *lruCache) NumObjects() int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.alive
+}
+
+// SetCapacity set cache capacity.
+func (c *lruCache) SetCapacity(capacity int) {
+	c.mu.Lock()
+	c.capacity = capacity
+	c.evict()
+	c.mu.Unlock()
+}
+
+// GetNamespace return namespace object for given id.
+func (c *lruCache) GetNamespace(id uint64) Namespace {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if ns, ok := c.table[id]; ok {
+		return ns
+	}
+
+	ns := &lruNs{
+		lru:   c,
+		id:    id,
+		table: make(map[uint64]*lruNode),
+	}
+	c.table[id] = ns
+	return ns
+}
+
+func (c *lruCache) ZapNamespace(id uint64) {
+	c.mu.Lock()
+	if ns, exist := c.table[id]; exist {
+		ns.zapNB()
+		delete(c.table, id)
+	}
+	c.mu.Unlock()
+}
+
+func (c *lruCache) PurgeNamespace(id uint64, fin PurgeFin) {
+	c.mu.Lock()
+	if ns, exist := c.table[id]; exist {
+		ns.purgeNB(fin)
+	}
+	c.mu.Unlock()
+}
+
+// Purge purge entire cache.
+func (c *lruCache) Purge(fin PurgeFin) {
+	c.mu.Lock()
+	for _, ns := range c.table {
+		ns.purgeNB(fin)
+	}
+	c.mu.Unlock()
+}
+
+func (c *lruCache) Zap() {
+	c.mu.Lock()
+	for _, ns := range c.table {
+		ns.zapNB()
+	}
+	c.table = make(map[uint64]*lruNs)
+	c.mu.Unlock()
+}
+
+func (c *lruCache) evict() {
+	top := &c.recent
+	for n := c.recent.rPrev; c.used > c.capacity && n != top; {
+		n.state = nodeEvicted
+		n.rRemove()
+		n.derefNB()
+		c.used -= n.charge
+		n = c.recent.rPrev
+	}
+}
+
+type lruNs struct {
+	lru   *lruCache
+	id    uint64
+	table map[uint64]*lruNode
+	state nsState
+}
+
+func (ns *lruNs) Get(key uint64, setf SetFunc) Handle {
+	ns.lru.mu.Lock()
+
+	if ns.state != nsEffective {
+		ns.lru.mu.Unlock()
+		return nil
+	}
+
+	node, ok := ns.table[key]
+	if ok {
+		switch node.state {
+		case nodeEvicted:
+			// Insert to recent list.
+			node.state = nodeEffective
+			node.ref++
+			ns.lru.used += node.charge
+			ns.lru.evict()
+			fallthrough
+		case nodeEffective:
+			// Bump to front.
+			node.rRemove()
+			node.rInsert(&ns.lru.recent)
+		}
+		node.ref++
+	} else {
+		if setf == nil {
+			ns.lru.mu.Unlock()
+			return nil
+		}
+
+		charge, value := setf()
+		if value == nil {
+			ns.lru.mu.Unlock()
+			return nil
+		}
+
+		node = &lruNode{
+			ns:     ns,
+			key:    key,
+			value:  value,
+			charge: charge,
+			ref:    1,
+		}
+		ns.table[key] = node
+
+		ns.lru.size += charge
+		ns.lru.alive++
+		if charge > 0 {
+			node.ref++
+			node.rInsert(&ns.lru.recent)
+			ns.lru.used += charge
+			ns.lru.evict()
+		}
+	}
+
+	ns.lru.mu.Unlock()
+	return &lruHandle{node: node}
+}
+
+func (ns *lruNs) Delete(key uint64, fin DelFin) bool {
+	ns.lru.mu.Lock()
+
+	if ns.state != nsEffective {
+		if fin != nil {
+			fin(false, false)
+		}
+		ns.lru.mu.Unlock()
+		return false
+	}
+
+	node, exist := ns.table[key]
+	if !exist {
+		if fin != nil {
+			fin(false, false)
+		}
+		ns.lru.mu.Unlock()
+		return false
+	}
+
+	switch node.state {
+	case nodeDeleted:
+		if fin != nil {
+			fin(true, true)
+		}
+		ns.lru.mu.Unlock()
+		return false
+	case nodeEffective:
+		ns.lru.used -= node.charge
+		node.state = nodeDeleted
+		node.delfin = fin
+		node.rRemove()
+		node.derefNB()
+	default:
+		node.state = nodeDeleted
+		node.delfin = fin
+	}
+
+	ns.lru.mu.Unlock()
+	return true
+}
+
+func (ns *lruNs) purgeNB(fin PurgeFin) {
+	if ns.state != nsEffective {
+		return
+	}
+
+	for _, node := range ns.table {
+		switch node.state {
+		case nodeDeleted:
+		case nodeEffective:
+			ns.lru.used -= node.charge
+			node.state = nodeDeleted
+			node.purgefin = fin
+			node.rRemove()
+			node.derefNB()
+		default:
+			node.state = nodeDeleted
+			node.purgefin = fin
+		}
+	}
+}
+
+func (ns *lruNs) Purge(fin PurgeFin) {
+	ns.lru.mu.Lock()
+	ns.purgeNB(fin)
+	ns.lru.mu.Unlock()
+}
+
+func (ns *lruNs) zapNB() {
+	if ns.state != nsEffective {
+		return
+	}
+
+	ns.state = nsZapped
+
+	for _, node := range ns.table {
+		if node.state == nodeEffective {
+			ns.lru.used -= node.charge
+			node.rRemove()
+		}
+		ns.lru.size -= node.charge
+		node.state = nodeDeleted
+		node.fin()
+	}
+	ns.table = nil
+}
+
+func (ns *lruNs) Zap() {
+	ns.lru.mu.Lock()
+	ns.zapNB()
+	delete(ns.lru.table, ns.id)
+	ns.lru.mu.Unlock()
+}
+
+type lruNode struct {
+	ns *lruNs
+
+	rNext, rPrev *lruNode
+
+	key      uint64
+	value    interface{}
+	charge   int
+	ref      int
+	state    nodeState
+	delfin   DelFin
+	purgefin PurgeFin
+}
+
+func (n *lruNode) rInsert(at *lruNode) {
+	x := at.rNext
+	at.rNext = n
+	n.rPrev = at
+	n.rNext = x
+	x.rPrev = n
+}
+
+func (n *lruNode) rRemove() bool {
+	if n.rPrev == nil {
+		return false
+	}
+
+	n.rPrev.rNext = n.rNext
+	n.rNext.rPrev = n.rPrev
+	n.rPrev = nil
+	n.rNext = nil
+
+	return true
+}
+
+func (n *lruNode) fin() {
+	if r, ok := n.value.(util.Releaser); ok {
+		r.Release()
+	}
+	if n.purgefin != nil {
+		n.purgefin(n.ns.id, n.key)
+		n.delfin = nil
+		n.purgefin = nil
+	} else if n.delfin != nil {
+		n.delfin(true, false)
+		n.delfin = nil
+	}
+}
+
+func (n *lruNode) derefNB() {
+	n.ref--
+	if n.ref == 0 {
+		if n.ns.state == nsEffective {
+			// Remove elemement.
+			delete(n.ns.table, n.key)
+			n.ns.lru.size -= n.charge
+			n.ns.lru.alive--
+			n.fin()
+		}
+		n.value = nil
+	} else if n.ref < 0 {
+		panic("leveldb/cache: lruCache: negative node reference")
+	}
+}
+
+func (n *lruNode) deref() {
+	n.ns.lru.mu.Lock()
+	n.derefNB()
+	n.ns.lru.mu.Unlock()
+}
+
+type lruHandle struct {
+	node *lruNode
+	once uint32
+}
+
+func (h *lruHandle) Value() interface{} {
+	if atomic.LoadUint32(&h.once) == 0 {
+		return h.node.value
+	}
+	return nil
+}
+
+func (h *lruHandle) Release() {
+	if !atomic.CompareAndSwapUint32(&h.once, 0, 1) {
+		return
+	}
+	h.node.deref()
+	h.node = nil
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/config.go

@@ -0,0 +1,40 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+const (
+	kNumLevels = 7
+
+	// Level-0 compaction is started when we hit this many files.
+	kL0_CompactionTrigger float64 = 4
+
+	// Soft limit on number of level-0 files.  We slow down writes at this point.
+	kL0_SlowdownWritesTrigger = 8
+
+	// Maximum number of level-0 files.  We stop writes at this point.
+	kL0_StopWritesTrigger = 12
+
+	// Maximum level to which a new compacted memdb is pushed if it
+	// does not create overlap.  We try to push to level 2 to avoid the
+	// relatively expensive level 0=>1 compactions and to avoid some
+	// expensive manifest file operations.  We do not push all the way to
+	// the largest level since that can generate a lot of wasted disk
+	// space if the same key space is being repeatedly overwritten.
+	kMaxMemCompactLevel = 2
+
+	// Maximum size of a table.
+	kMaxTableSize = 2 * 1048576
+
+	// Maximum bytes of overlaps in grandparent (i.e., level+2) before we
+	// stop building a single file in a level->level+1 compaction.
+	kMaxGrandParentOverlapBytes = 10 * kMaxTableSize
+
+	// Maximum number of bytes in all compacted files.  We avoid expanding
+	// the lower level file set of a compaction if it would make the
+	// total compaction cover more than this many bytes.
+	kExpCompactionMaxBytes = 25 * kMaxTableSize
+)

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/corrupt_test.go

@@ -9,12 +9,13 @@ package leveldb
 import (
 	"bytes"
 	"fmt"
-	"github.com/syndtr/goleveldb/leveldb/filter"
-	"github.com/syndtr/goleveldb/leveldb/opt"
-	"github.com/syndtr/goleveldb/leveldb/storage"
 	"io"
 	"math/rand"
 	"testing"
+
+	"github.com/syndtr/goleveldb/leveldb/cache"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/storage"
 )
 
 const ctValSize = 1000
@@ -31,8 +32,8 @@ func newDbCorruptHarnessWopt(t *testing.T, o *opt.Options) *dbCorruptHarness {
 
 func newDbCorruptHarness(t *testing.T) *dbCorruptHarness {
 	return newDbCorruptHarnessWopt(t, &opt.Options{
-		BlockCacheCapacity: 100,
-		Strict:             opt.StrictJournalChecksum,
+		BlockCache: cache.NewLRUCache(100),
+		Strict:     opt.StrictJournalChecksum,
 	})
 }
 
@@ -95,22 +96,21 @@ func (h *dbCorruptHarness) deleteRand(n, max int, rnd *rand.Rand) {
 	}
 }
 
-func (h *dbCorruptHarness) corrupt(ft storage.FileType, fi, offset, n int) {
+func (h *dbCorruptHarness) corrupt(ft storage.FileType, offset, n int) {
 	p := &h.dbHarness
 	t := p.t
 
+	var file storage.File
 	ff, _ := p.stor.GetFiles(ft)
-	sff := files(ff)
-	sff.sort()
-	if fi < 0 {
-		fi = len(sff) - 1
+	for _, f := range ff {
+		if file == nil || f.Num() > file.Num() {
+			file = f
+		}
 	}
-	if fi >= len(sff) {
-		t.Fatalf("no such file with type %q with index %d", ft, fi)
+	if file == nil {
+		t.Fatalf("no such file with type %q", ft)
 	}
 
-	file := sff[fi]
-
 	r, err := file.Open()
 	if err != nil {
 		t.Fatal("cannot open file: ", err)
@@ -225,8 +225,8 @@ func TestCorruptDB_Journal(t *testing.T) {
 	h.build(100)
 	h.check(100, 100)
 	h.closeDB()
-	h.corrupt(storage.TypeJournal, -1, 19, 1)
-	h.corrupt(storage.TypeJournal, -1, 32*1024+1000, 1)
+	h.corrupt(storage.TypeJournal, 19, 1)
+	h.corrupt(storage.TypeJournal, 32*1024+1000, 1)
 
 	h.openDB()
 	h.check(36, 36)
@@ -242,7 +242,7 @@ func TestCorruptDB_Table(t *testing.T) {
 	h.compactRangeAt(0, "", "")
 	h.compactRangeAt(1, "", "")
 	h.closeDB()
-	h.corrupt(storage.TypeTable, -1, 100, 1)
+	h.corrupt(storage.TypeTable, 100, 1)
 
 	h.openDB()
 	h.check(99, 99)
@@ -256,7 +256,7 @@ func TestCorruptDB_TableIndex(t *testing.T) {
 	h.build(10000)
 	h.compactMem()
 	h.closeDB()
-	h.corrupt(storage.TypeTable, -1, -2000, 500)
+	h.corrupt(storage.TypeTable, -2000, 500)
 
 	h.openDB()
 	h.check(5000, 9999)
@@ -267,9 +267,9 @@ func TestCorruptDB_TableIndex(t *testing.T) {
 func TestCorruptDB_MissingManifest(t *testing.T) {
 	rnd := rand.New(rand.NewSource(0x0badda7a))
 	h := newDbCorruptHarnessWopt(t, &opt.Options{
-		BlockCacheCapacity: 100,
-		Strict:             opt.StrictJournalChecksum,
-		WriteBuffer:        1000 * 60,
+		BlockCache:  cache.NewLRUCache(100),
+		Strict:      opt.StrictJournalChecksum,
+		WriteBuffer: 1000 * 60,
 	})
 
 	h.build(1000)
@@ -355,7 +355,7 @@ func TestCorruptDB_CorruptedManifest(t *testing.T) {
 	h.compactMem()
 	h.compactRange("", "")
 	h.closeDB()
-	h.corrupt(storage.TypeManifest, -1, 0, 1000)
+	h.corrupt(storage.TypeManifest, 0, 1000)
 	h.openAssert(false)
 
 	h.recover()
@@ -370,7 +370,7 @@ func TestCorruptDB_CompactionInputError(t *testing.T) {
 	h.build(10)
 	h.compactMem()
 	h.closeDB()
-	h.corrupt(storage.TypeTable, -1, 100, 1)
+	h.corrupt(storage.TypeTable, 100, 1)
 
 	h.openDB()
 	h.check(9, 9)
@@ -387,7 +387,7 @@ func TestCorruptDB_UnrelatedKeys(t *testing.T) {
 	h.build(10)
 	h.compactMem()
 	h.closeDB()
-	h.corrupt(storage.TypeTable, -1, 100, 1)
+	h.corrupt(storage.TypeTable, 100, 1)
 
 	h.openDB()
 	h.put(string(tkey(1000)), string(tval(1000, ctValSize)))
@@ -470,31 +470,3 @@ func TestCorruptDB_MissingTableFiles(t *testing.T) {
 
 	h.close()
 }
-
-func TestCorruptDB_RecoverTable(t *testing.T) {
-	h := newDbCorruptHarnessWopt(t, &opt.Options{
-		WriteBuffer:         112 * opt.KiB,
-		CompactionTableSize: 90 * opt.KiB,
-		Filter:              filter.NewBloomFilter(10),
-	})
-
-	h.build(1000)
-	h.compactMem()
-	h.compactRangeAt(0, "", "")
-	h.compactRangeAt(1, "", "")
-	seq := h.db.seq
-	h.closeDB()
-	h.corrupt(storage.TypeTable, 0, 1000, 1)
-	h.corrupt(storage.TypeTable, 3, 10000, 1)
-	// Corrupted filter shouldn't affect recovery.
-	h.corrupt(storage.TypeTable, 3, 113888, 10)
-	h.corrupt(storage.TypeTable, -1, 20000, 1)
-
-	h.recover()
-	if h.db.seq != seq {
-		t.Errorf("invalid seq, want=%d got=%d", seq, h.db.seq)
-	}
-	h.check(985, 985)
-
-	h.close()
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go

@@ -7,7 +7,7 @@
 package leveldb
 
 import (
-	"container/list"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -17,7 +17,6 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/journal"
 	"github.com/syndtr/goleveldb/leveldb/memdb"
@@ -47,7 +46,7 @@ type DB struct {
 
 	// Snapshot.
 	snapsMu   sync.Mutex
-	snapsList *list.List
+	snapsRoot snapshotElement
 
 	// Stats.
 	aliveSnaps, aliveIters int32
@@ -57,20 +56,18 @@ type DB struct {
 	writeMergedC chan bool
 	writeLockC   chan struct{}
 	writeAckC    chan error
-	writeDelay   time.Duration
-	writeDelayN  int
 	journalC     chan *Batch
 	journalAckC  chan error
 
 	// Compaction.
-	tcompCmdC        chan cCmd
-	tcompPauseC      chan chan<- struct{}
-	mcompCmdC        chan cCmd
-	compErrC         chan error
-	compPerErrC      chan error
-	compErrSetC      chan error
-	compWriteLocking bool
-	compStats        []cStats
+	tcompCmdC     chan cCmd
+	tcompPauseC   chan chan<- struct{}
+	tcompTriggerC chan struct{}
+	mcompCmdC     chan cCmd
+	mcompTriggerC chan struct{}
+	compErrC      chan error
+	compErrSetC   chan error
+	compStats     [kNumLevels]cStats
 
 	// Close.
 	closeW sync.WaitGroup
@@ -85,11 +82,9 @@ func openDB(s *session) (*DB, error) {
 	db := &DB{
 		s: s,
 		// Initial sequence
-		seq: s.stSeqNum,
+		seq: s.stSeq,
 		// MemDB
 		memPool: make(chan *memdb.DB, 1),
-		// Snapshot
-		snapsList: list.New(),
 		// Write
 		writeC:       make(chan *Batch),
 		writeMergedC: make(chan bool),
@@ -98,55 +93,40 @@ func openDB(s *session) (*DB, error) {
 		journalC:     make(chan *Batch),
 		journalAckC:  make(chan error),
 		// Compaction
-		tcompCmdC:   make(chan cCmd),
-		tcompPauseC: make(chan chan<- struct{}),
-		mcompCmdC:   make(chan cCmd),
-		compErrC:    make(chan error),
-		compPerErrC: make(chan error),
-		compErrSetC: make(chan error),
-		compStats:   make([]cStats, s.o.GetNumLevel()),
+		tcompCmdC:     make(chan cCmd),
+		tcompPauseC:   make(chan chan<- struct{}),
+		tcompTriggerC: make(chan struct{}, 1),
+		mcompCmdC:     make(chan cCmd),
+		mcompTriggerC: make(chan struct{}, 1),
+		compErrC:      make(chan error),
+		compErrSetC:   make(chan error),
 		// Close
 		closeC: make(chan struct{}),
 	}
+	db.initSnapshot()
 
-	// Read-only mode.
-	readOnly := s.o.GetReadOnly()
-
-	if readOnly {
-		// Recover journals (read-only mode).
-		if err := db.recoverJournalRO(); err != nil {
-			return nil, err
-		}
-	} else {
-		// Recover journals.
-		if err := db.recoverJournal(); err != nil {
-			return nil, err
-		}
-
-		// Remove any obsolete files.
-		if err := db.checkAndCleanFiles(); err != nil {
-			// Close journal.
-			if db.journal != nil {
-				db.journal.Close()
-				db.journalWriter.Close()
-			}
-			return nil, err
-		}
-
+	if err := db.recoverJournal(); err != nil {
+		return nil, err
 	}
 
-	// Doesn't need to be included in the wait group.
+	// Remove any obsolete files.
+	if err := db.checkAndCleanFiles(); err != nil {
+		// Close journal.
+		if db.journal != nil {
+			db.journal.Close()
+			db.journalWriter.Close()
+		}
+		return nil, err
+	}
+
+	// Don't include compaction error goroutine into wait group.
 	go db.compactionError()
-	go db.mpoolDrain()
 
-	if readOnly {
-		db.SetReadOnly()
-	} else {
-		db.closeW.Add(3)
-		go db.tCompaction()
-		go db.mCompaction()
-		go db.jWriter()
-	}
+	db.closeW.Add(3)
+	go db.tCompaction()
+	go db.mCompaction()
+	go db.jWriter()
+	go db.mpoolDrain()
 
 	s.logf("db@open done T·%v", time.Since(start))
 
@@ -273,10 +253,6 @@ func RecoverFile(path string, o *opt.Options) (db *DB, err error) {
 }
 
 func recoverTable(s *session, o *opt.Options) error {
-	o = dupOptions(o)
-	// Mask StrictReader, lets StrictRecovery doing its job.
-	o.Strict &= ^opt.StrictReader
-
 	// Get all tables and sort it by file number.
 	tableFiles_, err := s.getFiles(storage.TypeTable)
 	if err != nil {
@@ -285,17 +261,10 @@ func recoverTable(s *session, o *opt.Options) error {
 	tableFiles := files(tableFiles_)
 	tableFiles.sort()
 
-	var (
-		maxSeq                                                            uint64
-		recoveredKey, goodKey, corruptedKey, corruptedBlock, droppedTable int
-
-		// We will drop corrupted table.
-		strict = o.GetStrict(opt.StrictRecovery)
-		noSync = o.GetNoSync()
-
-		rec   = &sessionRecord{}
-		bpool = util.NewBufferPool(o.GetBlockSize() + 5)
-	)
+	var mSeq uint64
+	var good, corrupted int
+	rec := new(sessionRecord)
+	bpool := util.NewBufferPool(o.GetBlockSize() + 5)
 	buildTable := func(iter iterator.Iterator) (tmp storage.File, size int64, err error) {
 		tmp = s.newTemp()
 		writer, err := tmp.Create()
@@ -329,11 +298,9 @@ func recoverTable(s *session, o *opt.Options) error {
 		if err != nil {
 			return
 		}
-		if !noSync {
-			err = writer.Sync()
-			if err != nil {
-				return
-			}
+		err = writer.Sync()
+		if err != nil {
+			return
 		}
 		size = int64(tw.BytesLen())
 		return
@@ -344,12 +311,7 @@ func recoverTable(s *session, o *opt.Options) error {
 		if err != nil {
 			return err
 		}
-		var closed bool
-		defer func() {
-			if !closed {
-				reader.Close()
-			}
-		}()
+		defer reader.Close()
 
 		// Get file size.
 		size, err := reader.Seek(0, 2)
@@ -357,34 +319,25 @@ func recoverTable(s *session, o *opt.Options) error {
 			return err
 		}
 
-		var (
-			tSeq                                     uint64
-			tgoodKey, tcorruptedKey, tcorruptedBlock int
-			imin, imax                               []byte
-		)
-		tr, err := table.NewReader(reader, size, storage.NewFileInfo(file), nil, bpool, o)
-		if err != nil {
-			return err
-		}
+		var tSeq uint64
+		var tgood, tcorrupted, blockerr int
+		var imin, imax []byte
+		tr := table.NewReader(reader, size, nil, bpool, o)
 		iter := tr.NewIterator(nil, nil)
-		if itererr, ok := iter.(iterator.ErrorCallbackSetter); ok {
-			itererr.SetErrorCallback(func(err error) {
-				if errors.IsCorrupted(err) {
-					s.logf("table@recovery block corruption @%d %q", file.Num(), err)
-					tcorruptedBlock++
-				}
-			})
-		}
+		iter.(iterator.ErrorCallbackSetter).SetErrorCallback(func(err error) {
+			s.logf("table@recovery found error @%d %q", file.Num(), err)
+			blockerr++
+		})
 
 		// Scan the table.
 		for iter.Next() {
 			key := iter.Key()
-			_, seq, _, kerr := parseIkey(key)
-			if kerr != nil {
-				tcorruptedKey++
+			_, seq, _, ok := parseIkey(key)
+			if !ok {
+				tcorrupted++
 				continue
 			}
-			tgoodKey++
+			tgood++
 			if seq > tSeq {
 				tSeq = seq
 			}
@@ -399,18 +352,8 @@ func recoverTable(s *session, o *opt.Options) error {
 		}
 		iter.Release()
 
-		goodKey += tgoodKey
-		corruptedKey += tcorruptedKey
-		corruptedBlock += tcorruptedBlock
-
-		if strict && (tcorruptedKey > 0 || tcorruptedBlock > 0) {
-			droppedTable++
-			s.logf("table@recovery dropped @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", file.Num(), tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq)
-			return nil
-		}
-
-		if tgoodKey > 0 {
-			if tcorruptedKey > 0 || tcorruptedBlock > 0 {
+		if tgood > 0 {
+			if tcorrupted > 0 || blockerr > 0 {
 				// Rebuild the table.
 				s.logf("table@recovery rebuilding @%d", file.Num())
 				iter := tr.NewIterator(nil, nil)
@@ -419,25 +362,25 @@ func recoverTable(s *session, o *opt.Options) error {
 				if err != nil {
 					return err
 				}
-				closed = true
 				reader.Close()
 				if err := file.Replace(tmp); err != nil {
 					return err
 				}
 				size = newSize
 			}
-			if tSeq > maxSeq {
-				maxSeq = tSeq
+			if tSeq > mSeq {
+				mSeq = tSeq
 			}
-			recoveredKey += tgoodKey
 			// Add table to level 0.
 			rec.addTable(0, file.Num(), uint64(size), imin, imax)
-			s.logf("table@recovery recovered @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", file.Num(), tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq)
+			s.logf("table@recovery recovered @%d N·%d C·%d B·%d S·%d Q·%d", file.Num(), tgood, tcorrupted, blockerr, size, tSeq)
 		} else {
-			droppedTable++
-			s.logf("table@recovery unrecoverable @%d Ck·%d Cb·%d S·%d", file.Num(), tcorruptedKey, tcorruptedBlock, size)
+			s.logf("table@recovery unrecoverable @%d C·%d B·%d S·%d", file.Num(), tcorrupted, blockerr, size)
 		}
 
+		good += tgood
+		corrupted += tcorrupted
+
 		return nil
 	}
 
@@ -454,11 +397,11 @@ func recoverTable(s *session, o *opt.Options) error {
 			}
 		}
 
-		s.logf("table@recovery recovered F·%d N·%d Gk·%d Ck·%d Q·%d", len(tableFiles), recoveredKey, goodKey, corruptedKey, maxSeq)
+		s.logf("table@recovery recovered F·%d N·%d C·%d Q·%d", len(tableFiles), good, corrupted, mSeq)
 	}
 
 	// Set sequence number.
-	rec.setSeqNum(maxSeq)
+	rec.setSeq(mSeq + 1)
 
 	// Create new manifest.
 	if err := s.create(); err != nil {
@@ -470,136 +413,128 @@ func recoverTable(s *session, o *opt.Options) error {
 }
 
 func (db *DB) recoverJournal() error {
-	// Get all journals and sort it by file number.
-	allJournalFiles, err := db.s.getFiles(storage.TypeJournal)
+	// Get all tables and sort it by file number.
+	journalFiles_, err := db.s.getFiles(storage.TypeJournal)
 	if err != nil {
 		return err
 	}
-	files(allJournalFiles).sort()
+	journalFiles := files(journalFiles_)
+	journalFiles.sort()
 
-	// Journals that will be recovered.
-	var recJournalFiles []storage.File
-	for _, jf := range allJournalFiles {
-		if jf.Num() >= db.s.stJournalNum || jf.Num() == db.s.stPrevJournalNum {
-			recJournalFiles = append(recJournalFiles, jf)
+	// Discard older journal.
+	prev := -1
+	for i, file := range journalFiles {
+		if file.Num() >= db.s.stJournalNum {
+			if prev >= 0 {
+				i--
+				journalFiles[i] = journalFiles[prev]
+			}
+			journalFiles = journalFiles[i:]
+			break
+		} else if file.Num() == db.s.stPrevJournalNum {
+			prev = i
 		}
 	}
 
-	var (
-		of  storage.File // Obsolete file.
-		rec = &sessionRecord{}
-	)
+	var jr *journal.Reader
+	var of storage.File
+	var mem *memdb.DB
+	batch := new(Batch)
+	cm := newCMem(db.s)
+	buf := new(util.Buffer)
+	// Options.
+	strict := db.s.o.GetStrict(opt.StrictJournal)
+	checksum := db.s.o.GetStrict(opt.StrictJournalChecksum)
+	writeBuffer := db.s.o.GetWriteBuffer()
+	recoverJournal := func(file storage.File) error {
+		db.logf("journal@recovery recovering @%d", file.Num())
+		reader, err := file.Open()
+		if err != nil {
+			return err
+		}
+		defer reader.Close()
 
-	// Recover journals.
-	if len(recJournalFiles) > 0 {
-		db.logf("journal@recovery F·%d", len(recJournalFiles))
+		// Create/reset journal reader instance.
+		if jr == nil {
+			jr = journal.NewReader(reader, dropper{db.s, file}, strict, checksum)
+		} else {
+			jr.Reset(reader, dropper{db.s, file}, strict, checksum)
+		}
 
-		// Mark file number as used.
-		db.s.markFileNum(recJournalFiles[len(recJournalFiles)-1].Num())
+		// Flush memdb and remove obsolete journal file.
+		if of != nil {
+			if mem.Len() > 0 {
+				if err := cm.flush(mem, 0); err != nil {
+					return err
+				}
+			}
+			if err := cm.commit(file.Num(), db.seq); err != nil {
+				return err
+			}
+			cm.reset()
+			of.Remove()
+			of = nil
+		}
 
-		var (
-			// Options.
-			strict      = db.s.o.GetStrict(opt.StrictJournal)
-			checksum    = db.s.o.GetStrict(opt.StrictJournalChecksum)
-			writeBuffer = db.s.o.GetWriteBuffer()
-
-			jr    *journal.Reader
-			mdb   = memdb.New(db.s.icmp, writeBuffer)
-			buf   = &util.Buffer{}
-			batch = &Batch{}
-		)
-
-		for _, jf := range recJournalFiles {
-			db.logf("journal@recovery recovering @%d", jf.Num())
-
-			fr, err := jf.Open()
+		// Replay journal to memdb.
+		mem.Reset()
+		for {
+			r, err := jr.Next()
 			if err != nil {
+				if err == io.EOF {
+					break
+				}
 				return err
 			}
 
-			// Create or reset journal reader instance.
-			if jr == nil {
-				jr = journal.NewReader(fr, dropper{db.s, jf}, strict, checksum)
-			} else {
-				jr.Reset(fr, dropper{db.s, jf}, strict, checksum)
-			}
-
-			// Flush memdb and remove obsolete journal file.
-			if of != nil {
-				if mdb.Len() > 0 {
-					if _, err := db.s.flushMemdb(rec, mdb, -1); err != nil {
-						fr.Close()
-						return err
-					}
-				}
-
-				rec.setJournalNum(jf.Num())
-				rec.setSeqNum(db.seq)
-				if err := db.s.commit(rec); err != nil {
-					fr.Close()
+			buf.Reset()
+			if _, err := buf.ReadFrom(r); err != nil {
+				if err == io.ErrUnexpectedEOF {
+					continue
+				} else {
 					return err
 				}
-				rec.resetAddedTables()
-
-				of.Remove()
-				of = nil
+			}
+			if err := batch.decode(buf.Bytes()); err != nil {
+				return err
+			}
+			if err := batch.memReplay(mem); err != nil {
+				return err
 			}
 
-			// Replay journal to memdb.
-			mdb.Reset()
-			for {
-				r, err := jr.Next()
-				if err != nil {
-					if err == io.EOF {
-						break
-					}
+			// Save sequence number.
+			db.seq = batch.seq + uint64(batch.len())
 
-					fr.Close()
-					return errors.SetFile(err, jf)
-				}
-
-				buf.Reset()
-				if _, err := buf.ReadFrom(r); err != nil {
-					if err == io.ErrUnexpectedEOF {
-						// This is error returned due to corruption, with strict == false.
-						continue
-					}
-
-					fr.Close()
-					return errors.SetFile(err, jf)
-				}
-				if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mdb); err != nil {
-					if !strict && errors.IsCorrupted(err) {
-						db.s.logf("journal error: %v (skipped)", err)
-						// We won't apply sequence number as it might be corrupted.
-						continue
-					}
-
-					fr.Close()
-					return errors.SetFile(err, jf)
-				}
-
-				// Save sequence number.
-				db.seq = batch.seq + uint64(batch.Len())
-
-				// Flush it if large enough.
-				if mdb.Size() >= writeBuffer {
-					if _, err := db.s.flushMemdb(rec, mdb, 0); err != nil {
-						fr.Close()
-						return err
-					}
-
-					mdb.Reset()
+			// Flush it if large enough.
+			if mem.Size() >= writeBuffer {
+				if err := cm.flush(mem, 0); err != nil {
+					return err
 				}
+				mem.Reset()
 			}
-
-			fr.Close()
-			of = jf
 		}
 
-		// Flush the last memdb.
-		if mdb.Len() > 0 {
-			if _, err := db.s.flushMemdb(rec, mdb, 0); err != nil {
+		of = file
+		return nil
+	}
+
+	// Recover all journals.
+	if len(journalFiles) > 0 {
+		db.logf("journal@recovery F·%d", len(journalFiles))
+
+		// Mark file number as used.
+		db.s.markFileNum(journalFiles[len(journalFiles)-1].Num())
+
+		mem = memdb.New(db.s.icmp, writeBuffer)
+		for _, file := range journalFiles {
+			if err := recoverJournal(file); err != nil {
+				return err
+			}
+		}
+
+		// Flush the last journal.
+		if mem.Len() > 0 {
+			if err := cm.flush(mem, 0); err != nil {
 				return err
 			}
 		}
@@ -611,10 +546,8 @@ func (db *DB) recoverJournal() error {
 	}
 
 	// Commit.
-	rec.setJournalNum(db.journalFile.Num())
-	rec.setSeqNum(db.seq)
-	if err := db.s.commit(rec); err != nil {
-		// Close journal on error.
+	if err := cm.commit(db.journalFile.Num(), db.seq); err != nil {
+		// Close journal.
 		if db.journal != nil {
 			db.journal.Close()
 			db.journalWriter.Close()
@@ -630,105 +563,8 @@ func (db *DB) recoverJournal() error {
 	return nil
 }
 
-func (db *DB) recoverJournalRO() error {
-	// Get all journals and sort it by file number.
-	allJournalFiles, err := db.s.getFiles(storage.TypeJournal)
-	if err != nil {
-		return err
-	}
-	files(allJournalFiles).sort()
-
-	// Journals that will be recovered.
-	var recJournalFiles []storage.File
-	for _, jf := range allJournalFiles {
-		if jf.Num() >= db.s.stJournalNum || jf.Num() == db.s.stPrevJournalNum {
-			recJournalFiles = append(recJournalFiles, jf)
-		}
-	}
-
-	var (
-		// Options.
-		strict      = db.s.o.GetStrict(opt.StrictJournal)
-		checksum    = db.s.o.GetStrict(opt.StrictJournalChecksum)
-		writeBuffer = db.s.o.GetWriteBuffer()
-
-		mdb = memdb.New(db.s.icmp, writeBuffer)
-	)
-
-	// Recover journals.
-	if len(recJournalFiles) > 0 {
-		db.logf("journal@recovery RO·Mode F·%d", len(recJournalFiles))
-
-		var (
-			jr    *journal.Reader
-			buf   = &util.Buffer{}
-			batch = &Batch{}
-		)
-
-		for _, jf := range recJournalFiles {
-			db.logf("journal@recovery recovering @%d", jf.Num())
-
-			fr, err := jf.Open()
-			if err != nil {
-				return err
-			}
-
-			// Create or reset journal reader instance.
-			if jr == nil {
-				jr = journal.NewReader(fr, dropper{db.s, jf}, strict, checksum)
-			} else {
-				jr.Reset(fr, dropper{db.s, jf}, strict, checksum)
-			}
-
-			// Replay journal to memdb.
-			for {
-				r, err := jr.Next()
-				if err != nil {
-					if err == io.EOF {
-						break
-					}
-
-					fr.Close()
-					return errors.SetFile(err, jf)
-				}
-
-				buf.Reset()
-				if _, err := buf.ReadFrom(r); err != nil {
-					if err == io.ErrUnexpectedEOF {
-						// This is error returned due to corruption, with strict == false.
-						continue
-					}
-
-					fr.Close()
-					return errors.SetFile(err, jf)
-				}
-				if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mdb); err != nil {
-					if !strict && errors.IsCorrupted(err) {
-						db.s.logf("journal error: %v (skipped)", err)
-						// We won't apply sequence number as it might be corrupted.
-						continue
-					}
-
-					fr.Close()
-					return errors.SetFile(err, jf)
-				}
-
-				// Save sequence number.
-				db.seq = batch.seq + uint64(batch.Len())
-			}
-
-			fr.Close()
-		}
-	}
-
-	// Set memDB.
-	db.mem = &memDB{db: db, DB: mdb, ref: 1}
-
-	return nil
-}
-
 func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, err error) {
-	ikey := newIkey(key, seq, ktSeek)
+	ikey := newIKey(key, seq, tSeek)
 
 	em, fm := db.getMems()
 	for _, m := range [...]*memDB{em, fm} {
@@ -737,15 +573,11 @@ func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, er
 		}
 		defer m.decref()
 
-		mk, mv, me := m.Find(ikey)
+		mk, mv, me := m.mdb.Find(ikey)
 		if me == nil {
-			ukey, _, kt, kerr := parseIkey(mk)
-			if kerr != nil {
-				// Shouldn't have had happen.
-				panic(kerr)
-			}
-			if db.s.icmp.uCompare(ukey, key) == 0 {
-				if kt == ktDel {
+			ukey, _, t, ok := parseIkey(mk)
+			if ok && db.s.icmp.uCompare(ukey, key) == 0 {
+				if t == tDel {
 					return nil, ErrNotFound
 				}
 				return append([]byte{}, mv...), nil
@@ -756,60 +588,17 @@ func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, er
 	}
 
 	v := db.s.version()
-	value, cSched, err := v.get(ikey, ro, false)
+	value, cSched, err := v.get(ikey, ro)
 	v.release()
 	if cSched {
 		// Trigger table compaction.
-		db.compSendTrigger(db.tcompCmdC)
-	}
-	return
-}
-
-func (db *DB) has(key []byte, seq uint64, ro *opt.ReadOptions) (ret bool, err error) {
-	ikey := newIkey(key, seq, ktSeek)
-
-	em, fm := db.getMems()
-	for _, m := range [...]*memDB{em, fm} {
-		if m == nil {
-			continue
-		}
-		defer m.decref()
-
-		mk, _, me := m.Find(ikey)
-		if me == nil {
-			ukey, _, kt, kerr := parseIkey(mk)
-			if kerr != nil {
-				// Shouldn't have had happen.
-				panic(kerr)
-			}
-			if db.s.icmp.uCompare(ukey, key) == 0 {
-				if kt == ktDel {
-					return false, nil
-				}
-				return true, nil
-			}
-		} else if me != ErrNotFound {
-			return false, me
-		}
-	}
-
-	v := db.s.version()
-	_, cSched, err := v.get(ikey, ro, true)
-	v.release()
-	if cSched {
-		// Trigger table compaction.
-		db.compSendTrigger(db.tcompCmdC)
-	}
-	if err == nil {
-		ret = true
-	} else if err == ErrNotFound {
-		err = nil
+		db.compTrigger(db.tcompTriggerC)
 	}
 	return
 }
 
 // Get gets the value for the given key. It returns ErrNotFound if the
-// DB does not contains the key.
+// DB does not contain the key.
 //
 // The returned slice is its own copy, it is safe to modify the contents
 // of the returned slice.
@@ -820,23 +609,7 @@ func (db *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
 		return
 	}
 
-	se := db.acquireSnapshot()
-	defer db.releaseSnapshot(se)
-	return db.get(key, se.seq, ro)
-}
-
-// Has returns true if the DB does contains the given key.
-//
-// It is safe to modify the contents of the argument after Get returns.
-func (db *DB) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) {
-	err = db.ok()
-	if err != nil {
-		return
-	}
-
-	se := db.acquireSnapshot()
-	defer db.releaseSnapshot(se)
-	return db.has(key, se.seq, ro)
+	return db.get(key, db.getSeq(), ro)
 }
 
 // NewIterator returns an iterator for the latest snapshot of the
@@ -860,11 +633,9 @@ func (db *DB) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Itera
 		return iterator.NewEmptyIterator(err)
 	}
 
-	se := db.acquireSnapshot()
-	defer db.releaseSnapshot(se)
-	// Iterator holds 'version' lock, 'version' is immutable so snapshot
-	// can be released after iterator created.
-	return db.newIterator(se.seq, slice, ro)
+	snap := db.newSnapshot()
+	defer snap.Release()
+	return snap.NewIterator(slice, ro)
 }
 
 // GetSnapshot returns a latest snapshot of the underlying DB. A snapshot
@@ -884,7 +655,7 @@ func (db *DB) GetSnapshot() (*Snapshot, error) {
 //
 // Property names:
 //	leveldb.num-files-at-level{n}
-//		Returns the number of files at level 'n'.
+//		Returns the number of filer at level 'n'.
 //	leveldb.stats
 //		Returns statistics of the underlying DB.
 //	leveldb.sstables
@@ -907,21 +678,20 @@ func (db *DB) GetProperty(name string) (value string, err error) {
 
 	const prefix = "leveldb."
 	if !strings.HasPrefix(name, prefix) {
-		return "", ErrNotFound
+		return "", errors.New("leveldb: GetProperty: unknown property: " + name)
 	}
 	p := name[len(prefix):]
 
 	v := db.s.version()
 	defer v.release()
 
-	numFilesPrefix := "num-files-at-level"
 	switch {
-	case strings.HasPrefix(p, numFilesPrefix):
+	case strings.HasPrefix(p, "num-files-at-level"):
 		var level uint
 		var rest string
-		n, _ := fmt.Sscanf(p[len(numFilesPrefix):], "%d%s", &level, &rest)
-		if n != 1 || int(level) >= db.s.o.GetNumLevel() {
-			err = ErrNotFound
+		n, _ := fmt.Scanf("%d%s", &level, &rest)
+		if n != 1 || level >= kNumLevels {
+			err = errors.New("leveldb: GetProperty: invalid property: " + name)
 		} else {
 			value = fmt.Sprint(v.tLen(int(level)))
 		}
@@ -948,8 +718,8 @@ func (db *DB) GetProperty(name string) (value string, err error) {
 	case p == "blockpool":
 		value = fmt.Sprintf("%v", db.s.tops.bpool)
 	case p == "cachedblock":
-		if db.s.tops.bcache != nil {
-			value = fmt.Sprintf("%d", db.s.tops.bcache.Size())
+		if bc := db.s.o.GetBlockCache(); bc != nil {
+			value = fmt.Sprintf("%d", bc.Size())
 		} else {
 			value = "<nil>"
 		}
@@ -960,7 +730,7 @@ func (db *DB) GetProperty(name string) (value string, err error) {
 	case p == "aliveiters":
 		value = fmt.Sprintf("%d", atomic.LoadInt32(&db.aliveIters))
 	default:
-		err = ErrNotFound
+		err = errors.New("leveldb: GetProperty: unknown property: " + name)
 	}
 
 	return
@@ -982,8 +752,8 @@ func (db *DB) SizeOf(ranges []util.Range) (Sizes, error) {
 
 	sizes := make(Sizes, 0, len(ranges))
 	for _, r := range ranges {
-		imin := newIkey(r.Start, kMaxSeq, ktSeek)
-		imax := newIkey(r.Limit, kMaxSeq, ktSeek)
+		imin := newIKey(r.Start, kMaxSeq, tSeek)
+		imax := newIKey(r.Limit, kMaxSeq, tSeek)
 		start, err := v.offsetOf(imin)
 		if err != nil {
 			return nil, err
@@ -1023,29 +793,21 @@ func (db *DB) Close() error {
 	var err error
 	select {
 	case err = <-db.compErrC:
-		if err == ErrReadOnly {
-			err = nil
-		}
 	default:
 	}
 
-	// Signal all goroutines.
 	close(db.closeC)
 
-	// Wait for all gorotines to exit.
+	// Wait for the close WaitGroup.
 	db.closeW.Wait()
 
-	// Lock writer and closes journal.
+	// Close journal.
 	db.writeLockC <- struct{}{}
 	if db.journal != nil {
 		db.journal.Close()
 		db.journalWriter.Close()
 	}
 
-	if db.writeDelayN > 0 {
-		db.logf("db@write was delayed N·%d T·%v", db.writeDelayN, db.writeDelay)
-	}
-
 	// Close session.
 	db.s.close()
 	db.logf("db@close done T·%v", time.Since(start))
@@ -1065,6 +827,7 @@ func (db *DB) Close() error {
 	db.journalWriter = nil
 	db.journalFile = nil
 	db.frozenJournalFile = nil
+	db.snapsRoot = snapshotElement{}
 	db.closer = nil
 
 	return err

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go

@@ -7,11 +7,11 @@
 package leveldb
 
 import (
+	"errors"
 	"sync"
 	"time"
 
-	"github.com/syndtr/goleveldb/leveldb/errors"
-	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/memdb"
 )
 
 var (
@@ -61,18 +61,58 @@ func (p *cStatsStaging) stopTimer() {
 	}
 }
 
+type cMem struct {
+	s     *session
+	level int
+	rec   *sessionRecord
+}
+
+func newCMem(s *session) *cMem {
+	return &cMem{s: s, rec: new(sessionRecord)}
+}
+
+func (c *cMem) flush(mem *memdb.DB, level int) error {
+	s := c.s
+
+	// Write memdb to table.
+	iter := mem.NewIterator(nil)
+	defer iter.Release()
+	t, n, err := s.tops.createFrom(iter)
+	if err != nil {
+		return err
+	}
+
+	// Pick level.
+	if level < 0 {
+		level = s.version_NB().pickLevel(t.imin.ukey(), t.imax.ukey())
+	}
+	c.rec.addTableFile(level, t)
+
+	s.logf("mem@flush created L%d@%d N·%d S·%s %q:%q", level, t.file.Num(), n, shortenb(int(t.size)), t.imin, t.imax)
+
+	c.level = level
+	return nil
+}
+
+func (c *cMem) reset() {
+	c.rec = new(sessionRecord)
+}
+
+func (c *cMem) commit(journal, seq uint64) error {
+	c.rec.setJournalNum(journal)
+	c.rec.setSeq(seq)
+
+	// Commit changes.
+	return c.s.commit(c.rec)
+}
+
 func (db *DB) compactionError() {
 	var err error
 noerr:
-	// No error.
 	for {
 		select {
 		case err = <-db.compErrSetC:
-			switch {
-			case err == nil:
-			case err == ErrReadOnly, errors.IsCorrupted(err):
-				goto hasperr
-			default:
+			if err != nil {
 				goto haserr
 			}
 		case _, _ = <-db.closeC:
@@ -80,39 +120,17 @@ noerr:
 		}
 	}
 haserr:
-	// Transient error.
 	for {
 		select {
 		case db.compErrC <- err:
 		case err = <-db.compErrSetC:
-			switch {
-			case err == nil:
+			if err == nil {
 				goto noerr
-			case err == ErrReadOnly, errors.IsCorrupted(err):
-				goto hasperr
-			default:
 			}
 		case _, _ = <-db.closeC:
 			return
 		}
 	}
-hasperr:
-	// Persistent error.
-	for {
-		select {
-		case db.compErrC <- err:
-		case db.compPerErrC <- err:
-		case db.writeLockC <- struct{}{}:
-			// Hold write lock, so that write won't pass-through.
-			db.compWriteLocking = true
-		case _, _ = <-db.closeC:
-			if db.compWriteLocking {
-				// We should release the lock or Close will hang.
-				<-db.writeLockC
-			}
-			return
-		}
-	}
 }
 
 type compactionTransactCounter int
@@ -121,17 +139,12 @@ func (cnt *compactionTransactCounter) incr() {
 	*cnt++
 }
 
-type compactionTransactInterface interface {
-	run(cnt *compactionTransactCounter) error
-	revert() error
-}
-
-func (db *DB) compactionTransact(name string, t compactionTransactInterface) {
+func (db *DB) compactionTransact(name string, exec func(cnt *compactionTransactCounter) error, rollback func() error) {
 	defer func() {
 		if x := recover(); x != nil {
-			if x == errCompactionTransactExiting {
-				if err := t.revert(); err != nil {
-					db.logf("%s revert error %q", name, err)
+			if x == errCompactionTransactExiting && rollback != nil {
+				if err := rollback(); err != nil {
+					db.logf("%s rollback error %q", name, err)
 				}
 			}
 			panic(x)
@@ -143,13 +156,9 @@ func (db *DB) compactionTransact(name string, t compactionTransactInterface) {
 		backoffMax = 8 * time.Second
 		backoffMul = 2 * time.Second
 	)
-	var (
-		backoff  = backoffMin
-		backoffT = time.NewTimer(backoff)
-		lastCnt  = compactionTransactCounter(0)
-
-		disableBackoff = db.s.o.GetDisableCompactionBackoff()
-	)
+	backoff := backoffMin
+	backoffT := time.NewTimer(backoff)
+	lastCnt := compactionTransactCounter(0)
 	for n := 0; ; n++ {
 		// Check wether the DB is closed.
 		if db.isClosed() {
@@ -161,19 +170,11 @@ func (db *DB) compactionTransact(name string, t compactionTransactInterface) {
 
 		// Execute.
 		cnt := compactionTransactCounter(0)
-		err := t.run(&cnt)
-		if err != nil {
-			db.logf("%s error I·%d %q", name, cnt, err)
-		}
+		err := exec(&cnt)
 
 		// Set compaction error status.
 		select {
 		case db.compErrSetC <- err:
-		case perr := <-db.compPerErrC:
-			if err != nil {
-				db.logf("%s exiting (persistent error %q)", name, perr)
-				db.compactionExitTransact()
-			}
 		case _, _ = <-db.closeC:
 			db.logf("%s exiting", name)
 			db.compactionExitTransact()
@@ -181,102 +182,70 @@ func (db *DB) compactionTransact(name string, t compactionTransactInterface) {
 		if err == nil {
 			return
 		}
-		if errors.IsCorrupted(err) {
-			db.logf("%s exiting (corruption detected)", name)
+		db.logf("%s error I·%d %q", name, cnt, err)
+
+		// Reset backoff duration if counter is advancing.
+		if cnt > lastCnt {
+			backoff = backoffMin
+			lastCnt = cnt
+		}
+
+		// Backoff.
+		backoffT.Reset(backoff)
+		if backoff < backoffMax {
+			backoff *= backoffMul
+			if backoff > backoffMax {
+				backoff = backoffMax
+			}
+		}
+		select {
+		case <-backoffT.C:
+		case _, _ = <-db.closeC:
+			db.logf("%s exiting", name)
 			db.compactionExitTransact()
 		}
-
-		if !disableBackoff {
-			// Reset backoff duration if counter is advancing.
-			if cnt > lastCnt {
-				backoff = backoffMin
-				lastCnt = cnt
-			}
-
-			// Backoff.
-			backoffT.Reset(backoff)
-			if backoff < backoffMax {
-				backoff *= backoffMul
-				if backoff > backoffMax {
-					backoff = backoffMax
-				}
-			}
-			select {
-			case <-backoffT.C:
-			case _, _ = <-db.closeC:
-				db.logf("%s exiting", name)
-				db.compactionExitTransact()
-			}
-		}
 	}
 }
 
-type compactionTransactFunc struct {
-	runFunc    func(cnt *compactionTransactCounter) error
-	revertFunc func() error
-}
-
-func (t *compactionTransactFunc) run(cnt *compactionTransactCounter) error {
-	return t.runFunc(cnt)
-}
-
-func (t *compactionTransactFunc) revert() error {
-	if t.revertFunc != nil {
-		return t.revertFunc()
-	}
-	return nil
-}
-
-func (db *DB) compactionTransactFunc(name string, run func(cnt *compactionTransactCounter) error, revert func() error) {
-	db.compactionTransact(name, &compactionTransactFunc{run, revert})
-}
-
 func (db *DB) compactionExitTransact() {
 	panic(errCompactionTransactExiting)
 }
 
 func (db *DB) memCompaction() {
-	mdb := db.getFrozenMem()
-	if mdb == nil {
+	mem := db.getFrozenMem()
+	if mem == nil {
 		return
 	}
-	defer mdb.decref()
+	defer mem.decref()
 
-	db.logf("memdb@flush N·%d S·%s", mdb.Len(), shortenb(mdb.Size()))
+	c := newCMem(db.s)
+	stats := new(cStatsStaging)
+
+	db.logf("mem@flush N·%d S·%s", mem.mdb.Len(), shortenb(mem.mdb.Size()))
 
 	// Don't compact empty memdb.
-	if mdb.Len() == 0 {
-		db.logf("memdb@flush skipping")
-		// drop frozen memdb
+	if mem.mdb.Len() == 0 {
+		db.logf("mem@flush skipping")
+		// drop frozen mem
 		db.dropFrozenMem()
 		return
 	}
 
 	// Pause table compaction.
-	resumeC := make(chan struct{})
+	ch := make(chan struct{})
 	select {
-	case db.tcompPauseC <- (chan<- struct{})(resumeC):
-	case <-db.compPerErrC:
-		close(resumeC)
-		resumeC = nil
+	case db.tcompPauseC <- (chan<- struct{})(ch):
 	case _, _ = <-db.closeC:
 		return
 	}
 
-	var (
-		rec        = &sessionRecord{}
-		stats      = &cStatsStaging{}
-		flushLevel int
-	)
-
-	db.compactionTransactFunc("memdb@flush", func(cnt *compactionTransactCounter) (err error) {
+	db.compactionTransact("mem@flush", func(cnt *compactionTransactCounter) (err error) {
 		stats.startTimer()
-		flushLevel, err = db.s.flushMemdb(rec, mdb.DB, -1)
-		stats.stopTimer()
-		return
+		defer stats.stopTimer()
+		return c.flush(mem.mdb, -1)
 	}, func() error {
-		for _, r := range rec.addedTables {
-			db.logf("memdb@flush revert @%d", r.num)
+		for _, r := range c.rec.addedTables {
+			db.logf("mem@flush rollback @%d", r.num)
 			f := db.s.getTableFile(r.num)
 			if err := f.Remove(); err != nil {
 				return err
@@ -285,243 +254,43 @@ func (db *DB) memCompaction() {
 		return nil
 	})
 
-	db.compactionTransactFunc("memdb@commit", func(cnt *compactionTransactCounter) (err error) {
+	db.compactionTransact("mem@commit", func(cnt *compactionTransactCounter) (err error) {
 		stats.startTimer()
-		rec.setJournalNum(db.journalFile.Num())
-		rec.setSeqNum(db.frozenSeq)
-		err = db.s.commit(rec)
-		stats.stopTimer()
-		return
+		defer stats.stopTimer()
+		return c.commit(db.journalFile.Num(), db.frozenSeq)
 	}, nil)
 
-	db.logf("memdb@flush committed F·%d T·%v", len(rec.addedTables), stats.duration)
+	db.logf("mem@flush commited F·%d T·%v", len(c.rec.addedTables), stats.duration)
 
-	for _, r := range rec.addedTables {
+	for _, r := range c.rec.addedTables {
 		stats.write += r.size
 	}
-	db.compStats[flushLevel].add(stats)
+	db.compStats[c.level].add(stats)
 
-	// Drop frozen memdb.
+	// Drop frozen mem.
 	db.dropFrozenMem()
 
 	// Resume table compaction.
-	if resumeC != nil {
-		select {
-		case <-resumeC:
-			close(resumeC)
-		case _, _ = <-db.closeC:
-			return
-		}
+	select {
+	case <-ch:
+	case _, _ = <-db.closeC:
+		return
 	}
 
 	// Trigger table compaction.
-	db.compSendTrigger(db.tcompCmdC)
-}
-
-type tableCompactionBuilder struct {
-	db           *DB
-	s            *session
-	c            *compaction
-	rec          *sessionRecord
-	stat0, stat1 *cStatsStaging
-
-	snapHasLastUkey bool
-	snapLastUkey    []byte
-	snapLastSeq     uint64
-	snapIter        int
-	snapKerrCnt     int
-	snapDropCnt     int
-
-	kerrCnt int
-	dropCnt int
-
-	minSeq    uint64
-	strict    bool
-	tableSize int
-
-	tw *tWriter
-}
-
-func (b *tableCompactionBuilder) appendKV(key, value []byte) error {
-	// Create new table if not already.
-	if b.tw == nil {
-		// Check for pause event.
-		if b.db != nil {
-			select {
-			case ch := <-b.db.tcompPauseC:
-				b.db.pauseCompaction(ch)
-			case _, _ = <-b.db.closeC:
-				b.db.compactionExitTransact()
-			default:
-			}
-		}
-
-		// Create new table.
-		var err error
-		b.tw, err = b.s.tops.create()
-		if err != nil {
-			return err
-		}
-	}
-
-	// Write key/value into table.
-	return b.tw.append(key, value)
-}
-
-func (b *tableCompactionBuilder) needFlush() bool {
-	return b.tw.tw.BytesLen() >= b.tableSize
-}
-
-func (b *tableCompactionBuilder) flush() error {
-	t, err := b.tw.finish()
-	if err != nil {
-		return err
-	}
-	b.rec.addTableFile(b.c.level+1, t)
-	b.stat1.write += t.size
-	b.s.logf("table@build created L%d@%d N·%d S·%s %q:%q", b.c.level+1, t.file.Num(), b.tw.tw.EntriesLen(), shortenb(int(t.size)), t.imin, t.imax)
-	b.tw = nil
-	return nil
-}
-
-func (b *tableCompactionBuilder) cleanup() {
-	if b.tw != nil {
-		b.tw.drop()
-		b.tw = nil
-	}
-}
-
-func (b *tableCompactionBuilder) run(cnt *compactionTransactCounter) error {
-	snapResumed := b.snapIter > 0
-	hasLastUkey := b.snapHasLastUkey // The key might has zero length, so this is necessary.
-	lastUkey := append([]byte{}, b.snapLastUkey...)
-	lastSeq := b.snapLastSeq
-	b.kerrCnt = b.snapKerrCnt
-	b.dropCnt = b.snapDropCnt
-	// Restore compaction state.
-	b.c.restore()
-
-	defer b.cleanup()
-
-	b.stat1.startTimer()
-	defer b.stat1.stopTimer()
-
-	iter := b.c.newIterator()
-	defer iter.Release()
-	for i := 0; iter.Next(); i++ {
-		// Incr transact counter.
-		cnt.incr()
-
-		// Skip until last state.
-		if i < b.snapIter {
-			continue
-		}
-
-		resumed := false
-		if snapResumed {
-			resumed = true
-			snapResumed = false
-		}
-
-		ikey := iter.Key()
-		ukey, seq, kt, kerr := parseIkey(ikey)
-
-		if kerr == nil {
-			shouldStop := !resumed && b.c.shouldStopBefore(ikey)
-
-			if !hasLastUkey || b.s.icmp.uCompare(lastUkey, ukey) != 0 {
-				// First occurrence of this user key.
-
-				// Only rotate tables if ukey doesn't hop across.
-				if b.tw != nil && (shouldStop || b.needFlush()) {
-					if err := b.flush(); err != nil {
-						return err
-					}
-
-					// Creates snapshot of the state.
-					b.c.save()
-					b.snapHasLastUkey = hasLastUkey
-					b.snapLastUkey = append(b.snapLastUkey[:0], lastUkey...)
-					b.snapLastSeq = lastSeq
-					b.snapIter = i
-					b.snapKerrCnt = b.kerrCnt
-					b.snapDropCnt = b.dropCnt
-				}
-
-				hasLastUkey = true
-				lastUkey = append(lastUkey[:0], ukey...)
-				lastSeq = kMaxSeq
-			}
-
-			switch {
-			case lastSeq <= b.minSeq:
-				// Dropped because newer entry for same user key exist
-				fallthrough // (A)
-			case kt == ktDel && seq <= b.minSeq && b.c.baseLevelForKey(lastUkey):
-				// For this user key:
-				// (1) there is no data in higher levels
-				// (2) data in lower levels will have larger seq numbers
-				// (3) data in layers that are being compacted here and have
-				//     smaller seq numbers will be dropped in the next
-				//     few iterations of this loop (by rule (A) above).
-				// Therefore this deletion marker is obsolete and can be dropped.
-				lastSeq = seq
-				b.dropCnt++
-				continue
-			default:
-				lastSeq = seq
-			}
-		} else {
-			if b.strict {
-				return kerr
-			}
-
-			// Don't drop corrupted keys.
-			hasLastUkey = false
-			lastUkey = lastUkey[:0]
-			lastSeq = kMaxSeq
-			b.kerrCnt++
-		}
-
-		if err := b.appendKV(ikey, iter.Value()); err != nil {
-			return err
-		}
-	}
-
-	if err := iter.Error(); err != nil {
-		return err
-	}
-
-	// Finish last table.
-	if b.tw != nil && !b.tw.empty() {
-		return b.flush()
-	}
-	return nil
-}
-
-func (b *tableCompactionBuilder) revert() error {
-	for _, at := range b.rec.addedTables {
-		b.s.logf("table@build revert @%d", at.num)
-		f := b.s.getTableFile(at.num)
-		if err := f.Remove(); err != nil {
-			return err
-		}
-	}
-	return nil
+	db.compTrigger(db.mcompTriggerC)
 }
 
 func (db *DB) tableCompaction(c *compaction, noTrivial bool) {
-	defer c.release()
-
-	rec := &sessionRecord{}
-	rec.addCompPtr(c.level, c.imax)
+	rec := new(sessionRecord)
+	rec.addCompactionPointer(c.level, c.imax)
 
 	if !noTrivial && c.trivial() {
 		t := c.tables[0][0]
 		db.logf("table@move L%d@%d -> L%d", c.level, t.file.Num(), c.level+1)
-		rec.delTable(c.level, t.file.Num())
+		rec.deleteTable(c.level, t.file.Num())
 		rec.addTableFile(c.level+1, t)
-		db.compactionTransactFunc("table@move", func(cnt *compactionTransactCounter) (err error) {
+		db.compactionTransact("table@move", func(cnt *compactionTransactCounter) (err error) {
 			return db.s.commit(rec)
 		}, nil)
 		return
@@ -532,34 +301,184 @@ func (db *DB) tableCompaction(c *compaction, noTrivial bool) {
 		for _, t := range tables {
 			stats[i].read += t.size
 			// Insert deleted tables into record
-			rec.delTable(c.level+i, t.file.Num())
+			rec.deleteTable(c.level+i, t.file.Num())
 		}
 	}
 	sourceSize := int(stats[0].read + stats[1].read)
 	minSeq := db.minSeq()
 	db.logf("table@compaction L%d·%d -> L%d·%d S·%s Q·%d", c.level, len(c.tables[0]), c.level+1, len(c.tables[1]), shortenb(sourceSize), minSeq)
 
-	b := &tableCompactionBuilder{
-		db:        db,
-		s:         db.s,
-		c:         c,
-		rec:       rec,
-		stat1:     &stats[1],
-		minSeq:    minSeq,
-		strict:    db.s.o.GetStrict(opt.StrictCompaction),
-		tableSize: db.s.o.GetCompactionTableSize(c.level + 1),
-	}
-	db.compactionTransact("table@build", b)
+	var snapUkey []byte
+	var snapHasUkey bool
+	var snapSeq uint64
+	var snapIter int
+	var snapDropCnt int
+	var dropCnt int
+	db.compactionTransact("table@build", func(cnt *compactionTransactCounter) (err error) {
+		ukey := append([]byte{}, snapUkey...)
+		hasUkey := snapHasUkey
+		lseq := snapSeq
+		dropCnt = snapDropCnt
+		snapSched := snapIter == 0
+
+		var tw *tWriter
+		finish := func() error {
+			t, err := tw.finish()
+			if err != nil {
+				return err
+			}
+			rec.addTableFile(c.level+1, t)
+			stats[1].write += t.size
+			db.logf("table@build created L%d@%d N·%d S·%s %q:%q", c.level+1, t.file.Num(), tw.tw.EntriesLen(), shortenb(int(t.size)), t.imin, t.imax)
+			return nil
+		}
+
+		defer func() {
+			stats[1].stopTimer()
+			if tw != nil {
+				tw.drop()
+				tw = nil
+			}
+		}()
+
+		stats[1].startTimer()
+		iter := c.newIterator()
+		defer iter.Release()
+		for i := 0; iter.Next(); i++ {
+			// Incr transact counter.
+			cnt.incr()
+
+			// Skip until last state.
+			if i < snapIter {
+				continue
+			}
+
+			ikey := iKey(iter.Key())
+
+			if c.shouldStopBefore(ikey) && tw != nil {
+				err = finish()
+				if err != nil {
+					return
+				}
+				snapSched = true
+				tw = nil
+			}
+
+			// Scheduled for snapshot, snapshot will used to retry compaction
+			// if error occured.
+			if snapSched {
+				snapUkey = append(snapUkey[:0], ukey...)
+				snapHasUkey = hasUkey
+				snapSeq = lseq
+				snapIter = i
+				snapDropCnt = dropCnt
+				snapSched = false
+			}
+
+			if seq, vt, ok := ikey.parseNum(); !ok {
+				// Don't drop error keys
+				ukey = ukey[:0]
+				hasUkey = false
+				lseq = kMaxSeq
+			} else {
+				if !hasUkey || db.s.icmp.uCompare(ikey.ukey(), ukey) != 0 {
+					// First occurrence of this user key
+					ukey = append(ukey[:0], ikey.ukey()...)
+					hasUkey = true
+					lseq = kMaxSeq
+				}
+
+				drop := false
+				if lseq <= minSeq {
+					// Dropped because newer entry for same user key exist
+					drop = true // (A)
+				} else if vt == tDel && seq <= minSeq && c.baseLevelForKey(ukey) {
+					// For this user key:
+					// (1) there is no data in higher levels
+					// (2) data in lower levels will have larger seq numbers
+					// (3) data in layers that are being compacted here and have
+					//     smaller seq numbers will be dropped in the next
+					//     few iterations of this loop (by rule (A) above).
+					// Therefore this deletion marker is obsolete and can be dropped.
+					drop = true
+				}
+
+				lseq = seq
+				if drop {
+					dropCnt++
+					continue
+				}
+			}
+
+			// Create new table if not already
+			if tw == nil {
+				// Check for pause event.
+				select {
+				case ch := <-db.tcompPauseC:
+					db.pauseCompaction(ch)
+				case _, _ = <-db.closeC:
+					db.compactionExitTransact()
+				default:
+				}
+
+				// Create new table.
+				tw, err = db.s.tops.create()
+				if err != nil {
+					return
+				}
+			}
+
+			// Write key/value into table
+			err = tw.append(ikey, iter.Value())
+			if err != nil {
+				return
+			}
+
+			// Finish table if it is big enough
+			if tw.tw.BytesLen() >= kMaxTableSize {
+				err = finish()
+				if err != nil {
+					return
+				}
+				snapSched = true
+				tw = nil
+			}
+		}
+
+		err = iter.Error()
+		if err != nil {
+			return
+		}
+
+		// Finish last table
+		if tw != nil && !tw.empty() {
+			err = finish()
+			if err != nil {
+				return
+			}
+			tw = nil
+		}
+		return
+	}, func() error {
+		for _, r := range rec.addedTables {
+			db.logf("table@build rollback @%d", r.num)
+			f := db.s.getTableFile(r.num)
+			if err := f.Remove(); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
 
 	// Commit changes
-	db.compactionTransactFunc("table@commit", func(cnt *compactionTransactCounter) (err error) {
+	db.compactionTransact("table@commit", func(cnt *compactionTransactCounter) (err error) {
 		stats[1].startTimer()
 		defer stats[1].stopTimer()
 		return db.s.commit(rec)
 	}, nil)
 
 	resultSize := int(stats[1].write)
-	db.logf("table@compaction committed F%s S%s Ke·%d D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), b.kerrCnt, b.dropCnt, stats[1].duration)
+	db.logf("table@compaction commited F%s S%s D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), dropCnt, stats[1].duration)
 
 	// Save compaction stats
 	for i := range stats {
@@ -575,14 +494,14 @@ func (db *DB) tableRangeCompaction(level int, umin, umax []byte) {
 			db.tableCompaction(c, true)
 		}
 	} else {
-		v := db.s.version()
+		v := db.s.version_NB()
+
 		m := 1
 		for i, t := range v.tables[1:] {
 			if t.overlaps(db.s.icmp, umin, umax, false) {
 				m = i + 1
 			}
 		}
-		v.release()
 
 		for level := 0; level < m; level++ {
 			if c := db.s.getCompactionRange(level, umin, umax); c != nil {
@@ -599,9 +518,7 @@ func (db *DB) tableAutoCompaction() {
 }
 
 func (db *DB) tableNeedCompaction() bool {
-	v := db.s.version()
-	defer v.release()
-	return v.needCompaction()
+	return db.s.version_NB().needCompaction()
 }
 
 func (db *DB) pauseCompaction(ch chan<- struct{}) {
@@ -621,12 +538,7 @@ type cIdle struct {
 }
 
 func (r cIdle) ack(err error) {
-	if r.ackC != nil {
-		defer func() {
-			recover()
-		}()
-		r.ackC <- err
-	}
+	r.ackC <- err
 }
 
 type cRange struct {
@@ -636,45 +548,29 @@ type cRange struct {
 }
 
 func (r cRange) ack(err error) {
+	defer func() {
+		recover()
+	}()
 	if r.ackC != nil {
-		defer func() {
-			recover()
-		}()
 		r.ackC <- err
 	}
 }
 
-// This will trigger auto compation and/or wait for all compaction to be done.
-func (db *DB) compSendIdle(compC chan<- cCmd) (err error) {
+func (db *DB) compSendIdle(compC chan<- cCmd) error {
 	ch := make(chan error)
 	defer close(ch)
 	// Send cmd.
 	select {
 	case compC <- cIdle{ch}:
-	case err = <-db.compErrC:
-		return
+	case err := <-db.compErrC:
+		return err
 	case _, _ = <-db.closeC:
 		return ErrClosed
 	}
 	// Wait cmd.
-	select {
-	case err = <-ch:
-	case err = <-db.compErrC:
-	case _, _ = <-db.closeC:
-		return ErrClosed
-	}
-	return err
+	return <-ch
 }
 
-// This will trigger auto compaction but will not wait for it.
-func (db *DB) compSendTrigger(compC chan<- cCmd) {
-	select {
-	case compC <- cIdle{}:
-	default:
-	}
-}
-
-// Send range compaction request.
 func (db *DB) compSendRange(compC chan<- cCmd, level int, min, max []byte) (err error) {
 	ch := make(chan error)
 	defer close(ch)
@@ -688,14 +584,19 @@ func (db *DB) compSendRange(compC chan<- cCmd, level int, min, max []byte) (err
 	}
 	// Wait cmd.
 	select {
-	case err = <-ch:
 	case err = <-db.compErrC:
-	case _, _ = <-db.closeC:
-		return ErrClosed
+	case err = <-ch:
 	}
 	return err
 }
 
+func (db *DB) compTrigger(compTriggerC chan struct{}) {
+	select {
+	case compTriggerC <- struct{}{}:
+	default:
+	}
+}
+
 func (db *DB) mCompaction() {
 	var x cCmd
 
@@ -714,14 +615,11 @@ func (db *DB) mCompaction() {
 	for {
 		select {
 		case x = <-db.mcompCmdC:
-			switch x.(type) {
-			case cIdle:
-				db.memCompaction()
-				x.ack(nil)
-				x = nil
-			default:
-				panic("leveldb: unknown command")
-			}
+			db.memCompaction()
+			x.ack(nil)
+			x = nil
+		case <-db.mcompTriggerC:
+			db.memCompaction()
 		case _, _ = <-db.closeC:
 			return
 		}
@@ -752,6 +650,7 @@ func (db *DB) tCompaction() {
 		if db.tableNeedCompaction() {
 			select {
 			case x = <-db.tcompCmdC:
+			case <-db.tcompTriggerC:
 			case ch := <-db.tcompPauseC:
 				db.pauseCompaction(ch)
 				continue
@@ -767,6 +666,7 @@ func (db *DB) tCompaction() {
 			ackQ = ackQ[:0]
 			select {
 			case x = <-db.tcompCmdC:
+			case <-db.tcompTriggerC:
 			case ch := <-db.tcompPauseC:
 				db.pauseCompaction(ch)
 				continue
@@ -781,8 +681,6 @@ func (db *DB) tCompaction() {
 			case cRange:
 				db.tableRangeCompaction(cmd.level, cmd.min, cmd.max)
 				x.ack(nil)
-			default:
-				panic("leveldb: unknown command")
 			}
 			x = nil
 		}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_iter.go

@@ -8,7 +8,6 @@ package leveldb
 
 import (
 	"errors"
-	"math/rand"
 	"runtime"
 	"sync"
 	"sync/atomic"
@@ -40,16 +39,16 @@ func (db *DB) newRawIterator(slice *util.Range, ro *opt.ReadOptions) iterator.It
 	ti := v.getIterators(slice, ro)
 	n := len(ti) + 2
 	i := make([]iterator.Iterator, 0, n)
-	emi := em.NewIterator(slice)
+	emi := em.mdb.NewIterator(slice)
 	emi.SetReleaser(&memdbReleaser{m: em})
 	i = append(i, emi)
 	if fm != nil {
-		fmi := fm.NewIterator(slice)
+		fmi := fm.mdb.NewIterator(slice)
 		fmi.SetReleaser(&memdbReleaser{m: fm})
 		i = append(i, fmi)
 	}
 	i = append(i, ti...)
-	strict := opt.GetStrict(db.s.o.Options, ro, opt.StrictReader)
+	strict := db.s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator)
 	mi := iterator.NewMergedIterator(i, db.s.icmp, strict)
 	mi.SetReleaser(&versionReleaser{v: v})
 	return mi
@@ -60,10 +59,10 @@ func (db *DB) newIterator(seq uint64, slice *util.Range, ro *opt.ReadOptions) *d
 	if slice != nil {
 		islice = &util.Range{}
 		if slice.Start != nil {
-			islice.Start = newIkey(slice.Start, kMaxSeq, ktSeek)
+			islice.Start = newIKey(slice.Start, kMaxSeq, tSeek)
 		}
 		if slice.Limit != nil {
-			islice.Limit = newIkey(slice.Limit, kMaxSeq, ktSeek)
+			islice.Limit = newIKey(slice.Limit, kMaxSeq, tSeek)
 		}
 	}
 	rawIter := db.newRawIterator(islice, ro)
@@ -72,7 +71,7 @@ func (db *DB) newIterator(seq uint64, slice *util.Range, ro *opt.ReadOptions) *d
 		icmp:   db.s.icmp,
 		iter:   rawIter,
 		seq:    seq,
-		strict: opt.GetStrict(db.s.o.Options, ro, opt.StrictReader),
+		strict: db.s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator),
 		key:    make([]byte, 0),
 		value:  make([]byte, 0),
 	}
@@ -81,10 +80,6 @@ func (db *DB) newIterator(seq uint64, slice *util.Range, ro *opt.ReadOptions) *d
 	return iter
 }
 
-func (db *DB) iterSamplingRate() int {
-	return rand.Intn(2 * db.s.o.GetIteratorSamplingRate())
-}
-
 type dir int
 
 const (
@@ -103,21 +98,11 @@ type dbIter struct {
 	seq    uint64
 	strict bool
 
-	smaplingGap int
-	dir         dir
-	key         []byte
-	value       []byte
-	err         error
-	releaser    util.Releaser
-}
-
-func (i *dbIter) sampleSeek() {
-	ikey := i.iter.Key()
-	i.smaplingGap -= len(ikey) + len(i.iter.Value())
-	for i.smaplingGap < 0 {
-		i.smaplingGap += i.db.iterSamplingRate()
-		i.db.sampleSeek(ikey)
-	}
+	dir      dir
+	key      []byte
+	value    []byte
+	err      error
+	releaser util.Releaser
 }
 
 func (i *dbIter) setErr(err error) {
@@ -177,7 +162,7 @@ func (i *dbIter) Seek(key []byte) bool {
 		return false
 	}
 
-	ikey := newIkey(key, i.seq, ktSeek)
+	ikey := newIKey(key, i.seq, tSeek)
 	if i.iter.Seek(ikey) {
 		i.dir = dirSOI
 		return i.next()
@@ -189,15 +174,15 @@ func (i *dbIter) Seek(key []byte) bool {
 
 func (i *dbIter) next() bool {
 	for {
-		if ukey, seq, kt, kerr := parseIkey(i.iter.Key()); kerr == nil {
-			i.sampleSeek()
+		ukey, seq, t, ok := parseIkey(i.iter.Key())
+		if ok {
 			if seq <= i.seq {
-				switch kt {
-				case ktDel:
+				switch t {
+				case tDel:
 					// Skip deleted key.
 					i.key = append(i.key[:0], ukey...)
 					i.dir = dirForward
-				case ktVal:
+				case tVal:
 					if i.dir == dirSOI || i.icmp.uCompare(ukey, i.key) > 0 {
 						i.key = append(i.key[:0], ukey...)
 						i.value = append(i.value[:0], i.iter.Value()...)
@@ -207,7 +192,7 @@ func (i *dbIter) next() bool {
 				}
 			}
 		} else if i.strict {
-			i.setErr(kerr)
+			i.setErr(errInvalidIkey)
 			break
 		}
 		if !i.iter.Next() {
@@ -240,20 +225,20 @@ func (i *dbIter) prev() bool {
 	del := true
 	if i.iter.Valid() {
 		for {
-			if ukey, seq, kt, kerr := parseIkey(i.iter.Key()); kerr == nil {
-				i.sampleSeek()
+			ukey, seq, t, ok := parseIkey(i.iter.Key())
+			if ok {
 				if seq <= i.seq {
 					if !del && i.icmp.uCompare(ukey, i.key) < 0 {
 						return true
 					}
-					del = (kt == ktDel)
+					del = (t == tDel)
 					if !del {
 						i.key = append(i.key[:0], ukey...)
 						i.value = append(i.value[:0], i.iter.Value()...)
 					}
 				}
 			} else if i.strict {
-				i.setErr(kerr)
+				i.setErr(errInvalidIkey)
 				return false
 			}
 			if !i.iter.Prev() {
@@ -282,13 +267,13 @@ func (i *dbIter) Prev() bool {
 		return i.Last()
 	case dirForward:
 		for i.iter.Prev() {
-			if ukey, _, _, kerr := parseIkey(i.iter.Key()); kerr == nil {
-				i.sampleSeek()
+			ukey, _, _, ok := parseIkey(i.iter.Key())
+			if ok {
 				if i.icmp.uCompare(ukey, i.key) < 0 {
 					goto cont
 				}
 			} else if i.strict {
-				i.setErr(kerr)
+				i.setErr(errInvalidIkey)
 				return false
 			}
 		}
@@ -336,13 +321,9 @@ func (i *dbIter) Release() {
 }
 
 func (i *dbIter) SetReleaser(releaser util.Releaser) {
-	if i.dir == dirReleased {
-		panic(util.ErrReleased)
+	if i.dir != dirReleased {
+		i.releaser = releaser
 	}
-	if i.releaser != nil && releaser != nil {
-		panic(util.ErrHasReleaser)
-	}
-	i.releaser = releaser
 }
 
 func (i *dbIter) Error() error {

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_snapshot.go

@@ -7,8 +7,6 @@
 package leveldb
 
 import (
-	"container/list"
-	"fmt"
 	"runtime"
 	"sync"
 	"sync/atomic"
@@ -21,41 +19,51 @@ import (
 type snapshotElement struct {
 	seq uint64
 	ref int
-	e   *list.Element
+	// Next and previous pointers in the doubly-linked list of elements.
+	next, prev *snapshotElement
+}
+
+// Initialize the snapshot.
+func (db *DB) initSnapshot() {
+	db.snapsRoot.next = &db.snapsRoot
+	db.snapsRoot.prev = &db.snapsRoot
 }
 
 // Acquires a snapshot, based on latest sequence.
 func (db *DB) acquireSnapshot() *snapshotElement {
 	db.snapsMu.Lock()
-	defer db.snapsMu.Unlock()
-
 	seq := db.getSeq()
-
-	if e := db.snapsList.Back(); e != nil {
-		se := e.Value.(*snapshotElement)
-		if se.seq == seq {
-			se.ref++
-			return se
-		} else if seq < se.seq {
-			panic("leveldb: sequence number is not increasing")
+	elem := db.snapsRoot.prev
+	if elem == &db.snapsRoot || elem.seq != seq {
+		at := db.snapsRoot.prev
+		next := at.next
+		elem = &snapshotElement{
+			seq:  seq,
+			prev: at,
+			next: next,
 		}
+		at.next = elem
+		next.prev = elem
 	}
-	se := &snapshotElement{seq: seq, ref: 1}
-	se.e = db.snapsList.PushBack(se)
-	return se
+	elem.ref++
+	db.snapsMu.Unlock()
+	return elem
 }
 
 // Releases given snapshot element.
-func (db *DB) releaseSnapshot(se *snapshotElement) {
-	db.snapsMu.Lock()
-	defer db.snapsMu.Unlock()
-
-	se.ref--
-	if se.ref == 0 {
-		db.snapsList.Remove(se.e)
-		se.e = nil
-	} else if se.ref < 0 {
-		panic("leveldb: Snapshot: negative element reference")
+func (db *DB) releaseSnapshot(elem *snapshotElement) {
+	if !db.isClosed() {
+		db.snapsMu.Lock()
+		elem.ref--
+		if elem.ref == 0 {
+			elem.prev.next = elem.next
+			elem.next.prev = elem.prev
+			elem.next = nil
+			elem.prev = nil
+		} else if elem.ref < 0 {
+			panic("leveldb: Snapshot: negative element reference")
+		}
+		db.snapsMu.Unlock()
 	}
 }
 
@@ -63,11 +71,10 @@ func (db *DB) releaseSnapshot(se *snapshotElement) {
 func (db *DB) minSeq() uint64 {
 	db.snapsMu.Lock()
 	defer db.snapsMu.Unlock()
-
-	if e := db.snapsList.Front(); e != nil {
-		return e.Value.(*snapshotElement).seq
+	elem := db.snapsRoot.prev
+	if elem != &db.snapsRoot {
+		return elem.seq
 	}
-
 	return db.getSeq()
 }
 
@@ -90,12 +97,8 @@ func (db *DB) newSnapshot() *Snapshot {
 	return snap
 }
 
-func (snap *Snapshot) String() string {
-	return fmt.Sprintf("leveldb.Snapshot{%d}", snap.elem.seq)
-}
-
 // Get gets the value for the given key. It returns ErrNotFound if
-// the DB does not contains the key.
+// the DB does not contain the key.
 //
 // The caller should not modify the contents of the returned slice, but
 // it is safe to modify the contents of the argument after Get returns.
@@ -113,23 +116,6 @@ func (snap *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err er
 	return snap.db.get(key, snap.elem.seq, ro)
 }
 
-// Has returns true if the DB does contains the given key.
-//
-// It is safe to modify the contents of the argument after Get returns.
-func (snap *Snapshot) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) {
-	err = snap.db.ok()
-	if err != nil {
-		return
-	}
-	snap.mu.RLock()
-	defer snap.mu.RUnlock()
-	if snap.released {
-		err = ErrSnapshotReleased
-		return
-	}
-	return snap.db.has(key, snap.elem.seq, ro)
-}
-
 // NewIterator returns an iterator for the snapshot of the uderlying DB.
 // The returned iterator is not goroutine-safe, but it is safe to use
 // multiple iterators concurrently, with each in a dedicated goroutine.