cmd/syncthing: Skip a calculation if timediff is zero (fixes #2854)

* master: (61 commits)
  gui: Better accessibility for folder & device panels (fixes #2288)
  cmd/syncthing: Extract interfaces for things the API depends on
  cmd/syncthing: Return 500 with an error object instead of empty 200 on marshalling failure in REST response
  lib/model: Properly handle deleting multiple files when doing scans with subs (fixes #2851)
  lib/model: Correct handling of multiple subs when scanning (fixes #2851)
  lib/relay/client: Log relay client messages (fixes #2624)
  Confirmation box for when adding multiple folders on the same path (#1960)
  meta: Amend wweich in NICKS
  gui: Update translation files and assets
  gui: Differentiate local and remote devices more clearly
  gui: Improve layout of footer on narrow screens (fixes #2663)
  connections: The Max{Send,Recv}Kbps variables are supposed to be in KiB/s
  tests: messagediff argument order should be expected, actual
  Use Bootstrap tooltips instead of plain title attributes
  Amend norgeous
  Let "systemctl help" command work
  Add dinosore
  Don't show restart prompt when changing folder label (fixes #2840)
  Amend wweich
  Mend GUI tests
  ...

.gitattributes

@@ -2,7 +2,7 @@
 *   text=auto
 
 # Except the dependencies, which we leave alone
-vendor/**   -text=auto
+Godeps/**   -text=auto
 
 # Diffs on these files are meaningless
 gui.files.go    -diff

AUTHORS

@@ -24,9 +24,7 @@ Chris Howie <me@chrishowie.com>
 Chris Joel <chris@scriptolo.gy>
 Colin Kennedy <moshen.colin@gmail.com>
 Daniel Bergmann <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
-Daniel Harte <daniel@harte.me> <daniel@danielharte.co.uk> <norgeous@users.noreply.github.com>
 Daniel Martí <mvdan@mvdan.cc>
-David Rimmer <dinosore@dbrsoftware.co.uk>
 Denis A. <denisva@gmail.com>
 Dennis Wilson <dw@risu.io>
 Dominik Heidler <dominik@heidler.eu>
@@ -49,10 +47,8 @@ Jens Diemer <github.com@jensdiemer.de> <git@jensdiemer.de>
 Jochen Voss <voss@seehuhn.de>
 Johan Vromans <jvromans@squirrel.nl>
 Karol Różycki <rozycki.karol@gmail.com>
-Kelong Cong <kc04bc@gmx.com> <kc1212@users.noreply.github.com>
 Ken'ichi Kamada <kamada@nanohz.org>
 Kevin Allen <kma1660@gmail.com>
-Lars K.W. Gohlke <lkwg82@gmx.de>
 Laurent Etiemble <laurent.etiemble@gmail.com> <laurent.etiemble@monobjc.net>
 Lode Hoste <zillode@zillode.be>
 Lord Landon Agahnim <lordlandon@gmail.com>

Godeps/Godeps.json

@@ -0,0 +1,90 @@
+{
+	"ImportPath": "github.com/syncthing/syncthing",
+	"GoVersion": "go1.5.2",
+	"Packages": [
+		"./cmd/..."
+	],
+	"Deps": [
+		{
+			"ImportPath": "github.com/bkaradzic/go-lz4",
+			"Rev": "74ddf82598bc4745b965729e9c6a463bedd33049"
+		},
+		{
+			"ImportPath": "github.com/calmh/du",
+			"Rev": "3c0690cca16228b97741327b1b6781397afbdb24"
+		},
+		{
+			"ImportPath": "github.com/calmh/luhn",
+			"Rev": "0c8388ff95fa92d4094011e5a04fc99dea3d1632"
+		},
+		{
+			"ImportPath": "github.com/calmh/xdr",
+			"Rev": "9eb3e1a622d9364deb39c831f7e5f164393d7e37"
+		},
+		{
+			"ImportPath": "github.com/golang/snappy",
+			"Rev": "723cc1e459b8eea2dea4583200fd60757d40097a"
+		},
+		{
+			"ImportPath": "github.com/juju/ratelimit",
+			"Rev": "772f5c38e468398c4511514f4f6aa9a4185bc0a0"
+		},
+		{
+			"ImportPath": "github.com/kardianos/osext",
+			"Rev": "29ae4ffbc9a6fe9fb2bc5029050ce6996ea1d3bc"
+		},
+		{
+			"ImportPath": "github.com/rcrowley/go-metrics",
+			"Rev": "1ce93efbc8f9c568886b2ef85ce305b2217b3de3"
+		},
+		{
+			"ImportPath": "github.com/syndtr/goleveldb/leveldb",
+			"Rev": "1a9d62f03ea92815b46fcaab357cfd4df264b1a0"
+		},
+		{
+			"ImportPath": "github.com/thejerf/suture",
+			"Comment": "v1.0.1",
+			"Rev": "99c1f2d613756768fc4299acd9dc621e11ed3fd7"
+		},
+		{
+			"ImportPath": "github.com/vitrun/qart/coding",
+			"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
+		},
+		{
+			"ImportPath": "github.com/vitrun/qart/gf256",
+			"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
+		},
+		{
+			"ImportPath": "github.com/vitrun/qart/qr",
+			"Rev": "ccb109cf25f0cd24474da73b9fee4e7a3e8a8ce0"
+		},
+		{
+			"ImportPath": "golang.org/x/crypto/bcrypt",
+			"Rev": "575fdbe86e5dd89229707ebec0575ce7d088a4a6"
+		},
+		{
+			"ImportPath": "golang.org/x/crypto/blowfish",
+			"Rev": "575fdbe86e5dd89229707ebec0575ce7d088a4a6"
+		},
+		{
+			"ImportPath": "golang.org/x/net/internal/iana",
+			"Rev": "042ba42fa6633b34205efc66ba5719cd3afd8d38"
+		},
+		{
+			"ImportPath": "golang.org/x/net/ipv6",
+			"Rev": "042ba42fa6633b34205efc66ba5719cd3afd8d38"
+		},
+		{
+			"ImportPath": "golang.org/x/net/proxy",
+			"Rev": "042ba42fa6633b34205efc66ba5719cd3afd8d38"
+		},
+		{
+			"ImportPath": "golang.org/x/text/transform",
+			"Rev": "5eb8d4684c4796dd36c74f6452f2c0fa6c79597e"
+		},
+		{
+			"ImportPath": "golang.org/x/text/unicode/norm",
+			"Rev": "5eb8d4684c4796dd36c74f6452f2c0fa6c79597e"
+		}
+	]
+}

Godeps/Readme

@@ -0,0 +1,5 @@
+This directory tree is generated automatically by godep.
+
+Please do not edit.
+
+See https://github.com/tools/godep for more information.

Godeps/_workspace/.gitignore

@@ -0,0 +1,2 @@
+/pkg
+/bin

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/.gitignore

@@ -0,0 +1 @@
+/lz4-example/lz4-example

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/.travis.yml

@@ -0,0 +1,9 @@
+language: go
+
+go:
+  - 1.1
+  - 1.2
+  - 1.3
+  - 1.4
+  - 1.5
+  - tip

Godeps/_workspace/src/github.com/calmh/xdr/.gitignore

@@ -0,0 +1 @@
+coverage.out

Godeps/_workspace/src/github.com/calmh/xdr/.travis.yml

@@ -0,0 +1,19 @@
+language: go
+go:
+- tip
+
+install:
+- export PATH=$PATH:$HOME/gopath/bin
+- go get golang.org/x/tools/cover
+- go get github.com/mattn/goveralls
+
+script:
+- ./generate.sh
+- go test -coverprofile=coverage.out
+
+after_success:
+- goveralls -coverprofile=coverage.out -service=travis-ci -package=calmh/xdr -repotoken="$COVERALLS_TOKEN"
+
+env:
+  global:
+    secure: SmgnrGfp2zLrA44ChRMpjPeujubt9veZ8Fx/OseMWECmacyV5N/TuDhzIbwo6QwV4xB0sBacoPzvxQbJRVjNKsPiSu72UbcQmQ7flN4Tf7nW09tSh1iW8NgrpBCq/3UYLoBu2iPBEBKm93IK0aGNAKs6oEkB0fU27iTVBwiTXOY=

Godeps/_workspace/src/github.com/calmh/xdr/README.md

@@ -6,5 +6,7 @@ xdr
 [![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat)](http://godoc.org/github.com/calmh/xdr)
 [![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://opensource.org/licenses/MIT)
 
-This is an XDR marshalling/unmarshalling library. It uses code generation and
+This is an XDR encoding/decoding library. It uses code generation and
-not reflection.
+not reflection. It supports the IPDR bastardized XDR format when built
+with `-tags ipdr`.
+

Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go

@@ -36,73 +36,31 @@ type structInfo struct {
 	Fields []fieldInfo
 }
 
-func (i structInfo) SizeExpr() string {
+var headerTpl = template.Must(template.New("header").Parse(`// ************************************************************
-	var xdrSizes = map[string]int{
-		"int8":   4,
-		"uint8":  4,
-		"int16":  4,
-		"uint16": 4,
-		"int32":  4,
-		"uint32": 4,
-		"int64":  8,
-		"uint64": 8,
-		"int":    8,
-		"bool":   4,
-	}
-
-	var terms []string
-	nl := ""
-	for _, f := range i.Fields {
-		if size := xdrSizes[f.FieldType]; size > 0 {
-			if f.IsSlice {
-				terms = append(terms, nl+"4+len(o."+f.Name+")*"+strconv.Itoa(size))
-			} else {
-				terms = append(terms, strconv.Itoa(size))
-			}
-		} else {
-			switch f.FieldType {
-			case "string", "[]byte":
-				if f.IsSlice {
-					terms = append(terms, nl+"4+xdr.SizeOfSlice(o."+f.Name+")")
-				} else {
-					terms = append(terms, nl+"4+len(o."+f.Name+")+xdr.Padding(len(o."+f.Name+"))")
-				}
-			default:
-				if f.IsSlice {
-					terms = append(terms, nl+"4+xdr.SizeOfSlice(o."+f.Name+")")
-				} else {
-					terms = append(terms, nl+"o."+f.Name+".XDRSize()")
-				}
-			}
-		}
-		nl = "\n"
-	}
-	return strings.Join(terms, "+")
-}
-
-var headerData = `// ************************************************************
 // This file is automatically generated by genxdr. Do not edit.
 // ************************************************************
 
 package {{.Package}}
 
 import (
+	"bytes"
+	"io"
+
 	"github.com/calmh/xdr"
 )
-`
+`))
 
-var encoderData = `
+var encodeTpl = template.Must(template.New("encoder").Parse(`
-func (o {{.Name}}) XDRSize() int {
+func (o {{.TypeName}}) EncodeXDR(w io.Writer) (int, error) {
-	return {{.SizeExpr}}
+	var xw = xdr.NewWriter(w)
+	return o.EncodeXDRInto(xw)
 }//+n
 
-func (o {{.Name}}) MarshalXDR() ([]byte, error) {
+func (o {{.TypeName}}) MarshalXDR() ([]byte, error) {
-	buf:= make([]byte, o.XDRSize())
+	return o.AppendXDR(make([]byte, 0, 128))
-	m := &xdr.Marshaller{Data: buf}
-	return buf, o.MarshalXDRInto(m)
 }//+n
 
-func (o {{.Name}}) MustMarshalXDR() []byte {
+func (o {{.TypeName}}) MustMarshalXDR() []byte {
 	bs, err := o.MarshalXDR()
 	if err != nil {
 		panic(err)
@@ -110,155 +68,141 @@ func (o {{.Name}}) MustMarshalXDR() []byte {
 	return bs
 }//+n
 
-func (o {{.Name}}) MarshalXDRInto(m *xdr.Marshaller) error {
+func (o {{.TypeName}}) AppendXDR(bs []byte) ([]byte, error) {
-	{{range $fi := .Fields}}
+	var aw = xdr.AppendWriter(bs)
-		{{if $fi.IsSlice}}
+	var xw = xdr.NewWriter(&aw)
-			{{template "marshalSlice" $fi}}
+	_, err := o.EncodeXDRInto(xw)
-		{{else}}
+	return []byte(aw), err
-			{{template "marshalValue" $fi}}
-		{{end}}
-	{{end}}
-	return m.Error
 }//+n
 
-{{define "marshalValue"}}
+func (o {{.TypeName}}) EncodeXDRInto(xw *xdr.Writer) (int, error) {
-	{{if ne .Convert ""}}
+	{{range $fieldInfo := .Fields}}
-		m.Marshal{{.Encoder}}({{.Convert}}(o.{{.Name}}))
+		{{if not $fieldInfo.IsSlice}}
-	{{else if .IsBasic}}
+			{{if ne $fieldInfo.Convert ""}}
-		{{if ge .Max 1}}
+				xw.Write{{$fieldInfo.Encoder}}({{$fieldInfo.Convert}}(o.{{$fieldInfo.Name}}))
-			if l := len(o.{{.Name}}); l > {{.Max}} {
+			{{else if $fieldInfo.IsBasic}}
-				return xdr.ElementSizeExceeded("{{.Name}}", l, {{.Max}})
+				{{if ge $fieldInfo.Max 1}}
+					if l := len(o.{{$fieldInfo.Name}}); l > {{$fieldInfo.Max}} {
+						return xw.Tot(), xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", l, {{$fieldInfo.Max}})
+					}
+				{{end}}
+				xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}})
+			{{else}}
+				_, err := o.{{$fieldInfo.Name}}.EncodeXDRInto(xw)
+				if err != nil {
+					return xw.Tot(), err
+				}
+			{{end}}
+		{{else}}
+			{{if ge $fieldInfo.Max 1}}
+				if l := len(o.{{$fieldInfo.Name}}); l > {{$fieldInfo.Max}} {
+					return xw.Tot(), xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", l, {{$fieldInfo.Max}})
+				}
+			{{end}}
+			xw.WriteUint32(uint32(len(o.{{$fieldInfo.Name}})))
+			for i := range o.{{$fieldInfo.Name}} {
+			{{if ne $fieldInfo.Convert ""}}
+				xw.Write{{$fieldInfo.Encoder}}({{$fieldInfo.Convert}}(o.{{$fieldInfo.Name}}[i]))
+			{{else if $fieldInfo.IsBasic}}
+				xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}}[i])
+			{{else}}
+				_, err := o.{{$fieldInfo.Name}}[i].EncodeXDRInto(xw)
+				if err != nil {
+					return xw.Tot(), err
+				}
+			{{end}}
 			}
 		{{end}}
-		m.Marshal{{.Encoder}}(o.{{.Name}})
-	{{else}}
-		if err := o.{{.Name}}.MarshalXDRInto(m); err != nil {
-			return err
-		}
 	{{end}}
-{{end}}
+	return xw.Tot(), xw.Error()
-
-{{define "marshalSlice"}}
-	{{if ge .Max 1}}
-		if l := len(o.{{.Name}}); l > {{.Max}} {
-			return xdr.ElementSizeExceeded("{{.Name}}", l, {{.Max}})
-		}
-	{{end}}
-
-	m.MarshalUint32(uint32(len(o.{{.Name}})))
-	for i := range o.{{.Name}} {
-		{{if ne .Convert ""}}
-			m.Marshal{{.Encoder}}({{.Convert}}(o.{{.Name}}[i]))
-		{{else if .IsBasic}}
-			m.Marshal{{.Encoder}}(o.{{.Name}}[i])
-		{{else}}
-			if err := o.{{.Name}}[i].MarshalXDRInto(m); err != nil {
-				return err
-			}
-		{{end}}
-	}
-{{end}}
-
-func (o *{{.Name}}) UnmarshalXDR(bs []byte) error {
-	u := &xdr.Unmarshaller{Data: bs}
-	return o.UnmarshalXDRFrom(u)
-}
-
-func (o *{{.Name}}) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	{{range $fi := .Fields}}
-		{{if $fi.IsSlice}}
-			{{template "unmarshalSlice" $fi}}
-		{{else}}
-			{{template "unmarshalValue" $fi}}
-		{{end}}
-	{{end}}
-	return u.Error
 }//+n
 
-{{define "unmarshalValue"}}
+func (o *{{.TypeName}}) DecodeXDR(r io.Reader) error {
-	{{if ne .Convert ""}}
+	xr := xdr.NewReader(r)
-		o.{{.Name}} = {{.FieldType}}(u.Unmarshal{{.Encoder}}())
+	return o.DecodeXDRFrom(xr)
-	{{else if .IsBasic}}
+}//+n
-		{{if ge .Max 1}}
-			o.{{.Name}} = u.Unmarshal{{.Encoder}}Max({{.Max}})
-		{{else}}
-			o.{{.Name}} = u.Unmarshal{{.Encoder}}()
-		{{end}}
-	{{else}}
-		(&o.{{.Name}}).UnmarshalXDRFrom(u)
-	{{end}}
-{{end}}
 
-{{define "unmarshalSlice"}}
+func (o *{{.TypeName}}) UnmarshalXDR(bs []byte) error {
-	_{{.Name}}Size := int(u.UnmarshalUint32())
+	var br = bytes.NewReader(bs)
-	if _{{.Name}}Size < 0 {
+	var xr = xdr.NewReader(br)
-		return xdr.ElementSizeExceeded("{{.Name}}", _{{.Name}}Size, {{.Max}})
+	return o.DecodeXDRFrom(xr)
-	} else if _{{.Name}}Size == 0 {
+}//+n
-		o.{{.Name}} = nil
+
-	} else {
+func (o *{{.TypeName}}) DecodeXDRFrom(xr *xdr.Reader) error {
-		{{if ge .Max 1}}
+	{{range $fieldInfo := .Fields}}
-			if _{{.Name}}Size > {{.Max}} {
+		{{if not $fieldInfo.IsSlice}}
-				return xdr.ElementSizeExceeded("{{.Name}}", _{{.Name}}Size, {{.Max}})
+			{{if ne $fieldInfo.Convert ""}}
-			}
+				o.{{$fieldInfo.Name}} = {{$fieldInfo.FieldType}}(xr.Read{{$fieldInfo.Encoder}}())
-		{{end}}
+			{{else if $fieldInfo.IsBasic}}
-		if _{{.Name}}Size <= len(o.{{.Name}}) {
+				{{if ge $fieldInfo.Max 1}}
-			{{if eq .FieldType "string"}}
+					o.{{$fieldInfo.Name}} = xr.Read{{$fieldInfo.Encoder}}Max({{$fieldInfo.Max}})
-				for i := _{{.Name}}Size; i < len(o.{{.Name}}); i++ { o.{{.Name}}[i] = "" }
-			{{end}}
-			{{if eq .FieldType "[]byte"}}
-				for i := _{{.Name}}Size; i < len(o.{{.Name}}); i++ { o.{{.Name}}[i] = nil }
-			{{end}}
-			o.{{.Name}} = o.{{.Name}}[:_{{.Name}}Size]
-		} else {
-			o.{{.Name}} = make([]{{.FieldType}}, _{{.Name}}Size)
-		}
-		for i := range o.{{.Name}} {
-			{{if ne .Convert ""}}
-				o.{{.Name}}[i] = {{.FieldType}}(u.Unmarshal{{.Encoder}}())
-			{{else if .IsBasic}}
-				{{if ge .Submax 1}}
-					o.{{.Name}}[i] = u.Unmarshal{{.Encoder}}Max({{.Submax}})
 				{{else}}
-					o.{{.Name}}[i] = u.Unmarshal{{.Encoder}}()
+					o.{{$fieldInfo.Name}} = xr.Read{{$fieldInfo.Encoder}}()
 				{{end}}
 			{{else}}
-				(&o.{{.Name}}[i]).UnmarshalXDRFrom(u)
+				(&o.{{$fieldInfo.Name}}).DecodeXDRFrom(xr)
 			{{end}}
-		}
+		{{else}}
-	}
+			_{{$fieldInfo.Name}}Size := int(xr.ReadUint32())
-{{end}}
+			if _{{$fieldInfo.Name}}Size < 0 {
-`
+				return xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", _{{$fieldInfo.Name}}Size, {{$fieldInfo.Max}})
-
+			}
-var (
+			{{if ge $fieldInfo.Max 1}}
-	encodeTpl = template.Must(template.New("encoder").Parse(encoderData))
+				if _{{$fieldInfo.Name}}Size > {{$fieldInfo.Max}} {
-	headerTpl = template.Must(template.New("header").Parse(headerData))
+					return xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", _{{$fieldInfo.Name}}Size, {{$fieldInfo.Max}})
-)
+				}
+			{{end}}
+			o.{{$fieldInfo.Name}} = make([]{{$fieldInfo.FieldType}}, _{{$fieldInfo.Name}}Size)
+			for i := range o.{{$fieldInfo.Name}} {
+				{{if ne $fieldInfo.Convert ""}}
+					o.{{$fieldInfo.Name}}[i] = {{$fieldInfo.FieldType}}(xr.Read{{$fieldInfo.Encoder}}())
+				{{else if $fieldInfo.IsBasic}}
+					{{if ge $fieldInfo.Submax 1}}
+						o.{{$fieldInfo.Name}}[i] = xr.Read{{$fieldInfo.Encoder}}Max({{$fieldInfo.Submax}})
+					{{else}}
+						o.{{$fieldInfo.Name}}[i] = xr.Read{{$fieldInfo.Encoder}}()
+					{{end}}
+				{{else}}
+					(&o.{{$fieldInfo.Name}}[i]).DecodeXDRFrom(xr)
+				{{end}}
+			}
+		{{end}}
+	{{end}}
+	return xr.Error()
+}`))
 
 var emptyTypeTpl = template.Must(template.New("encoder").Parse(`
-func (o {{.Name}}) XDRSize() int {
+func (o {{.TypeName}}) EncodeXDR(w io.Writer) (int, error) {
-	return 0
+	return 0, nil
-}
+}//+n
 
-func (o {{.Name}}) MarshalXDR() ([]byte, error) {
+func (o {{.TypeName}}) MarshalXDR() ([]byte, error) {
 	return nil, nil
 }//+n
 
-func (o {{.Name}}) MustMarshalXDR() []byte {
+func (o {{.TypeName}}) MustMarshalXDR() []byte {
 	return nil
 }//+n
 
-func (o {{.Name}}) MarshalXDRInto(m *xdr.Marshaller) error {
+func (o {{.TypeName}}) AppendXDR(bs []byte) ([]byte, error) {
+	return bs, nil
+}//+n
+
+func (o {{.TypeName}}) EncodeXDRInto(xw *xdr.Writer) (int, error) {
+	return xw.Tot(), xw.Error()
+}//+n
+
+func (o *{{.TypeName}}) DecodeXDR(r io.Reader) error {
 	return nil
 }//+n
 
-func (o *{{.Name}}) UnmarshalXDR(bs []byte) error {
+func (o *{{.TypeName}}) UnmarshalXDR(bs []byte) error {
 	return nil
 }//+n
 
-func (o *{{.Name}}) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
+func (o *{{.TypeName}}) DecodeXDRFrom(xr *xdr.Reader) error {
-	return nil
+	return xr.Error()
-}//+n
+}`))
-`))
 
 var maxRe = regexp.MustCompile(`(?:\Wmax:)(\d+)(?:\s*,\s*(\d+))?`)
 
@@ -342,7 +286,7 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 				f = fieldInfo{
 					Name:      fn,
 					IsBasic:   true,
-					FieldType: "[]" + tn,
+					FieldType: tn,
 					Encoder:   enc.Encoder,
 					Convert:   enc.Type,
 					Max:       max1,
@@ -385,14 +329,17 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 }
 
 func generateCode(output io.Writer, s structInfo) {
+	name := s.Name
+	fs := s.Fields
+
 	var buf bytes.Buffer
 	var err error
-	if len(s.Fields) == 0 {
+	if len(fs) == 0 {
 		// This is an empty type. We can create a quite simple codec for it.
-		err = emptyTypeTpl.Execute(&buf, s)
+		err = emptyTypeTpl.Execute(&buf, map[string]interface{}{"TypeName": name})
 	} else {
 		// Generate with the default template.
-		err = encodeTpl.Execute(&buf, s)
+		err = encodeTpl.Execute(&buf, map[string]interface{}{"TypeName": name, "Fields": fs})
 	}
 	if err != nil {
 		panic(err)
@@ -400,7 +347,12 @@ func generateCode(output io.Writer, s structInfo) {
 
 	bs := regexp.MustCompile(`(\s*\n)+`).ReplaceAll(buf.Bytes(), []byte("\n"))
 	bs = bytes.Replace(bs, []byte("//+n"), []byte("\n"), -1)
-	output.Write(bs)
+
+	bs, err = format.Source(bs)
+	if err != nil {
+		panic(err)
+	}
+	fmt.Fprintln(output, string(bs))
 }
 
 func uncamelize(s string) string {
@@ -432,46 +384,46 @@ func generateDiagram(output io.Writer, s structInfo) {
 		tn := f.FieldType
 		name := uncamelize(f.Name)
 
-		suffix := ""
 		if f.IsSlice {
 			fmt.Fprintf(output, "| %s |\n", center("Number of "+name, 61))
 			fmt.Fprintln(output, line)
-			suffix = " (n items)"
-			fmt.Fprintf(output, "/ %s /\n", center("", 61))
 		}
 		switch tn {
 		case "bool":
 			fmt.Fprintf(output, "| %s |V|\n", center(name+" (V=0 or 1)", 59))
+			fmt.Fprintln(output, line)
 		case "int16", "uint16":
-			fmt.Fprintf(output, "| %s | %s |\n", center("16 zero bits", 29), center(name, 29))
+			fmt.Fprintf(output, "| %s | %s |\n", center("0x0000", 29), center(name, 29))
-		case "int8", "uint8":
+			fmt.Fprintln(output, line)
-			fmt.Fprintf(output, "| %s | %s |\n", center("24 zero bits", 45), center(name, 13))
 		case "int32", "uint32":
-			fmt.Fprintf(output, "| %s |\n", center(name+suffix, 61))
+			fmt.Fprintf(output, "| %s |\n", center(name, 61))
+			fmt.Fprintln(output, line)
 		case "int64", "uint64":
 			fmt.Fprintf(output, "| %-61s |\n", "")
 			fmt.Fprintf(output, "+ %s +\n", center(name+" (64 bits)", 61))
 			fmt.Fprintf(output, "| %-61s |\n", "")
-		case "string", "[]byte":
+			fmt.Fprintln(output, line)
+		case "string", "byte": // XXX We assume slice of byte!
+			fmt.Fprintf(output, "| %s |\n", center("Length of "+name, 61))
+			fmt.Fprintln(output, line)
 			fmt.Fprintf(output, "/ %61s /\n", "")
-			fmt.Fprintf(output, "\\ %s \\\n", center(name+" (length + padded data)", 61))
+			fmt.Fprintf(output, "\\ %s \\\n", center(name+" (variable length)", 61))
 			fmt.Fprintf(output, "/ %61s /\n", "")
+			fmt.Fprintln(output, line)
 		default:
 			if f.IsSlice {
 				tn = "Zero or more " + tn + " Structures"
+				fmt.Fprintf(output, "/ %s /\n", center("", 61))
 				fmt.Fprintf(output, "\\ %s \\\n", center(tn, 61))
+				fmt.Fprintf(output, "/ %s /\n", center("", 61))
 			} else {
 				tn = tn + " Structure"
 				fmt.Fprintf(output, "/ %s /\n", center("", 61))
 				fmt.Fprintf(output, "\\ %s \\\n", center(tn, 61))
 				fmt.Fprintf(output, "/ %s /\n", center("", 61))
 			}
+			fmt.Fprintln(output, line)
 		}
-
-		if f.IsSlice {
-			fmt.Fprintf(output, "/ %s /\n", center("", 61))
-		}
-		fmt.Fprintln(output, line)
 	}
 	fmt.Fprintln(output)
 	fmt.Fprintln(output)
@@ -496,9 +448,9 @@ func generateXdr(output io.Writer, s structInfo) {
 		}
 
 		switch tn {
-		case "int8", "int16", "int32":
+		case "int16", "int32":
 			fmt.Fprintf(output, "\tint %s%s;\n", fn, suf)
-		case "uint8", "uint16", "uint32":
+		case "uint16", "uint32":
 			fmt.Fprintf(output, "\tunsigned int %s%s;\n", fn, suf)
 		case "int64":
 			fmt.Fprintf(output, "\thyper %s%s;\n", fn, suf)
@@ -506,7 +458,7 @@ func generateXdr(output io.Writer, s structInfo) {
 			fmt.Fprintf(output, "\tunsigned hyper %s%s;\n", fn, suf)
 		case "string":
 			fmt.Fprintf(output, "\tstring %s<%s>;\n", fn, l)
-		case "[]byte":
+		case "byte":
 			fmt.Fprintf(output, "\topaque %s<%s>;\n", fn, l)
 		default:
 			fmt.Fprintf(output, "\t%s %s%s;\n", tn, fn, suf)
@@ -558,22 +510,6 @@ func main() {
 	i := inspector(&structs)
 	ast.Inspect(f, i)
 
-	buf := new(bytes.Buffer)
-	headerTpl.Execute(buf, map[string]string{"Package": f.Name.Name})
-	for _, s := range structs {
-		fmt.Fprintf(buf, "\n/*\n\n")
-		generateDiagram(buf, s)
-		generateXdr(buf, s)
-		fmt.Fprintf(buf, "*/\n")
-		generateCode(buf, s)
-	}
-
-	bs, err := format.Source(buf.Bytes())
-	if err != nil {
-		log.Print(buf.String())
-		log.Fatal(err)
-	}
-
 	var output io.Writer = os.Stdout
 	if *outputFile != "" {
 		fd, err := os.Create(*outputFile)
@@ -582,5 +518,13 @@ func main() {
 		}
 		output = fd
 	}
-	output.Write(bs)
+
+	headerTpl.Execute(output, map[string]string{"Package": f.Name.Name})
+	for _, s := range structs {
+		fmt.Fprintf(output, "\n/*\n\n")
+		generateDiagram(output, s)
+		generateXdr(output, s)
+		fmt.Fprintf(output, "*/\n")
+		generateCode(output, s)
+	}
 }

Godeps/_workspace/src/github.com/calmh/xdr/debug.go

@@ -0,0 +1,16 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+package xdr
+
+import (
+	"log"
+	"os"
+)
+
+var (
+	debug = len(os.Getenv("XDRTRACE")) > 0
+	dl    = log.New(os.Stdout, "xdr: ", log.Lshortfile|log.Ltime|log.Lmicroseconds)
+)
+
+const maxDebugBytes = 32

Godeps/_workspace/src/github.com/calmh/xdr/doc.go

@@ -1,5 +1,5 @@
 // Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
 // is governed by an MIT-style license that can be found in the LICENSE file.
 
-// Package xdr implements an XDR (RFC 4506) marshaller/unmarshaller.
+// Package xdr implements an XDR (RFC 4506) encoder/decoder.
 package xdr

Godeps/_workspace/src/github.com/calmh/xdr/pad_ipdr.go

@@ -0,0 +1,10 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+// +build ipdr
+
+package xdr
+
+func pad(l int) int {
+	return 0
+}

Godeps/_workspace/src/github.com/calmh/xdr/pad_xdr.go

@@ -0,0 +1,14 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+// +build !ipdr
+
+package xdr
+
+func pad(l int) int {
+	d := l % 4
+	if d == 0 {
+		return 0
+	}
+	return 4 - d
+}

Godeps/_workspace/src/github.com/calmh/xdr/reader.go

@@ -0,0 +1,171 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+package xdr
+
+import (
+	"fmt"
+	"io"
+	"reflect"
+	"unsafe"
+)
+
+type Reader struct {
+	r   io.Reader
+	err error
+	b   [8]byte
+}
+
+func NewReader(r io.Reader) *Reader {
+	return &Reader{
+		r: r,
+	}
+}
+
+func (r *Reader) ReadRaw(bs []byte) (int, error) {
+	if r.err != nil {
+		return 0, r.err
+	}
+
+	var n int
+	n, r.err = io.ReadFull(r.r, bs)
+	return n, r.err
+}
+
+func (r *Reader) ReadString() string {
+	return r.ReadStringMax(0)
+}
+
+func (r *Reader) ReadStringMax(max int) string {
+	buf := r.ReadBytesMaxInto(max, nil)
+	bh := (*reflect.SliceHeader)(unsafe.Pointer(&buf))
+	sh := reflect.StringHeader{
+		Data: bh.Data,
+		Len:  bh.Len,
+	}
+	return *((*string)(unsafe.Pointer(&sh)))
+}
+
+func (r *Reader) ReadBytes() []byte {
+	return r.ReadBytesInto(nil)
+}
+
+func (r *Reader) ReadBytesMax(max int) []byte {
+	return r.ReadBytesMaxInto(max, nil)
+}
+
+func (r *Reader) ReadBytesInto(dst []byte) []byte {
+	return r.ReadBytesMaxInto(0, dst)
+}
+
+func (r *Reader) ReadBytesMaxInto(max int, dst []byte) []byte {
+	if r.err != nil {
+		return nil
+	}
+
+	l := int(r.ReadUint32())
+	if r.err != nil {
+		return nil
+	}
+	if l < 0 || max > 0 && l > max {
+		// l may be negative on 32 bit builds
+		r.err = ElementSizeExceeded("bytes field", l, max)
+		return nil
+	}
+
+	if fullLen := l + pad(l); fullLen > len(dst) {
+		dst = make([]byte, fullLen)
+	} else {
+		dst = dst[:fullLen]
+	}
+
+	var n int
+	n, r.err = io.ReadFull(r.r, dst)
+	if r.err != nil {
+		if debug {
+			dl.Printf("rd bytes (%d): %v", len(dst), r.err)
+		}
+		return nil
+	}
+
+	if debug {
+		if n > maxDebugBytes {
+			dl.Printf("rd bytes (%d): %x...", len(dst), dst[:maxDebugBytes])
+		} else {
+			dl.Printf("rd bytes (%d): %x", len(dst), dst)
+		}
+	}
+	return dst[:l]
+}
+
+func (r *Reader) ReadBool() bool {
+	return r.ReadUint8() != 0
+}
+
+func (r *Reader) ReadUint32() uint32 {
+	if r.err != nil {
+		return 0
+	}
+
+	_, r.err = io.ReadFull(r.r, r.b[:4])
+	if r.err != nil {
+		if debug {
+			dl.Printf("rd uint32: %v", r.err)
+		}
+		return 0
+	}
+
+	v := uint32(r.b[3]) | uint32(r.b[2])<<8 | uint32(r.b[1])<<16 | uint32(r.b[0])<<24
+
+	if debug {
+		dl.Printf("rd uint32=%d (0x%08x)", v, v)
+	}
+	return v
+}
+
+func (r *Reader) ReadUint64() uint64 {
+	if r.err != nil {
+		return 0
+	}
+
+	_, r.err = io.ReadFull(r.r, r.b[:8])
+	if r.err != nil {
+		if debug {
+			dl.Printf("rd uint64: %v", r.err)
+		}
+		return 0
+	}
+
+	v := uint64(r.b[7]) | uint64(r.b[6])<<8 | uint64(r.b[5])<<16 | uint64(r.b[4])<<24 |
+		uint64(r.b[3])<<32 | uint64(r.b[2])<<40 | uint64(r.b[1])<<48 | uint64(r.b[0])<<56
+
+	if debug {
+		dl.Printf("rd uint64=%d (0x%016x)", v, v)
+	}
+	return v
+}
+
+type XDRError struct {
+	op  string
+	err error
+}
+
+func (e XDRError) Error() string {
+	return "xdr " + e.op + ": " + e.err.Error()
+}
+
+func (e XDRError) IsEOF() bool {
+	return e.err == io.EOF
+}
+
+func (r *Reader) Error() error {
+	if r.err == nil {
+		return nil
+	}
+	return XDRError{"read", r.err}
+}
+
+func ElementSizeExceeded(field string, size, limit int) error {
+	return fmt.Errorf("%s exceeds size limit; %d > %d", field, size, limit)
+}

Godeps/_workspace/src/github.com/calmh/xdr/reader_ipdr.go

@@ -0,0 +1,49 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// +build ipdr
+
+package xdr
+
+import "io"
+
+func (r *Reader) ReadUint8() uint8 {
+	if r.err != nil {
+		return 0
+	}
+
+	_, r.err = io.ReadFull(r.r, r.b[:1])
+	if r.err != nil {
+		if debug {
+			dl.Printf("rd uint8: %v", r.err)
+		}
+		return 0
+	}
+
+	if debug {
+		dl.Printf("rd uint8=%d (0x%02x)", r.b[0], r.b[0])
+	}
+	return r.b[0]
+}
+
+func (r *Reader) ReadUint16() uint16 {
+	if r.err != nil {
+		return 0
+	}
+
+	_, r.err = io.ReadFull(r.r, r.b[:2])
+	if r.err != nil {
+		if debug {
+			dl.Printf("rd uint16: %v", r.err)
+		}
+		return 0
+	}
+
+	v := uint16(r.b[1]) | uint16(r.b[0])<<8
+
+	if debug {
+		dl.Printf("rd uint16=%d (0x%04x)", v, v)
+	}
+	return v
+}

Godeps/_workspace/src/github.com/calmh/xdr/reader_xdr.go

@@ -0,0 +1,15 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+// All rights reserved. Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file.
+
+// +build !ipdr
+
+package xdr
+
+func (r *Reader) ReadUint8() uint8 {
+	return uint8(r.ReadUint32())
+}
+
+func (r *Reader) ReadUint16() uint16 {
+	return uint16(r.ReadUint32())
+}

Godeps/_workspace/src/github.com/calmh/xdr/writer.go

@@ -0,0 +1,146 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+package xdr
+
+import (
+	"io"
+	"reflect"
+	"unsafe"
+)
+
+var padBytes = []byte{0, 0, 0}
+
+type Writer struct {
+	w   io.Writer
+	tot int
+	err error
+	b   [8]byte
+}
+
+type AppendWriter []byte
+
+func (w *AppendWriter) Write(bs []byte) (int, error) {
+	*w = append(*w, bs...)
+	return len(bs), nil
+}
+
+func NewWriter(w io.Writer) *Writer {
+	return &Writer{
+		w: w,
+	}
+}
+
+func (w *Writer) WriteRaw(bs []byte) (int, error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+
+	var n int
+	n, w.err = w.w.Write(bs)
+	return n, w.err
+}
+
+func (w *Writer) WriteString(s string) (int, error) {
+	sh := *((*reflect.StringHeader)(unsafe.Pointer(&s)))
+	bh := reflect.SliceHeader{
+		Data: sh.Data,
+		Len:  sh.Len,
+		Cap:  sh.Len,
+	}
+	return w.WriteBytes(*(*[]byte)(unsafe.Pointer(&bh)))
+}
+
+func (w *Writer) WriteBytes(bs []byte) (int, error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+
+	w.WriteUint32(uint32(len(bs)))
+	if w.err != nil {
+		return 0, w.err
+	}
+
+	if debug {
+		if len(bs) > maxDebugBytes {
+			dl.Printf("wr bytes (%d): %x...", len(bs), bs[:maxDebugBytes])
+		} else {
+			dl.Printf("wr bytes (%d): %x", len(bs), bs)
+		}
+	}
+
+	var l, n int
+	n, w.err = w.w.Write(bs)
+	l += n
+
+	if p := pad(len(bs)); w.err == nil && p > 0 {
+		n, w.err = w.w.Write(padBytes[:p])
+		l += n
+	}
+
+	w.tot += l
+	return l, w.err
+}
+
+func (w *Writer) WriteBool(v bool) (int, error) {
+	if v {
+		return w.WriteUint8(1)
+	} else {
+		return w.WriteUint8(0)
+	}
+}
+
+func (w *Writer) WriteUint32(v uint32) (int, error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+
+	if debug {
+		dl.Printf("wr uint32=%d", v)
+	}
+
+	w.b[0] = byte(v >> 24)
+	w.b[1] = byte(v >> 16)
+	w.b[2] = byte(v >> 8)
+	w.b[3] = byte(v)
+
+	var l int
+	l, w.err = w.w.Write(w.b[:4])
+	w.tot += l
+	return l, w.err
+}
+
+func (w *Writer) WriteUint64(v uint64) (int, error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+
+	if debug {
+		dl.Printf("wr uint64=%d", v)
+	}
+
+	w.b[0] = byte(v >> 56)
+	w.b[1] = byte(v >> 48)
+	w.b[2] = byte(v >> 40)
+	w.b[3] = byte(v >> 32)
+	w.b[4] = byte(v >> 24)
+	w.b[5] = byte(v >> 16)
+	w.b[6] = byte(v >> 8)
+	w.b[7] = byte(v)
+
+	var l int
+	l, w.err = w.w.Write(w.b[:8])
+	w.tot += l
+	return l, w.err
+}
+
+func (w *Writer) Tot() int {
+	return w.tot
+}
+
+func (w *Writer) Error() error {
+	if w.err == nil {
+		return nil
+	}
+	return XDRError{"write", w.err}
+}

Godeps/_workspace/src/github.com/calmh/xdr/writer_ipdr.go

@@ -0,0 +1,41 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+// +build ipdr
+
+package xdr
+
+func (w *Writer) WriteUint8(v uint8) (int, error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+
+	if debug {
+		dl.Printf("wr uint8=%d", v)
+	}
+
+	w.b[0] = byte(v)
+
+	var l int
+	l, w.err = w.w.Write(w.b[:1])
+	w.tot += l
+	return l, w.err
+}
+
+func (w *Writer) WriteUint16(v uint16) (int, error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+
+	if debug {
+		dl.Printf("wr uint8=%d", v)
+	}
+
+	w.b[0] = byte(v >> 8)
+	w.b[1] = byte(v)
+
+	var l int
+	l, w.err = w.w.Write(w.b[:2])
+	w.tot += l
+	return l, w.err
+}

Godeps/_workspace/src/github.com/calmh/xdr/writer_xdr.go

@@ -0,0 +1,14 @@
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
+
+// +build !ipdr
+
+package xdr
+
+func (w *Writer) WriteUint8(v uint8) (int, error) {
+	return w.WriteUint32(uint32(v))
+}
+
+func (w *Writer) WriteUint16(v uint16) (int, error) {
+	return w.WriteUint32(uint32(v))
+}

Godeps/_workspace/src/github.com/golang/snappy/AUTHORS

@@ -11,5 +11,4 @@
 Damian Gryski <dgryski@gmail.com>
 Google Inc.
 Jan Mercl <0xjnml@gmail.com>
-Rodolfo Carvalho <rhcarvalho@gmail.com>
 Sebastien Binet <seb.binet@gmail.com>

Godeps/_workspace/src/github.com/golang/snappy/CONTRIBUTORS

@@ -32,6 +32,5 @@ Kai Backman <kaib@golang.org>
 Marc-Antoine Ruel <maruel@chromium.org>
 Nigel Tao <nigeltao@golang.org>
 Rob Pike <r@golang.org>
-Rodolfo Carvalho <rhcarvalho@gmail.com>
 Russ Cox <rsc@golang.org>
 Sebastien Binet <seb.binet@gmail.com>

Godeps/_workspace/src/github.com/golang/snappy/decode.go

@@ -17,9 +17,6 @@ var (
 	ErrTooLarge = errors.New("snappy: decoded block is too large")
 	// ErrUnsupported reports that the input isn't supported.
 	ErrUnsupported = errors.New("snappy: unsupported input")
-
-	errUnsupportedCopy4Tag      = errors.New("snappy: unsupported COPY_4 tag")
-	errUnsupportedLiteralLength = errors.New("snappy: unsupported literal length")
 )
 
 // DecodedLen returns the length of the decoded block.
@@ -43,36 +40,96 @@ func decodedLen(src []byte) (blockLen, headerLen int, err error) {
 	return int(v), n, nil
 }
 
-const (
-	decodeErrCodeCorrupt                  = 1
-	decodeErrCodeUnsupportedLiteralLength = 2
-	decodeErrCodeUnsupportedCopy4Tag      = 3
-)
-
 // Decode returns the decoded form of src. The returned slice may be a sub-
 // slice of dst if dst was large enough to hold the entire decoded block.
 // Otherwise, a newly allocated slice will be returned.
-//
+// It is valid to pass a nil dst.
-// The dst and src must not overlap. It is valid to pass a nil dst.
 func Decode(dst, src []byte) ([]byte, error) {
 	dLen, s, err := decodedLen(src)
 	if err != nil {
 		return nil, err
 	}
-	if dLen <= len(dst) {
+	if len(dst) < dLen {
-		dst = dst[:dLen]
-	} else {
 		dst = make([]byte, dLen)
 	}
-	switch decode(dst, src[s:]) {
+
-	case 0:
+	var d, offset, length int
-		return dst, nil
+	for s < len(src) {
-	case decodeErrCodeUnsupportedLiteralLength:
+		switch src[s] & 0x03 {
-		return nil, errUnsupportedLiteralLength
+		case tagLiteral:
-	case decodeErrCodeUnsupportedCopy4Tag:
+			x := uint(src[s] >> 2)
-		return nil, errUnsupportedCopy4Tag
+			switch {
+			case x < 60:
+				s++
+			case x == 60:
+				s += 2
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-1])
+			case x == 61:
+				s += 3
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-2]) | uint(src[s-1])<<8
+			case x == 62:
+				s += 4
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-3]) | uint(src[s-2])<<8 | uint(src[s-1])<<16
+			case x == 63:
+				s += 5
+				if s > len(src) {
+					return nil, ErrCorrupt
+				}
+				x = uint(src[s-4]) | uint(src[s-3])<<8 | uint(src[s-2])<<16 | uint(src[s-1])<<24
+			}
+			length = int(x + 1)
+			if length <= 0 {
+				return nil, errors.New("snappy: unsupported literal length")
+			}
+			if length > len(dst)-d || length > len(src)-s {
+				return nil, ErrCorrupt
+			}
+			copy(dst[d:], src[s:s+length])
+			d += length
+			s += length
+			continue
+
+		case tagCopy1:
+			s += 2
+			if s > len(src) {
+				return nil, ErrCorrupt
+			}
+			length = 4 + int(src[s-2])>>2&0x7
+			offset = int(src[s-2])&0xe0<<3 | int(src[s-1])
+
+		case tagCopy2:
+			s += 3
+			if s > len(src) {
+				return nil, ErrCorrupt
+			}
+			length = 1 + int(src[s-3])>>2
+			offset = int(src[s-2]) | int(src[s-1])<<8
+
+		case tagCopy4:
+			return nil, errors.New("snappy: unsupported COPY_4 tag")
+		}
+
+		end := d + length
+		if offset > d || end > len(dst) {
+			return nil, ErrCorrupt
+		}
+		for ; d < end; d++ {
+			dst[d] = dst[d-offset]
+		}
 	}
-	return nil, ErrCorrupt
+	if d != dLen {
+		return nil, ErrCorrupt
+	}
+	return dst[:d], nil
 }
 
 // NewReader returns a new Reader that decompresses from r, using the framing
@@ -81,12 +138,12 @@ func Decode(dst, src []byte) ([]byte, error) {
 func NewReader(r io.Reader) *Reader {
 	return &Reader{
 		r:       r,
-		decoded: make([]byte, maxBlockSize),
+		decoded: make([]byte, maxUncompressedChunkLen),
-		buf:     make([]byte, maxEncodedLenOfMaxBlockSize+checksumSize),
+		buf:     make([]byte, MaxEncodedLen(maxUncompressedChunkLen)+checksumSize),
 	}
 }
 
-// Reader is an io.Reader that can read Snappy-compressed bytes.
+// Reader is an io.Reader than can read Snappy-compressed bytes.
 type Reader struct {
 	r       io.Reader
 	err     error

Godeps/_workspace/src/github.com/golang/snappy/encode.go

@@ -0,0 +1,254 @@
+// Copyright 2011 The Snappy-Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package snappy
+
+import (
+	"encoding/binary"
+	"io"
+)
+
+// We limit how far copy back-references can go, the same as the C++ code.
+const maxOffset = 1 << 15
+
+// emitLiteral writes a literal chunk and returns the number of bytes written.
+func emitLiteral(dst, lit []byte) int {
+	i, n := 0, uint(len(lit)-1)
+	switch {
+	case n < 60:
+		dst[0] = uint8(n)<<2 | tagLiteral
+		i = 1
+	case n < 1<<8:
+		dst[0] = 60<<2 | tagLiteral
+		dst[1] = uint8(n)
+		i = 2
+	case n < 1<<16:
+		dst[0] = 61<<2 | tagLiteral
+		dst[1] = uint8(n)
+		dst[2] = uint8(n >> 8)
+		i = 3
+	case n < 1<<24:
+		dst[0] = 62<<2 | tagLiteral
+		dst[1] = uint8(n)
+		dst[2] = uint8(n >> 8)
+		dst[3] = uint8(n >> 16)
+		i = 4
+	case int64(n) < 1<<32:
+		dst[0] = 63<<2 | tagLiteral
+		dst[1] = uint8(n)
+		dst[2] = uint8(n >> 8)
+		dst[3] = uint8(n >> 16)
+		dst[4] = uint8(n >> 24)
+		i = 5
+	default:
+		panic("snappy: source buffer is too long")
+	}
+	if copy(dst[i:], lit) != len(lit) {
+		panic("snappy: destination buffer is too short")
+	}
+	return i + len(lit)
+}
+
+// emitCopy writes a copy chunk and returns the number of bytes written.
+func emitCopy(dst []byte, offset, length int) int {
+	i := 0
+	for length > 0 {
+		x := length - 4
+		if 0 <= x && x < 1<<3 && offset < 1<<11 {
+			dst[i+0] = uint8(offset>>8)&0x07<<5 | uint8(x)<<2 | tagCopy1
+			dst[i+1] = uint8(offset)
+			i += 2
+			break
+		}
+
+		x = length
+		if x > 1<<6 {
+			x = 1 << 6
+		}
+		dst[i+0] = uint8(x-1)<<2 | tagCopy2
+		dst[i+1] = uint8(offset)
+		dst[i+2] = uint8(offset >> 8)
+		i += 3
+		length -= x
+	}
+	return i
+}
+
+// Encode returns the encoded form of src. The returned slice may be a sub-
+// slice of dst if dst was large enough to hold the entire encoded block.
+// Otherwise, a newly allocated slice will be returned.
+// It is valid to pass a nil dst.
+func Encode(dst, src []byte) []byte {
+	if n := MaxEncodedLen(len(src)); len(dst) < n {
+		dst = make([]byte, n)
+	}
+
+	// The block starts with the varint-encoded length of the decompressed bytes.
+	d := binary.PutUvarint(dst, uint64(len(src)))
+
+	// Return early if src is short.
+	if len(src) <= 4 {
+		if len(src) != 0 {
+			d += emitLiteral(dst[d:], src)
+		}
+		return dst[:d]
+	}
+
+	// Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive.
+	const maxTableSize = 1 << 14
+	shift, tableSize := uint(32-8), 1<<8
+	for tableSize < maxTableSize && tableSize < len(src) {
+		shift--
+		tableSize *= 2
+	}
+	var table [maxTableSize]int
+
+	// Iterate over the source bytes.
+	var (
+		s   int // The iterator position.
+		t   int // The last position with the same hash as s.
+		lit int // The start position of any pending literal bytes.
+	)
+	for s+3 < len(src) {
+		// Update the hash table.
+		b0, b1, b2, b3 := src[s], src[s+1], src[s+2], src[s+3]
+		h := uint32(b0) | uint32(b1)<<8 | uint32(b2)<<16 | uint32(b3)<<24
+		p := &table[(h*0x1e35a7bd)>>shift]
+		// We need to to store values in [-1, inf) in table. To save
+		// some initialization time, (re)use the table's zero value
+		// and shift the values against this zero: add 1 on writes,
+		// subtract 1 on reads.
+		t, *p = *p-1, s+1
+		// If t is invalid or src[s:s+4] differs from src[t:t+4], accumulate a literal byte.
+		if t < 0 || s-t >= maxOffset || b0 != src[t] || b1 != src[t+1] || b2 != src[t+2] || b3 != src[t+3] {
+			s++
+			continue
+		}
+		// Otherwise, we have a match. First, emit any pending literal bytes.
+		if lit != s {
+			d += emitLiteral(dst[d:], src[lit:s])
+		}
+		// Extend the match to be as long as possible.
+		s0 := s
+		s, t = s+4, t+4
+		for s < len(src) && src[s] == src[t] {
+			s++
+			t++
+		}
+		// Emit the copied bytes.
+		d += emitCopy(dst[d:], s-t, s-s0)
+		lit = s
+	}
+
+	// Emit any final pending literal bytes and return.
+	if lit != len(src) {
+		d += emitLiteral(dst[d:], src[lit:])
+	}
+	return dst[:d]
+}
+
+// MaxEncodedLen returns the maximum length of a snappy block, given its
+// uncompressed length.
+func MaxEncodedLen(srcLen int) int {
+	// Compressed data can be defined as:
+	//    compressed := item* literal*
+	//    item       := literal* copy
+	//
+	// The trailing literal sequence has a space blowup of at most 62/60
+	// since a literal of length 60 needs one tag byte + one extra byte
+	// for length information.
+	//
+	// Item blowup is trickier to measure. Suppose the "copy" op copies
+	// 4 bytes of data. Because of a special check in the encoding code,
+	// we produce a 4-byte copy only if the offset is < 65536. Therefore
+	// the copy op takes 3 bytes to encode, and this type of item leads
+	// to at most the 62/60 blowup for representing literals.
+	//
+	// Suppose the "copy" op copies 5 bytes of data. If the offset is big
+	// enough, it will take 5 bytes to encode the copy op. Therefore the
+	// worst case here is a one-byte literal followed by a five-byte copy.
+	// That is, 6 bytes of input turn into 7 bytes of "compressed" data.
+	//
+	// This last factor dominates the blowup, so the final estimate is:
+	return 32 + srcLen + srcLen/6
+}
+
+// NewWriter returns a new Writer that compresses to w, using the framing
+// format described at
+// https://github.com/google/snappy/blob/master/framing_format.txt
+func NewWriter(w io.Writer) *Writer {
+	return &Writer{
+		w:   w,
+		enc: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)),
+	}
+}
+
+// Writer is an io.Writer than can write Snappy-compressed bytes.
+type Writer struct {
+	w           io.Writer
+	err         error
+	enc         []byte
+	buf         [checksumSize + chunkHeaderSize]byte
+	wroteHeader bool
+}
+
+// Reset discards the writer's state and switches the Snappy writer to write to
+// w. This permits reusing a Writer rather than allocating a new one.
+func (w *Writer) Reset(writer io.Writer) {
+	w.w = writer
+	w.err = nil
+	w.wroteHeader = false
+}
+
+// Write satisfies the io.Writer interface.
+func (w *Writer) Write(p []byte) (n int, errRet error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+	if !w.wroteHeader {
+		copy(w.enc, magicChunk)
+		if _, err := w.w.Write(w.enc[:len(magicChunk)]); err != nil {
+			w.err = err
+			return n, err
+		}
+		w.wroteHeader = true
+	}
+	for len(p) > 0 {
+		var uncompressed []byte
+		if len(p) > maxUncompressedChunkLen {
+			uncompressed, p = p[:maxUncompressedChunkLen], p[maxUncompressedChunkLen:]
+		} else {
+			uncompressed, p = p, nil
+		}
+		checksum := crc(uncompressed)
+
+		// Compress the buffer, discarding the result if the improvement
+		// isn't at least 12.5%.
+		chunkType := uint8(chunkTypeCompressedData)
+		chunkBody := Encode(w.enc, uncompressed)
+		if len(chunkBody) >= len(uncompressed)-len(uncompressed)/8 {
+			chunkType, chunkBody = chunkTypeUncompressedData, uncompressed
+		}
+
+		chunkLen := 4 + len(chunkBody)
+		w.buf[0] = chunkType
+		w.buf[1] = uint8(chunkLen >> 0)
+		w.buf[2] = uint8(chunkLen >> 8)
+		w.buf[3] = uint8(chunkLen >> 16)
+		w.buf[4] = uint8(checksum >> 0)
+		w.buf[5] = uint8(checksum >> 8)
+		w.buf[6] = uint8(checksum >> 16)
+		w.buf[7] = uint8(checksum >> 24)
+		if _, err := w.w.Write(w.buf[:]); err != nil {
+			w.err = err
+			return n, err
+		}
+		if _, err := w.w.Write(chunkBody); err != nil {
+			w.err = err
+			return n, err
+		}
+		n += len(uncompressed)
+	}
+	return n, nil
+}

Godeps/_workspace/src/github.com/golang/snappy/snappy.go

@@ -6,7 +6,7 @@
 // It aims for very high speeds and reasonable compression.
 //
 // The C++ snappy implementation is at https://github.com/google/snappy
-package snappy // import "github.com/golang/snappy"
+package snappy
 
 import (
 	"hash/crc32"
@@ -46,25 +46,9 @@ const (
 	chunkHeaderSize = 4
 	magicChunk      = "\xff\x06\x00\x00" + magicBody
 	magicBody       = "sNaPpY"
-
-	// maxBlockSize is the maximum size of the input to encodeBlock. It is not
-	// part of the wire format per se, but some parts of the encoder assume
-	// that an offset fits into a uint16.
-	//
-	// Also, for the framing format (Writer type instead of Encode function),
 	// https://github.com/google/snappy/blob/master/framing_format.txt says
-	// that "the uncompressed data in a chunk must be no longer than 65536
+	// that "the uncompressed data in a chunk must be no longer than 65536 bytes".
-	// bytes".
+	maxUncompressedChunkLen = 65536
-	maxBlockSize = 65536
-
-	// maxEncodedLenOfMaxBlockSize equals MaxEncodedLen(maxBlockSize), but is
-	// hard coded to be a const instead of a variable, so that obufLen can also
-	// be a const. Their equivalence is confirmed by
-	// TestMaxEncodedLenOfMaxBlockSize.
-	maxEncodedLenOfMaxBlockSize = 76490
-
-	obufHeaderLen = len(magicChunk) + checksumSize + chunkHeaderSize
-	obufLen       = obufHeaderLen + maxEncodedLenOfMaxBlockSize
 )
 
 const (

Godeps/_workspace/src/github.com/juju/ratelimit/ratelimit.go

@@ -8,10 +8,10 @@
 package ratelimit
 
 import (
-	"math"
 	"strconv"
 	"sync"
 	"time"
+	"math"
 )
 
 // Bucket represents a token bucket that fills at a predetermined rate.
@@ -171,30 +171,6 @@ func (tb *Bucket) takeAvailable(now time.Time, count int64) int64 {
 	return count
 }
 
-// Available returns the number of available tokens. It will be negative
-// when there are consumers waiting for tokens. Note that if this
-// returns greater than zero, it does not guarantee that calls that take
-// tokens from the buffer will succeed, as the number of available
-// tokens could have changed in the meantime. This method is intended
-// primarily for metrics reporting and debugging.
-func (tb *Bucket) Available() int64 {
-	return tb.available(time.Now())
-}
-
-// available is the internal version of available - it takes the current time as
-// an argument to enable easy testing.
-func (tb *Bucket) available(now time.Time) int64 {
-	tb.mu.Lock()
-	defer tb.mu.Unlock()
-	tb.adjust(now)
-	return tb.avail
-}
-
-// Capacity returns the capacity that the bucket was created with.
-func (tb *Bucket) Capacity() int64 {
-	return tb.capacity
-}
-
 // Rate returns the fill rate of the bucket, in tokens per second.
 func (tb *Bucket) Rate() float64 {
 	return 1e9 * float64(tb.quantum) / float64(tb.fillInterval)

Godeps/_workspace/src/github.com/kardianos/osext/osext.go

@@ -3,7 +3,7 @@
 // license that can be found in the LICENSE file.
 
 // Extensions to the standard "os" package.
-package osext // import "github.com/kardianos/osext"
+package osext
 
 import "path/filepath"
 

Godeps/_workspace/src/github.com/rcrowley/go-metrics/.gitignore

@@ -0,0 +1,9 @@
+*.[68]
+*.a
+*.out
+*.swp
+_obj
+_testmain.go
+cmd/metrics-bench/metrics-bench
+cmd/metrics-example/metrics-example
+cmd/never-read/never-read

Godeps/_workspace/src/github.com/rcrowley/go-metrics/.travis.yml

@@ -0,0 +1,13 @@
+language: go
+
+go:
+    - 1.2
+    - 1.3
+    - 1.4
+
+script:
+    - ./validate.sh
+
+# this should give us faster builds according to 
+# http://docs.travis-ci.com/user/migrating-from-legacy/
+sudo: false

Godeps/_workspace/src/github.com/rcrowley/go-metrics/README.md

@@ -39,7 +39,7 @@ t.Update(47)
 Periodically log every metric in human-readable form to standard error:
 
 ```go
-go metrics.Log(metrics.DefaultRegistry, 5 * time.Second, log.New(os.Stderr, "metrics: ", log.Lmicroseconds))
+go metrics.Log(metrics.DefaultRegistry, 60e9, log.New(os.Stderr, "metrics: ", log.Lmicroseconds))
 ```
 
 Periodically log every metric in slightly-more-parseable form to syslog:
@@ -103,19 +103,6 @@ import "github.com/rcrowley/go-metrics/stathat"
 go stathat.Stathat(metrics.DefaultRegistry, 10e9, "example@example.com")
 ```
 
-Maintain all metrics along with expvars at `/debug/metrics`:
-
-This uses the same mechanism as [the official expvar](http://golang.org/pkg/expvar/)
-but exposed under `/debug/metrics`, which shows a json representation of all your usual expvars
-as well as all your go-metrics.
-
-
-```go
-import "github.com/rcrowley/go-metrics/exp"
-
-exp.Exp(metrics.DefaultRegistry)
-```
-
 Installation
 ------------
 

Godeps/_workspace/src/github.com/rcrowley/go-metrics/librato/librato.go

@@ -23,7 +23,6 @@ func translateTimerAttributes(d time.Duration) (attrs map[string]interface{}) {
 
 type Reporter struct {
 	Email, Token    string
-	Namespace       string
 	Source          string
 	Interval        time.Duration
 	Registry        metrics.Registry
@@ -33,7 +32,7 @@ type Reporter struct {
 }
 
 func NewReporter(r metrics.Registry, d time.Duration, e string, t string, s string, p []float64, u time.Duration) *Reporter {
-	return &Reporter{e, t, "", s, d, r, p, translateTimerAttributes(u), int64(d / time.Second)}
+	return &Reporter{e, t, s, d, r, p, translateTimerAttributes(u), int64(d / time.Second)}
 }
 
 func Librato(r metrics.Registry, d time.Duration, e string, t string, s string, p []float64, u time.Duration) {
@@ -89,9 +88,6 @@ func (self *Reporter) BuildRequest(now time.Time, r metrics.Registry) (snapshot
 	snapshot.Counters = make([]Measurement, 0)
 	histogramGaugeCount := 1 + len(self.Percentiles)
 	r.Each(func(name string, metric interface{}) {
-		if self.Namespace != "" {
-			name = fmt.Sprintf("%s.%s", self.Namespace, name)
-		}
 		measurement := Measurement{}
 		measurement[Period] = self.Interval.Seconds()
 		switch m := metric.(type) {

Godeps/_workspace/src/github.com/rcrowley/go-metrics/log.go

@@ -5,17 +5,10 @@ import (
 	"time"
 )
 
-func Log(r Registry, freq time.Duration, l *log.Logger) {
-	LogScaled(r, freq, time.Nanosecond, l)
-}
-
 // Output each metric in the given registry periodically using the given
-// logger. Print timings in `scale` units (eg time.Millisecond) rather than nanos.
+// logger.
-func LogScaled(r Registry, freq time.Duration, scale time.Duration, l *log.Logger) {
+func Log(r Registry, d time.Duration, l *log.Logger) {
-	du := float64(scale)
+	for _ = range time.Tick(d) {
-	duSuffix := scale.String()[1:]
-
-	for _ = range time.Tick(freq) {
 		r.Each(func(name string, i interface{}) {
 			switch metric := i.(type) {
 			case Counter:
@@ -58,15 +51,15 @@ func LogScaled(r Registry, freq time.Duration, scale time.Duration, l *log.Logge
 				ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999})
 				l.Printf("timer %s\n", name)
 				l.Printf("  count:       %9d\n", t.Count())
-				l.Printf("  min:         %12.2f%s\n", float64(t.Min())/du, duSuffix)
+				l.Printf("  min:         %9d\n", t.Min())
-				l.Printf("  max:         %12.2f%s\n", float64(t.Max())/du, duSuffix)
+				l.Printf("  max:         %9d\n", t.Max())
-				l.Printf("  mean:        %12.2f%s\n", t.Mean()/du, duSuffix)
+				l.Printf("  mean:        %12.2f\n", t.Mean())
-				l.Printf("  stddev:      %12.2f%s\n", t.StdDev()/du, duSuffix)
+				l.Printf("  stddev:      %12.2f\n", t.StdDev())
-				l.Printf("  median:      %12.2f%s\n", ps[0]/du, duSuffix)
+				l.Printf("  median:      %12.2f\n", ps[0])
-				l.Printf("  75%%:         %12.2f%s\n", ps[1]/du, duSuffix)
+				l.Printf("  75%%:         %12.2f\n", ps[1])
-				l.Printf("  95%%:         %12.2f%s\n", ps[2]/du, duSuffix)
+				l.Printf("  95%%:         %12.2f\n", ps[2])
-				l.Printf("  99%%:         %12.2f%s\n", ps[3]/du, duSuffix)
+				l.Printf("  99%%:         %12.2f\n", ps[3])
-				l.Printf("  99.9%%:       %12.2f%s\n", ps[4]/du, duSuffix)
+				l.Printf("  99.9%%:       %12.2f\n", ps[4])
 				l.Printf("  1-min rate:  %12.2f\n", t.Rate1())
 				l.Printf("  5-min rate:  %12.2f\n", t.Rate5())
 				l.Printf("  15-min rate: %12.2f\n", t.Rate15())

Godeps/_workspace/src/github.com/rcrowley/go-metrics/registry.go

@@ -157,13 +157,6 @@ func NewPrefixedRegistry(prefix string) Registry {
 	}
 }
 
-func NewPrefixedChildRegistry(parent Registry, prefix string) Registry {
-	return &PrefixedRegistry{
-		underlying: parent,
-		prefix:     prefix,
-	}
-}
-
 // Call the given function for each registered metric.
 func (r *PrefixedRegistry) Each(fn func(string, interface{})) {
 	r.underlying.Each(fn)

Godeps/_workspace/src/github.com/rcrowley/go-metrics/runtime.go

@@ -2,7 +2,6 @@ package metrics
 
 import (
 	"runtime"
-	"runtime/pprof"
 	"time"
 )
 
@@ -10,37 +9,35 @@ var (
 	memStats       runtime.MemStats
 	runtimeMetrics struct {
 		MemStats struct {
-			Alloc         Gauge
+			Alloc        Gauge
-			BuckHashSys   Gauge
+			BuckHashSys  Gauge
-			DebugGC       Gauge
+			DebugGC      Gauge
-			EnableGC      Gauge
+			EnableGC     Gauge
-			Frees         Gauge
+			Frees        Gauge
-			HeapAlloc     Gauge
+			HeapAlloc    Gauge
-			HeapIdle      Gauge
+			HeapIdle     Gauge
-			HeapInuse     Gauge
+			HeapInuse    Gauge
-			HeapObjects   Gauge
+			HeapObjects  Gauge
-			HeapReleased  Gauge
+			HeapReleased Gauge
-			HeapSys       Gauge
+			HeapSys      Gauge
-			LastGC        Gauge
+			LastGC       Gauge
-			Lookups       Gauge
+			Lookups      Gauge
-			Mallocs       Gauge
+			Mallocs      Gauge
-			MCacheInuse   Gauge
+			MCacheInuse  Gauge
-			MCacheSys     Gauge
+			MCacheSys    Gauge
-			MSpanInuse    Gauge
+			MSpanInuse   Gauge
-			MSpanSys      Gauge
+			MSpanSys     Gauge
-			NextGC        Gauge
+			NextGC       Gauge
-			NumGC         Gauge
+			NumGC        Gauge
-			GCCPUFraction GaugeFloat64
+			PauseNs      Histogram
-			PauseNs       Histogram
+			PauseTotalNs Gauge
-			PauseTotalNs  Gauge
+			StackInuse   Gauge
-			StackInuse    Gauge
+			StackSys     Gauge
-			StackSys      Gauge
+			Sys          Gauge
-			Sys           Gauge
+			TotalAlloc   Gauge
-			TotalAlloc    Gauge
 		}
 		NumCgoCall   Gauge
 		NumGoroutine Gauge
-		NumThread    Gauge
 		ReadMemStats Timer
 	}
 	frees       uint64
@@ -48,8 +45,6 @@ var (
 	mallocs     uint64
 	numGC       uint32
 	numCgoCalls int64
-
-	threadCreateProfile = pprof.Lookup("threadcreate")
 )
 
 // Capture new values for the Go runtime statistics exported in
@@ -102,7 +97,6 @@ func CaptureRuntimeMemStatsOnce(r Registry) {
 	runtimeMetrics.MemStats.MSpanSys.Update(int64(memStats.MSpanSys))
 	runtimeMetrics.MemStats.NextGC.Update(int64(memStats.NextGC))
 	runtimeMetrics.MemStats.NumGC.Update(int64(memStats.NumGC - numGC))
-	runtimeMetrics.MemStats.GCCPUFraction.Update(gcCPUFraction(&memStats))
 
 	// <https://code.google.com/p/go/source/browse/src/pkg/runtime/mgc0.c>
 	i := numGC % uint32(len(memStats.PauseNs))
@@ -138,8 +132,6 @@ func CaptureRuntimeMemStatsOnce(r Registry) {
 	numCgoCalls = currentNumCgoCalls
 
 	runtimeMetrics.NumGoroutine.Update(int64(runtime.NumGoroutine()))
-
-	runtimeMetrics.NumThread.Update(int64(threadCreateProfile.Count()))
 }
 
 // Register runtimeMetrics for the Go runtime statistics exported in runtime and
@@ -166,7 +158,6 @@ func RegisterRuntimeMemStats(r Registry) {
 	runtimeMetrics.MemStats.MSpanSys = NewGauge()
 	runtimeMetrics.MemStats.NextGC = NewGauge()
 	runtimeMetrics.MemStats.NumGC = NewGauge()
-	runtimeMetrics.MemStats.GCCPUFraction = NewGaugeFloat64()
 	runtimeMetrics.MemStats.PauseNs = NewHistogram(NewExpDecaySample(1028, 0.015))
 	runtimeMetrics.MemStats.PauseTotalNs = NewGauge()
 	runtimeMetrics.MemStats.StackInuse = NewGauge()
@@ -175,7 +166,6 @@ func RegisterRuntimeMemStats(r Registry) {
 	runtimeMetrics.MemStats.TotalAlloc = NewGauge()
 	runtimeMetrics.NumCgoCall = NewGauge()
 	runtimeMetrics.NumGoroutine = NewGauge()
-	runtimeMetrics.NumThread = NewGauge()
 	runtimeMetrics.ReadMemStats = NewTimer()
 
 	r.Register("runtime.MemStats.Alloc", runtimeMetrics.MemStats.Alloc)
@@ -198,7 +188,6 @@ func RegisterRuntimeMemStats(r Registry) {
 	r.Register("runtime.MemStats.MSpanSys", runtimeMetrics.MemStats.MSpanSys)
 	r.Register("runtime.MemStats.NextGC", runtimeMetrics.MemStats.NextGC)
 	r.Register("runtime.MemStats.NumGC", runtimeMetrics.MemStats.NumGC)
-	r.Register("runtime.MemStats.GCCPUFraction", runtimeMetrics.MemStats.GCCPUFraction)
 	r.Register("runtime.MemStats.PauseNs", runtimeMetrics.MemStats.PauseNs)
 	r.Register("runtime.MemStats.PauseTotalNs", runtimeMetrics.MemStats.PauseTotalNs)
 	r.Register("runtime.MemStats.StackInuse", runtimeMetrics.MemStats.StackInuse)
@@ -207,6 +196,5 @@ func RegisterRuntimeMemStats(r Registry) {
 	r.Register("runtime.MemStats.TotalAlloc", runtimeMetrics.MemStats.TotalAlloc)
 	r.Register("runtime.NumCgoCall", runtimeMetrics.NumCgoCall)
 	r.Register("runtime.NumGoroutine", runtimeMetrics.NumGoroutine)
-	r.Register("runtime.NumThread", runtimeMetrics.NumThread)
 	r.Register("runtime.ReadMemStats", runtimeMetrics.ReadMemStats)
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch.go

@@ -12,10 +12,8 @@ import (
 
 	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/memdb"
-	"github.com/syndtr/goleveldb/leveldb/storage"
 )
 
-// ErrBatchCorrupted records reason of batch corruption.
 type ErrBatchCorrupted struct {
 	Reason string
 }
@@ -25,7 +23,7 @@ func (e *ErrBatchCorrupted) Error() string {
 }
 
 func newErrBatchCorrupted(reason string) error {
-	return errors.NewErrCorrupted(storage.FileDesc{}, &ErrBatchCorrupted{reason})
+	return errors.NewErrCorrupted(nil, &ErrBatchCorrupted{reason})
 }
 
 const (
@@ -33,7 +31,6 @@ const (
 	batchGrowRec = 3000
 )
 
-// BatchReplay wraps basic batch operations.
 type BatchReplay interface {
 	Put(key, value []byte)
 	Delete(key []byte)
@@ -70,20 +67,20 @@ func (b *Batch) grow(n int) {
 	}
 }
 
-func (b *Batch) appendRec(kt keyType, key, value []byte) {
+func (b *Batch) appendRec(kt kType, key, value []byte) {
 	n := 1 + binary.MaxVarintLen32 + len(key)
-	if kt == keyTypeVal {
+	if kt == ktVal {
 		n += binary.MaxVarintLen32 + len(value)
 	}
 	b.grow(n)
 	off := len(b.data)
 	data := b.data[:off+n]
 	data[off] = byte(kt)
-	off++
+	off += 1
 	off += binary.PutUvarint(data[off:], uint64(len(key)))
 	copy(data[off:], key)
 	off += len(key)
-	if kt == keyTypeVal {
+	if kt == ktVal {
 		off += binary.PutUvarint(data[off:], uint64(len(value)))
 		copy(data[off:], value)
 		off += len(value)
@@ -97,13 +94,13 @@ func (b *Batch) appendRec(kt keyType, key, value []byte) {
 // Put appends 'put operation' of the given key/value pair to the batch.
 // It is safe to modify the contents of the argument after Put returns.
 func (b *Batch) Put(key, value []byte) {
-	b.appendRec(keyTypeVal, key, value)
+	b.appendRec(ktVal, key, value)
 }
 
 // Delete appends 'delete operation' of the given key to the batch.
 // It is safe to modify the contents of the argument after Delete returns.
 func (b *Batch) Delete(key []byte) {
-	b.appendRec(keyTypeDel, key, nil)
+	b.appendRec(ktDel, key, nil)
 }
 
 // Dump dumps batch contents. The returned slice can be loaded into the
@@ -124,14 +121,13 @@ func (b *Batch) Load(data []byte) error {
 
 // Replay replays batch contents.
 func (b *Batch) Replay(r BatchReplay) error {
-	return b.decodeRec(func(i int, kt keyType, key, value []byte) error {
+	return b.decodeRec(func(i int, kt kType, key, value []byte) {
 		switch kt {
-		case keyTypeVal:
+		case ktVal:
 			r.Put(key, value)
-		case keyTypeDel:
+		case ktDel:
 			r.Delete(key)
 		}
-		return nil
 	})
 }
 
@@ -197,19 +193,18 @@ func (b *Batch) decode(prevSeq uint64, data []byte) error {
 	return nil
 }
 
-func (b *Batch) decodeRec(f func(i int, kt keyType, key, value []byte) error) error {
+func (b *Batch) decodeRec(f func(i int, kt kType, key, value []byte)) (err error) {
 	off := batchHdrLen
 	for i := 0; i < b.rLen; i++ {
 		if off >= len(b.data) {
 			return newErrBatchCorrupted("invalid records length")
 		}
 
-		kt := keyType(b.data[off])
+		kt := kType(b.data[off])
-		if kt > keyTypeVal {
+		if kt > ktVal {
-			panic(kt)
 			return newErrBatchCorrupted("bad record: invalid type")
 		}
-		off++
+		off += 1
 
 		x, n := binary.Uvarint(b.data[off:])
 		off += n
@@ -219,7 +214,7 @@ func (b *Batch) decodeRec(f func(i int, kt keyType, key, value []byte) error) er
 		key := b.data[off : off+int(x)]
 		off += int(x)
 		var value []byte
-		if kt == keyTypeVal {
+		if kt == ktVal {
 			x, n := binary.Uvarint(b.data[off:])
 			off += n
 			if n <= 0 || off+int(x) > len(b.data) {
@@ -229,19 +224,16 @@ func (b *Batch) decodeRec(f func(i int, kt keyType, key, value []byte) error) er
 			off += int(x)
 		}
 
-		if err := f(i, kt, key, value); err != nil {
+		f(i, kt, key, value)
-			return err
-		}
 	}
 
 	return nil
 }
 
 func (b *Batch) memReplay(to *memdb.DB) error {
-	var ikScratch []byte
+	return b.decodeRec(func(i int, kt kType, key, value []byte) {
-	return b.decodeRec(func(i int, kt keyType, key, value []byte) error {
+		ikey := newIkey(key, b.seq+uint64(i), kt)
-		ikScratch = makeInternalKey(ikScratch, key, b.seq+uint64(i), kt)
+		to.Put(ikey, value)
-		return to.Put(ikScratch, value)
 	})
 }
 
@@ -253,9 +245,8 @@ func (b *Batch) memDecodeAndReplay(prevSeq uint64, data []byte, to *memdb.DB) er
 }
 
 func (b *Batch) revertMemReplay(to *memdb.DB) error {
-	var ikScratch []byte
+	return b.decodeRec(func(i int, kt kType, key, value []byte) {
-	return b.decodeRec(func(i int, kt keyType, key, value []byte) error {
+		ikey := newIkey(key, b.seq+uint64(i), kt)
-		ikScratch := makeInternalKey(ikScratch, key, b.seq+uint64(i), kt)
+		to.Delete(ikey)
-		return to.Delete(ikScratch)
 	})
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache.go

@@ -47,21 +47,17 @@ type Cacher interface {
 // so the the Release method will be called once object is released.
 type Value interface{}
 
-// NamespaceGetter provides convenient wrapper for namespace.
+type CacheGetter struct {
-type NamespaceGetter struct {
 	Cache *Cache
 	NS    uint64
 }
 
-// Get simply calls Cache.Get() method.
+func (g *CacheGetter) Get(key uint64, setFunc func() (size int, value Value)) *Handle {
-func (g *NamespaceGetter) Get(key uint64, setFunc func() (size int, value Value)) *Handle {
 	return g.Cache.Get(g.NS, key, setFunc)
 }
 
 // The hash tables implementation is based on:
-// "Dynamic-Sized Nonblocking Hash Tables", by Yujie Liu,
+// "Dynamic-Sized Nonblocking Hash Tables", by Yujie Liu, Kunlong Zhang, and Michael Spear. ACM Symposium on Principles of Distributed Computing, Jul 2014.
-// Kunlong Zhang, and Michael Spear.
-// ACM Symposium on Principles of Distributed Computing, Jul 2014.
 
 const (
 	mInitialSize           = 1 << 4
@@ -614,12 +610,10 @@ func (n *Node) unrefLocked() {
 	}
 }
 
-// Handle is a 'cache handle' of a 'cache node'.
 type Handle struct {
 	n unsafe.Pointer // *Node
 }
 
-// Value returns the value of the 'cache node'.
 func (h *Handle) Value() Value {
 	n := (*Node)(atomic.LoadPointer(&h.n))
 	if n != nil {
@@ -628,8 +622,6 @@ func (h *Handle) Value() Value {
 	return nil
 }
 
-// Release releases this 'cache handle'.
-// It is safe to call release multiple times.
 func (h *Handle) Release() {
 	nPtr := atomic.LoadPointer(&h.n)
 	if nPtr != nil && atomic.CompareAndSwapPointer(&h.n, nPtr, nil) {

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer.go

@@ -33,9 +33,9 @@ func (icmp *iComparer) Name() string {
 }
 
 func (icmp *iComparer) Compare(a, b []byte) int {
-	x := icmp.ucmp.Compare(internalKey(a).ukey(), internalKey(b).ukey())
+	x := icmp.ucmp.Compare(iKey(a).ukey(), iKey(b).ukey())
 	if x == 0 {
-		if m, n := internalKey(a).num(), internalKey(b).num(); m > n {
+		if m, n := iKey(a).num(), iKey(b).num(); m > n {
 			x = -1
 		} else if m < n {
 			x = 1
@@ -45,13 +45,13 @@ func (icmp *iComparer) Compare(a, b []byte) int {
 }
 
 func (icmp *iComparer) Separator(dst, a, b []byte) []byte {
-	ua, ub := internalKey(a).ukey(), internalKey(b).ukey()
+	ua, ub := iKey(a).ukey(), iKey(b).ukey()
 	dst = icmp.ucmp.Separator(dst, ua, ub)
 	if dst == nil {
 		return nil
 	}
 	if len(dst) < len(ua) && icmp.uCompare(ua, dst) < 0 {
-		dst = append(dst, keyMaxNumBytes...)
+		dst = append(dst, kMaxNumBytes...)
 	} else {
 		// Did not close possibilities that n maybe longer than len(ub).
 		dst = append(dst, a[len(a)-8:]...)
@@ -60,13 +60,13 @@ func (icmp *iComparer) Separator(dst, a, b []byte) []byte {
 }
 
 func (icmp *iComparer) Successor(dst, b []byte) []byte {
-	ub := internalKey(b).ukey()
+	ub := iKey(b).ukey()
 	dst = icmp.ucmp.Successor(dst, ub)
 	if dst == nil {
 		return nil
 	}
 	if len(dst) < len(ub) && icmp.uCompare(ub, dst) < 0 {
-		dst = append(dst, keyMaxNumBytes...)
+		dst = append(dst, kMaxNumBytes...)
 	} else {
 		// Did not close possibilities that n maybe longer than len(ub).
 		dst = append(dst, b[len(b)-8:]...)

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go

@@ -36,14 +36,14 @@ type DB struct {
 	s *session
 
 	// MemDB.
-	memMu           sync.RWMutex
+	memMu             sync.RWMutex
-	memPool         chan *memdb.DB
+	memPool           chan *memdb.DB
-	mem, frozenMem  *memDB
+	mem, frozenMem    *memDB
-	journal         *journal.Writer
+	journal           *journal.Writer
-	journalWriter   storage.Writer
+	journalWriter     storage.Writer
-	journalFd       storage.FileDesc
+	journalFile       storage.File
-	frozenJournalFd storage.FileDesc
+	frozenJournalFile storage.File
-	frozenSeq       uint64
+	frozenSeq         uint64
 
 	// Snapshot.
 	snapsMu   sync.Mutex
@@ -61,10 +61,8 @@ type DB struct {
 	writeDelayN  int
 	journalC     chan *Batch
 	journalAckC  chan error
-	tr           *Transaction
 
 	// Compaction.
-	compCommitLk     sync.Mutex
 	tcompCmdC        chan cCmd
 	tcompPauseC      chan chan<- struct{}
 	mcompCmdC        chan cCmd
@@ -72,8 +70,7 @@ type DB struct {
 	compPerErrC      chan error
 	compErrSetC      chan error
 	compWriteLocking bool
-	compStats        cStats
+	compStats        []cStats
-	memdbMaxLevel    int // For testing.
 
 	// Close.
 	closeW sync.WaitGroup
@@ -107,6 +104,7 @@ func openDB(s *session) (*DB, error) {
 		compErrC:    make(chan error),
 		compPerErrC: make(chan error),
 		compErrSetC: make(chan error),
+		compStats:   make([]cStats, s.o.GetNumLevel()),
 		// Close
 		closeC: make(chan struct{}),
 	}
@@ -211,7 +209,7 @@ func Open(stor storage.Storage, o *opt.Options) (db *DB, err error) {
 // The returned DB instance is goroutine-safe.
 // The DB must be closed after use, by calling Close method.
 func OpenFile(path string, o *opt.Options) (db *DB, err error) {
-	stor, err := storage.OpenFile(path, o.GetReadOnly())
+	stor, err := storage.OpenFile(path)
 	if err != nil {
 		return
 	}
@@ -261,7 +259,7 @@ func Recover(stor storage.Storage, o *opt.Options) (db *DB, err error) {
 // The returned DB instance is goroutine-safe.
 // The DB must be closed after use, by calling Close method.
 func RecoverFile(path string, o *opt.Options) (db *DB, err error) {
-	stor, err := storage.OpenFile(path, false)
+	stor, err := storage.OpenFile(path)
 	if err != nil {
 		return
 	}
@@ -280,11 +278,12 @@ func recoverTable(s *session, o *opt.Options) error {
 	o.Strict &= ^opt.StrictReader
 
 	// Get all tables and sort it by file number.
-	fds, err := s.stor.List(storage.TypeTable)
+	tableFiles_, err := s.getFiles(storage.TypeTable)
 	if err != nil {
 		return err
 	}
-	sortFds(fds)
+	tableFiles := files(tableFiles_)
+	tableFiles.sort()
 
 	var (
 		maxSeq                                                            uint64
@@ -297,17 +296,17 @@ func recoverTable(s *session, o *opt.Options) error {
 		rec   = &sessionRecord{}
 		bpool = util.NewBufferPool(o.GetBlockSize() + 5)
 	)
-	buildTable := func(iter iterator.Iterator) (tmpFd storage.FileDesc, size int64, err error) {
+	buildTable := func(iter iterator.Iterator) (tmp storage.File, size int64, err error) {
-		tmpFd = s.newTemp()
+		tmp = s.newTemp()
-		writer, err := s.stor.Create(tmpFd)
+		writer, err := tmp.Create()
 		if err != nil {
 			return
 		}
 		defer func() {
 			writer.Close()
 			if err != nil {
-				s.stor.Remove(tmpFd)
+				tmp.Remove()
-				tmpFd = storage.FileDesc{}
+				tmp = nil
 			}
 		}()
 
@@ -315,7 +314,7 @@ func recoverTable(s *session, o *opt.Options) error {
 		tw := table.NewWriter(writer, o)
 		for iter.Next() {
 			key := iter.Key()
-			if validInternalKey(key) {
+			if validIkey(key) {
 				err = tw.Append(key, iter.Value())
 				if err != nil {
 					return
@@ -339,9 +338,9 @@ func recoverTable(s *session, o *opt.Options) error {
 		size = int64(tw.BytesLen())
 		return
 	}
-	recoverTable := func(fd storage.FileDesc) error {
+	recoverTable := func(file storage.File) error {
-		s.logf("table@recovery recovering @%d", fd.Num)
+		s.logf("table@recovery recovering @%d", file.Num())
-		reader, err := s.stor.Open(fd)
+		reader, err := file.Open()
 		if err != nil {
 			return err
 		}
@@ -363,7 +362,7 @@ func recoverTable(s *session, o *opt.Options) error {
 			tgoodKey, tcorruptedKey, tcorruptedBlock int
 			imin, imax                               []byte
 		)
-		tr, err := table.NewReader(reader, size, fd, nil, bpool, o)
+		tr, err := table.NewReader(reader, size, storage.NewFileInfo(file), nil, bpool, o)
 		if err != nil {
 			return err
 		}
@@ -371,7 +370,7 @@ func recoverTable(s *session, o *opt.Options) error {
 		if itererr, ok := iter.(iterator.ErrorCallbackSetter); ok {
 			itererr.SetErrorCallback(func(err error) {
 				if errors.IsCorrupted(err) {
-					s.logf("table@recovery block corruption @%d %q", fd.Num, err)
+					s.logf("table@recovery block corruption @%d %q", file.Num(), err)
 					tcorruptedBlock++
 				}
 			})
@@ -380,7 +379,7 @@ func recoverTable(s *session, o *opt.Options) error {
 		// Scan the table.
 		for iter.Next() {
 			key := iter.Key()
-			_, seq, _, kerr := parseInternalKey(key)
+			_, seq, _, kerr := parseIkey(key)
 			if kerr != nil {
 				tcorruptedKey++
 				continue
@@ -406,23 +405,23 @@ func recoverTable(s *session, o *opt.Options) error {
 
 		if strict && (tcorruptedKey > 0 || tcorruptedBlock > 0) {
 			droppedTable++
-			s.logf("table@recovery dropped @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", fd.Num, tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq)
+			s.logf("table@recovery dropped @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", file.Num(), tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq)
 			return nil
 		}
 
 		if tgoodKey > 0 {
 			if tcorruptedKey > 0 || tcorruptedBlock > 0 {
 				// Rebuild the table.
-				s.logf("table@recovery rebuilding @%d", fd.Num)
+				s.logf("table@recovery rebuilding @%d", file.Num())
 				iter := tr.NewIterator(nil, nil)
-				tmpFd, newSize, err := buildTable(iter)
+				tmp, newSize, err := buildTable(iter)
 				iter.Release()
 				if err != nil {
 					return err
 				}
 				closed = true
 				reader.Close()
-				if err := s.stor.Rename(tmpFd, fd); err != nil {
+				if err := file.Replace(tmp); err != nil {
 					return err
 				}
 				size = newSize
@@ -432,30 +431,30 @@ func recoverTable(s *session, o *opt.Options) error {
 			}
 			recoveredKey += tgoodKey
 			// Add table to level 0.
-			rec.addTable(0, fd.Num, size, imin, imax)
+			rec.addTable(0, file.Num(), uint64(size), imin, imax)
-			s.logf("table@recovery recovered @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", fd.Num, tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq)
+			s.logf("table@recovery recovered @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", file.Num(), tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq)
 		} else {
 			droppedTable++
-			s.logf("table@recovery unrecoverable @%d Ck·%d Cb·%d S·%d", fd.Num, tcorruptedKey, tcorruptedBlock, size)
+			s.logf("table@recovery unrecoverable @%d Ck·%d Cb·%d S·%d", file.Num(), tcorruptedKey, tcorruptedBlock, size)
 		}
 
 		return nil
 	}
 
 	// Recover all tables.
-	if len(fds) > 0 {
+	if len(tableFiles) > 0 {
-		s.logf("table@recovery F·%d", len(fds))
+		s.logf("table@recovery F·%d", len(tableFiles))
 
 		// Mark file number as used.
-		s.markFileNum(fds[len(fds)-1].Num)
+		s.markFileNum(tableFiles[len(tableFiles)-1].Num())
 
-		for _, fd := range fds {
+		for _, file := range tableFiles {
-			if err := recoverTable(fd); err != nil {
+			if err := recoverTable(file); err != nil {
 				return err
 			}
 		}
 
-		s.logf("table@recovery recovered F·%d N·%d Gk·%d Ck·%d Q·%d", len(fds), recoveredKey, goodKey, corruptedKey, maxSeq)
+		s.logf("table@recovery recovered F·%d N·%d Gk·%d Ck·%d Q·%d", len(tableFiles), recoveredKey, goodKey, corruptedKey, maxSeq)
 	}
 
 	// Set sequence number.
@@ -472,31 +471,31 @@ func recoverTable(s *session, o *opt.Options) error {
 
 func (db *DB) recoverJournal() error {
 	// Get all journals and sort it by file number.
-	rawFds, err := db.s.stor.List(storage.TypeJournal)
+	allJournalFiles, err := db.s.getFiles(storage.TypeJournal)
 	if err != nil {
 		return err
 	}
-	sortFds(rawFds)
+	files(allJournalFiles).sort()
 
 	// Journals that will be recovered.
-	var fds []storage.FileDesc
+	var recJournalFiles []storage.File
-	for _, fd := range rawFds {
+	for _, jf := range allJournalFiles {
-		if fd.Num >= db.s.stJournalNum || fd.Num == db.s.stPrevJournalNum {
+		if jf.Num() >= db.s.stJournalNum || jf.Num() == db.s.stPrevJournalNum {
-			fds = append(fds, fd)
+			recJournalFiles = append(recJournalFiles, jf)
 		}
 	}
 
 	var (
-		ofd storage.FileDesc // Obsolete file.
+		of  storage.File // Obsolete file.
 		rec = &sessionRecord{}
 	)
 
 	// Recover journals.
-	if len(fds) > 0 {
+	if len(recJournalFiles) > 0 {
-		db.logf("journal@recovery F·%d", len(fds))
+		db.logf("journal@recovery F·%d", len(recJournalFiles))
 
 		// Mark file number as used.
-		db.s.markFileNum(fds[len(fds)-1].Num)
+		db.s.markFileNum(recJournalFiles[len(recJournalFiles)-1].Num())
 
 		var (
 			// Options.
@@ -510,31 +509,31 @@ func (db *DB) recoverJournal() error {
 			batch = &Batch{}
 		)
 
-		for _, fd := range fds {
+		for _, jf := range recJournalFiles {
-			db.logf("journal@recovery recovering @%d", fd.Num)
+			db.logf("journal@recovery recovering @%d", jf.Num())
 
-			fr, err := db.s.stor.Open(fd)
+			fr, err := jf.Open()
 			if err != nil {
 				return err
 			}
 
 			// Create or reset journal reader instance.
 			if jr == nil {
-				jr = journal.NewReader(fr, dropper{db.s, fd}, strict, checksum)
+				jr = journal.NewReader(fr, dropper{db.s, jf}, strict, checksum)
 			} else {
-				jr.Reset(fr, dropper{db.s, fd}, strict, checksum)
+				jr.Reset(fr, dropper{db.s, jf}, strict, checksum)
 			}
 
 			// Flush memdb and remove obsolete journal file.
-			if !ofd.Nil() {
+			if of != nil {
 				if mdb.Len() > 0 {
-					if _, err := db.s.flushMemdb(rec, mdb, 0); err != nil {
+					if _, err := db.s.flushMemdb(rec, mdb, -1); err != nil {
 						fr.Close()
 						return err
 					}
 				}
 
-				rec.setJournalNum(fd.Num)
+				rec.setJournalNum(jf.Num())
 				rec.setSeqNum(db.seq)
 				if err := db.s.commit(rec); err != nil {
 					fr.Close()
@@ -542,8 +541,8 @@ func (db *DB) recoverJournal() error {
 				}
 				rec.resetAddedTables()
 
-				db.s.stor.Remove(ofd)
+				of.Remove()
-				ofd = storage.FileDesc{}
+				of = nil
 			}
 
 			// Replay journal to memdb.
@@ -556,7 +555,7 @@ func (db *DB) recoverJournal() error {
 					}
 
 					fr.Close()
-					return errors.SetFd(err, fd)
+					return errors.SetFile(err, jf)
 				}
 
 				buf.Reset()
@@ -567,7 +566,7 @@ func (db *DB) recoverJournal() error {
 					}
 
 					fr.Close()
-					return errors.SetFd(err, fd)
+					return errors.SetFile(err, jf)
 				}
 				if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mdb); err != nil {
 					if !strict && errors.IsCorrupted(err) {
@@ -577,7 +576,7 @@ func (db *DB) recoverJournal() error {
 					}
 
 					fr.Close()
-					return errors.SetFd(err, fd)
+					return errors.SetFile(err, jf)
 				}
 
 				// Save sequence number.
@@ -595,7 +594,7 @@ func (db *DB) recoverJournal() error {
 			}
 
 			fr.Close()
-			ofd = fd
+			of = jf
 		}
 
 		// Flush the last memdb.
@@ -612,7 +611,7 @@ func (db *DB) recoverJournal() error {
 	}
 
 	// Commit.
-	rec.setJournalNum(db.journalFd.Num)
+	rec.setJournalNum(db.journalFile.Num())
 	rec.setSeqNum(db.seq)
 	if err := db.s.commit(rec); err != nil {
 		// Close journal on error.
@@ -624,8 +623,8 @@ func (db *DB) recoverJournal() error {
 	}
 
 	// Remove the last obsolete journal file.
-	if !ofd.Nil() {
+	if of != nil {
-		db.s.stor.Remove(ofd)
+		of.Remove()
 	}
 
 	return nil
@@ -633,17 +632,17 @@ func (db *DB) recoverJournal() error {
 
 func (db *DB) recoverJournalRO() error {
 	// Get all journals and sort it by file number.
-	rawFds, err := db.s.stor.List(storage.TypeJournal)
+	allJournalFiles, err := db.s.getFiles(storage.TypeJournal)
 	if err != nil {
 		return err
 	}
-	sortFds(rawFds)
+	files(allJournalFiles).sort()
 
 	// Journals that will be recovered.
-	var fds []storage.FileDesc
+	var recJournalFiles []storage.File
-	for _, fd := range rawFds {
+	for _, jf := range allJournalFiles {
-		if fd.Num >= db.s.stJournalNum || fd.Num == db.s.stPrevJournalNum {
+		if jf.Num() >= db.s.stJournalNum || jf.Num() == db.s.stPrevJournalNum {
-			fds = append(fds, fd)
+			recJournalFiles = append(recJournalFiles, jf)
 		}
 	}
 
@@ -657,8 +656,8 @@ func (db *DB) recoverJournalRO() error {
 	)
 
 	// Recover journals.
-	if len(fds) > 0 {
+	if len(recJournalFiles) > 0 {
-		db.logf("journal@recovery RO·Mode F·%d", len(fds))
+		db.logf("journal@recovery RO·Mode F·%d", len(recJournalFiles))
 
 		var (
 			jr    *journal.Reader
@@ -666,19 +665,19 @@ func (db *DB) recoverJournalRO() error {
 			batch = &Batch{}
 		)
 
-		for _, fd := range fds {
+		for _, jf := range recJournalFiles {
-			db.logf("journal@recovery recovering @%d", fd.Num)
+			db.logf("journal@recovery recovering @%d", jf.Num())
 
-			fr, err := db.s.stor.Open(fd)
+			fr, err := jf.Open()
 			if err != nil {
 				return err
 			}
 
 			// Create or reset journal reader instance.
 			if jr == nil {
-				jr = journal.NewReader(fr, dropper{db.s, fd}, strict, checksum)
+				jr = journal.NewReader(fr, dropper{db.s, jf}, strict, checksum)
 			} else {
-				jr.Reset(fr, dropper{db.s, fd}, strict, checksum)
+				jr.Reset(fr, dropper{db.s, jf}, strict, checksum)
 			}
 
 			// Replay journal to memdb.
@@ -690,7 +689,7 @@ func (db *DB) recoverJournalRO() error {
 					}
 
 					fr.Close()
-					return errors.SetFd(err, fd)
+					return errors.SetFile(err, jf)
 				}
 
 				buf.Reset()
@@ -701,7 +700,7 @@ func (db *DB) recoverJournalRO() error {
 					}
 
 					fr.Close()
-					return errors.SetFd(err, fd)
+					return errors.SetFile(err, jf)
 				}
 				if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mdb); err != nil {
 					if !strict && errors.IsCorrupted(err) {
@@ -711,7 +710,7 @@ func (db *DB) recoverJournalRO() error {
 					}
 
 					fr.Close()
-					return errors.SetFd(err, fd)
+					return errors.SetFile(err, jf)
 				}
 
 				// Save sequence number.
@@ -728,35 +727,46 @@ func (db *DB) recoverJournalRO() error {
 	return nil
 }
 
-func memGet(mdb *memdb.DB, ikey internalKey, icmp *iComparer) (ok bool, mv []byte, err error) {
+func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, err error) {
-	mk, mv, err := mdb.Find(ikey)
+	ikey := newIkey(key, seq, ktSeek)
-	if err == nil {
+
-		ukey, _, kt, kerr := parseInternalKey(mk)
+	em, fm := db.getMems()
-		if kerr != nil {
+	for _, m := range [...]*memDB{em, fm} {
-			// Shouldn't have had happen.
+		if m == nil {
-			panic(kerr)
+			continue
 		}
-		if icmp.uCompare(ukey, ikey.ukey()) == 0 {
+		defer m.decref()
-			if kt == keyTypeDel {
+
-				return true, nil, ErrNotFound
+		mk, mv, me := m.Find(ikey)
+		if me == nil {
+			ukey, _, kt, kerr := parseIkey(mk)
+			if kerr != nil {
+				// Shouldn't have had happen.
+				panic(kerr)
 			}
-			return true, mv, nil
+			if db.s.icmp.uCompare(ukey, key) == 0 {
-
+				if kt == ktDel {
+					return nil, ErrNotFound
+				}
+				return append([]byte{}, mv...), nil
+			}
+		} else if me != ErrNotFound {
+			return nil, me
 		}
-	} else if err != ErrNotFound {
+	}
-		return true, nil, err
+
+	v := db.s.version()
+	value, cSched, err := v.get(ikey, ro, false)
+	v.release()
+	if cSched {
+		// Trigger table compaction.
+		db.compSendTrigger(db.tcompCmdC)
 	}
 	return
 }
 
-func (db *DB) get(auxm *memdb.DB, auxt tFiles, key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, err error) {
+func (db *DB) has(key []byte, seq uint64, ro *opt.ReadOptions) (ret bool, err error) {
-	ikey := makeInternalKey(nil, key, seq, keyTypeSeek)
+	ikey := newIkey(key, seq, ktSeek)
-
-	if auxm != nil {
-		if ok, mv, me := memGet(auxm, ikey, db.s.icmp); ok {
-			return append([]byte{}, mv...), me
-		}
-	}
 
 	em, fm := db.getMems()
 	for _, m := range [...]*memDB{em, fm} {
@@ -765,55 +775,30 @@ func (db *DB) get(auxm *memdb.DB, auxt tFiles, key []byte, seq uint64, ro *opt.R
 		}
 		defer m.decref()
 
-		if ok, mv, me := memGet(m.DB, ikey, db.s.icmp); ok {
+		mk, _, me := m.Find(ikey)
-			return append([]byte{}, mv...), me
+		if me == nil {
+			ukey, _, kt, kerr := parseIkey(mk)
+			if kerr != nil {
+				// Shouldn't have had happen.
+				panic(kerr)
+			}
+			if db.s.icmp.uCompare(ukey, key) == 0 {
+				if kt == ktDel {
+					return false, nil
+				}
+				return true, nil
+			}
+		} else if me != ErrNotFound {
+			return false, me
 		}
 	}
 
 	v := db.s.version()
-	value, cSched, err := v.get(auxt, ikey, ro, false)
+	_, cSched, err := v.get(ikey, ro, true)
 	v.release()
 	if cSched {
 		// Trigger table compaction.
-		db.compTrigger(db.tcompCmdC)
+		db.compSendTrigger(db.tcompCmdC)
-	}
-	return
-}
-
-func nilIfNotFound(err error) error {
-	if err == ErrNotFound {
-		return nil
-	}
-	return err
-}
-
-func (db *DB) has(auxm *memdb.DB, auxt tFiles, key []byte, seq uint64, ro *opt.ReadOptions) (ret bool, err error) {
-	ikey := makeInternalKey(nil, key, seq, keyTypeSeek)
-
-	if auxm != nil {
-		if ok, _, me := memGet(auxm, ikey, db.s.icmp); ok {
-			return me == nil, nilIfNotFound(me)
-		}
-	}
-
-	em, fm := db.getMems()
-	for _, m := range [...]*memDB{em, fm} {
-		if m == nil {
-			continue
-		}
-		defer m.decref()
-
-		if ok, _, me := memGet(m.DB, ikey, db.s.icmp); ok {
-			return me == nil, nilIfNotFound(me)
-		}
-	}
-
-	v := db.s.version()
-	_, cSched, err := v.get(auxt, ikey, ro, true)
-	v.release()
-	if cSched {
-		// Trigger table compaction.
-		db.compTrigger(db.tcompCmdC)
 	}
 	if err == nil {
 		ret = true
@@ -837,7 +822,7 @@ func (db *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
 
 	se := db.acquireSnapshot()
 	defer db.releaseSnapshot(se)
-	return db.get(nil, nil, key, se.seq, ro)
+	return db.get(key, se.seq, ro)
 }
 
 // Has returns true if the DB does contains the given key.
@@ -851,11 +836,11 @@ func (db *DB) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) {
 
 	se := db.acquireSnapshot()
 	defer db.releaseSnapshot(se)
-	return db.has(nil, nil, key, se.seq, ro)
+	return db.has(key, se.seq, ro)
 }
 
 // NewIterator returns an iterator for the latest snapshot of the
-// underlying DB.
+// uderlying DB.
 // The returned iterator is not goroutine-safe, but it is safe to use
 // multiple iterators concurrently, with each in a dedicated goroutine.
 // It is also safe to use an iterator concurrently with modifying its
@@ -879,7 +864,7 @@ func (db *DB) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Itera
 	defer db.releaseSnapshot(se)
 	// Iterator holds 'version' lock, 'version' is immutable so snapshot
 	// can be released after iterator created.
-	return db.newIterator(nil, nil, se.seq, slice, ro)
+	return db.newIterator(se.seq, slice, ro)
 }
 
 // GetSnapshot returns a latest snapshot of the underlying DB. A snapshot
@@ -935,7 +920,7 @@ func (db *DB) GetProperty(name string) (value string, err error) {
 		var level uint
 		var rest string
 		n, _ := fmt.Sscanf(p[len(numFilesPrefix):], "%d%s", &level, &rest)
-		if n != 1 {
+		if n != 1 || int(level) >= db.s.o.GetNumLevel() {
 			err = ErrNotFound
 		} else {
 			value = fmt.Sprint(v.tLen(int(level)))
@@ -944,8 +929,8 @@ func (db *DB) GetProperty(name string) (value string, err error) {
 		value = "Compactions\n" +
 			" Level |   Tables   |    Size(MB)   |    Time(sec)  |    Read(MB)   |   Write(MB)\n" +
 			"-------+------------+---------------+---------------+---------------+---------------\n"
-		for level, tables := range v.levels {
+		for level, tables := range v.tables {
-			duration, read, write := db.compStats.getStat(level)
+			duration, read, write := db.compStats[level].get()
 			if len(tables) == 0 && duration == 0 {
 				continue
 			}
@@ -954,10 +939,10 @@ func (db *DB) GetProperty(name string) (value string, err error) {
 				float64(read)/1048576.0, float64(write)/1048576.0)
 		}
 	case p == "sstables":
-		for level, tables := range v.levels {
+		for level, tables := range v.tables {
 			value += fmt.Sprintf("--- level %d ---\n", level)
 			for _, t := range tables {
-				value += fmt.Sprintf("%d:%d[%q .. %q]\n", t.fd.Num, t.size, t.imin, t.imax)
+				value += fmt.Sprintf("%d:%d[%q .. %q]\n", t.file.Num(), t.size, t.imin, t.imax)
 			}
 		}
 	case p == "blockpool":
@@ -997,8 +982,8 @@ func (db *DB) SizeOf(ranges []util.Range) (Sizes, error) {
 
 	sizes := make(Sizes, 0, len(ranges))
 	for _, r := range ranges {
-		imin := makeInternalKey(nil, r.Start, keyMaxSeq, keyTypeSeek)
+		imin := newIkey(r.Start, kMaxSeq, ktSeek)
-		imax := makeInternalKey(nil, r.Limit, keyMaxSeq, keyTypeSeek)
+		imax := newIkey(r.Limit, kMaxSeq, ktSeek)
 		start, err := v.offsetOf(imin)
 		if err != nil {
 			return nil, err
@@ -1007,7 +992,7 @@ func (db *DB) SizeOf(ranges []util.Range) (Sizes, error) {
 		if err != nil {
 			return nil, err
 		}
-		var size int64
+		var size uint64
 		if limit >= start {
 			size = limit - start
 		}
@@ -1017,8 +1002,8 @@ func (db *DB) SizeOf(ranges []util.Range) (Sizes, error) {
 	return sizes, nil
 }
 
-// Close closes the DB. This will also releases any outstanding snapshot,
+// Close closes the DB. This will also releases any outstanding snapshot and
-// abort any in-flight compaction and discard open transaction.
+// abort any in-flight compaction.
 //
 // It is not safe to close a DB until all outstanding iterators are released.
 // It is valid to call Close multiple times. Other methods should not be
@@ -1047,18 +1032,11 @@ func (db *DB) Close() error {
 	// Signal all goroutines.
 	close(db.closeC)
 
-	// Discard open transaction.
-	if db.tr != nil {
-		db.tr.Discard()
-	}
-
-	// Acquire writer lock.
-	db.writeLockC <- struct{}{}
-
 	// Wait for all gorotines to exit.
 	db.closeW.Wait()
 
-	// Closes journal.
+	// Lock writer and closes journal.
+	db.writeLockC <- struct{}{}
 	if db.journal != nil {
 		db.journal.Close()
 		db.journalWriter.Close()
@@ -1085,6 +1063,8 @@ func (db *DB) Close() error {
 	db.frozenMem = nil
 	db.journal = nil
 	db.journalWriter = nil
+	db.journalFile = nil
+	db.frozenJournalFile = nil
 	db.closer = nil
 
 	return err

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go

@@ -12,76 +12,55 @@ import (
 
 	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/opt"
-	"github.com/syndtr/goleveldb/leveldb/storage"
 )
 
 var (
 	errCompactionTransactExiting = errors.New("leveldb: compaction transact exiting")
 )
 
-type cStat struct {
+type cStats struct {
+	sync.Mutex
 	duration time.Duration
-	read     int64
+	read     uint64
-	write    int64
+	write    uint64
 }
 
-func (p *cStat) add(n *cStatStaging) {
+func (p *cStats) add(n *cStatsStaging) {
+	p.Lock()
 	p.duration += n.duration
 	p.read += n.read
 	p.write += n.write
+	p.Unlock()
 }
 
-func (p *cStat) get() (duration time.Duration, read, write int64) {
+func (p *cStats) get() (duration time.Duration, read, write uint64) {
+	p.Lock()
+	defer p.Unlock()
 	return p.duration, p.read, p.write
 }
 
-type cStatStaging struct {
+type cStatsStaging struct {
 	start    time.Time
 	duration time.Duration
 	on       bool
-	read     int64
+	read     uint64
-	write    int64
+	write    uint64
 }
 
-func (p *cStatStaging) startTimer() {
+func (p *cStatsStaging) startTimer() {
 	if !p.on {
 		p.start = time.Now()
 		p.on = true
 	}
 }
 
-func (p *cStatStaging) stopTimer() {
+func (p *cStatsStaging) stopTimer() {
 	if p.on {
 		p.duration += time.Since(p.start)
 		p.on = false
 	}
 }
 
-type cStats struct {
-	lk    sync.Mutex
-	stats []cStat
-}
-
-func (p *cStats) addStat(level int, n *cStatStaging) {
-	p.lk.Lock()
-	if level >= len(p.stats) {
-		newStats := make([]cStat, level+1)
-		copy(newStats, p.stats)
-		p.stats = newStats
-	}
-	p.stats[level].add(n)
-	p.lk.Unlock()
-}
-
-func (p *cStats) getStat(level int) (duration time.Duration, read, write int64) {
-	p.lk.Lock()
-	defer p.lk.Unlock()
-	if level < len(p.stats) {
-		return p.stats[level].get()
-	}
-	return
-}
-
 func (db *DB) compactionError() {
 	var err error
 noerr:
@@ -256,14 +235,6 @@ func (db *DB) compactionExitTransact() {
 	panic(errCompactionTransactExiting)
 }
 
-func (db *DB) compactionCommit(name string, rec *sessionRecord) {
-	db.compCommitLk.Lock()
-	defer db.compCommitLk.Unlock() // Defer is necessary.
-	db.compactionTransactFunc(name+"@commit", func(cnt *compactionTransactCounter) error {
-		return db.s.commit(rec)
-	}, nil)
-}
-
 func (db *DB) memCompaction() {
 	mdb := db.getFrozenMem()
 	if mdb == nil {
@@ -294,40 +265,41 @@ func (db *DB) memCompaction() {
 
 	var (
 		rec        = &sessionRecord{}
-		stats      = &cStatStaging{}
+		stats      = &cStatsStaging{}
 		flushLevel int
 	)
 
-	// Generate tables.
 	db.compactionTransactFunc("memdb@flush", func(cnt *compactionTransactCounter) (err error) {
 		stats.startTimer()
-		flushLevel, err = db.s.flushMemdb(rec, mdb.DB, db.memdbMaxLevel)
+		flushLevel, err = db.s.flushMemdb(rec, mdb.DB, -1)
 		stats.stopTimer()
 		return
 	}, func() error {
 		for _, r := range rec.addedTables {
 			db.logf("memdb@flush revert @%d", r.num)
-			if err := db.s.stor.Remove(storage.FileDesc{Type: storage.TypeTable, Num: r.num}); err != nil {
+			f := db.s.getTableFile(r.num)
+			if err := f.Remove(); err != nil {
 				return err
 			}
 		}
 		return nil
 	})
 
-	rec.setJournalNum(db.journalFd.Num)
+	db.compactionTransactFunc("memdb@commit", func(cnt *compactionTransactCounter) (err error) {
-	rec.setSeqNum(db.frozenSeq)
+		stats.startTimer()
-
+		rec.setJournalNum(db.journalFile.Num())
-	// Commit.
+		rec.setSeqNum(db.frozenSeq)
-	stats.startTimer()
+		err = db.s.commit(rec)
-	db.compactionCommit("memdb", rec)
+		stats.stopTimer()
-	stats.stopTimer()
+		return
+	}, nil)
 
 	db.logf("memdb@flush committed F·%d T·%v", len(rec.addedTables), stats.duration)
 
 	for _, r := range rec.addedTables {
 		stats.write += r.size
 	}
-	db.compStats.addStat(flushLevel, stats)
+	db.compStats[flushLevel].add(stats)
 
 	// Drop frozen memdb.
 	db.dropFrozenMem()
@@ -343,7 +315,7 @@ func (db *DB) memCompaction() {
 	}
 
 	// Trigger table compaction.
-	db.compTrigger(db.tcompCmdC)
+	db.compSendTrigger(db.tcompCmdC)
 }
 
 type tableCompactionBuilder struct {
@@ -351,7 +323,7 @@ type tableCompactionBuilder struct {
 	s            *session
 	c            *compaction
 	rec          *sessionRecord
-	stat0, stat1 *cStatStaging
+	stat0, stat1 *cStatsStaging
 
 	snapHasLastUkey bool
 	snapLastUkey    []byte
@@ -405,9 +377,9 @@ func (b *tableCompactionBuilder) flush() error {
 	if err != nil {
 		return err
 	}
-	b.rec.addTableFile(b.c.sourceLevel+1, t)
+	b.rec.addTableFile(b.c.level+1, t)
 	b.stat1.write += t.size
-	b.s.logf("table@build created L%d@%d N·%d S·%s %q:%q", b.c.sourceLevel+1, t.fd.Num, b.tw.tw.EntriesLen(), shortenb(int(t.size)), t.imin, t.imax)
+	b.s.logf("table@build created L%d@%d N·%d S·%s %q:%q", b.c.level+1, t.file.Num(), b.tw.tw.EntriesLen(), shortenb(int(t.size)), t.imin, t.imax)
 	b.tw = nil
 	return nil
 }
@@ -452,7 +424,7 @@ func (b *tableCompactionBuilder) run(cnt *compactionTransactCounter) error {
 		}
 
 		ikey := iter.Key()
-		ukey, seq, kt, kerr := parseInternalKey(ikey)
+		ukey, seq, kt, kerr := parseIkey(ikey)
 
 		if kerr == nil {
 			shouldStop := !resumed && b.c.shouldStopBefore(ikey)
@@ -478,14 +450,14 @@ func (b *tableCompactionBuilder) run(cnt *compactionTransactCounter) error {
 
 				hasLastUkey = true
 				lastUkey = append(lastUkey[:0], ukey...)
-				lastSeq = keyMaxSeq
+				lastSeq = kMaxSeq
 			}
 
 			switch {
 			case lastSeq <= b.minSeq:
 				// Dropped because newer entry for same user key exist
 				fallthrough // (A)
-			case kt == keyTypeDel && seq <= b.minSeq && b.c.baseLevelForKey(lastUkey):
+			case kt == ktDel && seq <= b.minSeq && b.c.baseLevelForKey(lastUkey):
 				// For this user key:
 				// (1) there is no data in higher levels
 				// (2) data in lower levels will have larger seq numbers
@@ -507,7 +479,7 @@ func (b *tableCompactionBuilder) run(cnt *compactionTransactCounter) error {
 			// Don't drop corrupted keys.
 			hasLastUkey = false
 			lastUkey = lastUkey[:0]
-			lastSeq = keyMaxSeq
+			lastSeq = kMaxSeq
 			b.kerrCnt++
 		}
 
@@ -530,7 +502,8 @@ func (b *tableCompactionBuilder) run(cnt *compactionTransactCounter) error {
 func (b *tableCompactionBuilder) revert() error {
 	for _, at := range b.rec.addedTables {
 		b.s.logf("table@build revert @%d", at.num)
-		if err := b.s.stor.Remove(storage.FileDesc{Type: storage.TypeTable, Num: at.num}); err != nil {
+		f := b.s.getTableFile(at.num)
+		if err := f.Remove(); err != nil {
 			return err
 		}
 	}
@@ -541,28 +514,30 @@ func (db *DB) tableCompaction(c *compaction, noTrivial bool) {
 	defer c.release()
 
 	rec := &sessionRecord{}
-	rec.addCompPtr(c.sourceLevel, c.imax)
+	rec.addCompPtr(c.level, c.imax)
 
 	if !noTrivial && c.trivial() {
-		t := c.levels[0][0]
+		t := c.tables[0][0]
-		db.logf("table@move L%d@%d -> L%d", c.sourceLevel, t.fd.Num, c.sourceLevel+1)
+		db.logf("table@move L%d@%d -> L%d", c.level, t.file.Num(), c.level+1)
-		rec.delTable(c.sourceLevel, t.fd.Num)
+		rec.delTable(c.level, t.file.Num())
-		rec.addTableFile(c.sourceLevel+1, t)
+		rec.addTableFile(c.level+1, t)
-		db.compactionCommit("table-move", rec)
+		db.compactionTransactFunc("table@move", func(cnt *compactionTransactCounter) (err error) {
+			return db.s.commit(rec)
+		}, nil)
 		return
 	}
 
-	var stats [2]cStatStaging
+	var stats [2]cStatsStaging
-	for i, tables := range c.levels {
+	for i, tables := range c.tables {
 		for _, t := range tables {
 			stats[i].read += t.size
 			// Insert deleted tables into record
-			rec.delTable(c.sourceLevel+i, t.fd.Num)
+			rec.delTable(c.level+i, t.file.Num())
 		}
 	}
 	sourceSize := int(stats[0].read + stats[1].read)
 	minSeq := db.minSeq()
-	db.logf("table@compaction L%d·%d -> L%d·%d S·%s Q·%d", c.sourceLevel, len(c.levels[0]), c.sourceLevel+1, len(c.levels[1]), shortenb(sourceSize), minSeq)
+	db.logf("table@compaction L%d·%d -> L%d·%d S·%s Q·%d", c.level, len(c.tables[0]), c.level+1, len(c.tables[1]), shortenb(sourceSize), minSeq)
 
 	b := &tableCompactionBuilder{
 		db:        db,
@@ -572,60 +547,49 @@ func (db *DB) tableCompaction(c *compaction, noTrivial bool) {
 		stat1:     &stats[1],
 		minSeq:    minSeq,
 		strict:    db.s.o.GetStrict(opt.StrictCompaction),
-		tableSize: db.s.o.GetCompactionTableSize(c.sourceLevel + 1),
+		tableSize: db.s.o.GetCompactionTableSize(c.level + 1),
 	}
 	db.compactionTransact("table@build", b)
 
-	// Commit.
+	// Commit changes
-	stats[1].startTimer()
+	db.compactionTransactFunc("table@commit", func(cnt *compactionTransactCounter) (err error) {
-	db.compactionCommit("table", rec)
+		stats[1].startTimer()
-	stats[1].stopTimer()
+		defer stats[1].stopTimer()
+		return db.s.commit(rec)
+	}, nil)
 
 	resultSize := int(stats[1].write)
 	db.logf("table@compaction committed F%s S%s Ke·%d D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), b.kerrCnt, b.dropCnt, stats[1].duration)
 
 	// Save compaction stats
 	for i := range stats {
-		db.compStats.addStat(c.sourceLevel+1, &stats[i])
+		db.compStats[c.level+1].add(&stats[i])
 	}
 }
 
-func (db *DB) tableRangeCompaction(level int, umin, umax []byte) error {
+func (db *DB) tableRangeCompaction(level int, umin, umax []byte) {
 	db.logf("table@compaction range L%d %q:%q", level, umin, umax)
+
 	if level >= 0 {
-		if c := db.s.getCompactionRange(level, umin, umax, true); c != nil {
+		if c := db.s.getCompactionRange(level, umin, umax); c != nil {
 			db.tableCompaction(c, true)
 		}
 	} else {
-		// Retry until nothing to compact.
+		v := db.s.version()
-		for {
+		m := 1
-			compacted := false
+		for i, t := range v.tables[1:] {
-
+			if t.overlaps(db.s.icmp, umin, umax, false) {
-			// Scan for maximum level with overlapped tables.
+				m = i + 1
-			v := db.s.version()
-			m := 1
-			for i := m; i < len(v.levels); i++ {
-				tables := v.levels[i]
-				if tables.overlaps(db.s.icmp, umin, umax, false) {
-					m = i
-				}
 			}
-			v.release()
+		}
+		v.release()
 
-			for level := 0; level < m; level++ {
+		for level := 0; level < m; level++ {
-				if c := db.s.getCompactionRange(level, umin, umax, false); c != nil {
+			if c := db.s.getCompactionRange(level, umin, umax); c != nil {
-					db.tableCompaction(c, true)
+				db.tableCompaction(c, true)
-					compacted = true
-				}
-			}
-
-			if !compacted {
-				break
 			}
 		}
 	}
-
-	return nil
 }
 
 func (db *DB) tableAutoCompaction() {
@@ -652,11 +616,11 @@ type cCmd interface {
 	ack(err error)
 }
 
-type cAuto struct {
+type cIdle struct {
 	ackC chan<- error
 }
 
-func (r cAuto) ack(err error) {
+func (r cIdle) ack(err error) {
 	if r.ackC != nil {
 		defer func() {
 			recover()
@@ -680,21 +644,13 @@ func (r cRange) ack(err error) {
 	}
 }
 
-// This will trigger auto compaction but will not wait for it.
-func (db *DB) compTrigger(compC chan<- cCmd) {
-	select {
-	case compC <- cAuto{}:
-	default:
-	}
-}
-
 // This will trigger auto compation and/or wait for all compaction to be done.
-func (db *DB) compTriggerWait(compC chan<- cCmd) (err error) {
+func (db *DB) compSendIdle(compC chan<- cCmd) (err error) {
 	ch := make(chan error)
 	defer close(ch)
 	// Send cmd.
 	select {
-	case compC <- cAuto{ch}:
+	case compC <- cIdle{ch}:
 	case err = <-db.compErrC:
 		return
 	case _, _ = <-db.closeC:
@@ -710,8 +666,16 @@ func (db *DB) compTriggerWait(compC chan<- cCmd) (err error) {
 	return err
 }
 
+// This will trigger auto compaction but will not wait for it.
+func (db *DB) compSendTrigger(compC chan<- cCmd) {
+	select {
+	case compC <- cIdle{}:
+	default:
+	}
+}
+
 // Send range compaction request.
-func (db *DB) compTriggerRange(compC chan<- cCmd, level int, min, max []byte) (err error) {
+func (db *DB) compSendRange(compC chan<- cCmd, level int, min, max []byte) (err error) {
 	ch := make(chan error)
 	defer close(ch)
 	// Send cmd.
@@ -751,7 +715,7 @@ func (db *DB) mCompaction() {
 		select {
 		case x = <-db.mcompCmdC:
 			switch x.(type) {
-			case cAuto:
+			case cIdle:
 				db.memCompaction()
 				x.ack(nil)
 				x = nil
@@ -812,10 +776,11 @@ func (db *DB) tCompaction() {
 		}
 		if x != nil {
 			switch cmd := x.(type) {
-			case cAuto:
+			case cIdle:
 				ackQ = append(ackQ, x)
 			case cRange:
-				x.ack(db.tableRangeCompaction(cmd.level, cmd.min, cmd.max))
+				db.tableRangeCompaction(cmd.level, cmd.min, cmd.max)
+				x.ack(nil)
 			default:
 				panic("leveldb: unknown command")
 			}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_iter.go

@@ -19,7 +19,7 @@ import (
 )
 
 var (
-	errInvalidInternalKey = errors.New("leveldb: Iterator: invalid internal key")
+	errInvalidIkey = errors.New("leveldb: Iterator: invalid internal key")
 )
 
 type memdbReleaser struct {
@@ -33,50 +33,40 @@ func (mr *memdbReleaser) Release() {
 	})
 }
 
-func (db *DB) newRawIterator(auxm *memDB, auxt tFiles, slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
+func (db *DB) newRawIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
-	strict := opt.GetStrict(db.s.o.Options, ro, opt.StrictReader)
 	em, fm := db.getMems()
 	v := db.s.version()
 
-	tableIts := v.getIterators(slice, ro)
+	ti := v.getIterators(slice, ro)
-	n := len(tableIts) + len(auxt) + 3
+	n := len(ti) + 2
-	its := make([]iterator.Iterator, 0, n)
+	i := make([]iterator.Iterator, 0, n)
-
-	if auxm != nil {
-		ami := auxm.NewIterator(slice)
-		ami.SetReleaser(&memdbReleaser{m: auxm})
-		its = append(its, ami)
-	}
-	for _, t := range auxt {
-		its = append(its, v.s.tops.newIterator(t, slice, ro))
-	}
-
 	emi := em.NewIterator(slice)
 	emi.SetReleaser(&memdbReleaser{m: em})
-	its = append(its, emi)
+	i = append(i, emi)
 	if fm != nil {
 		fmi := fm.NewIterator(slice)
 		fmi.SetReleaser(&memdbReleaser{m: fm})
-		its = append(its, fmi)
+		i = append(i, fmi)
 	}
-	its = append(its, tableIts...)
+	i = append(i, ti...)
-	mi := iterator.NewMergedIterator(its, db.s.icmp, strict)
+	strict := opt.GetStrict(db.s.o.Options, ro, opt.StrictReader)
+	mi := iterator.NewMergedIterator(i, db.s.icmp, strict)
 	mi.SetReleaser(&versionReleaser{v: v})
 	return mi
 }
 
-func (db *DB) newIterator(auxm *memDB, auxt tFiles, seq uint64, slice *util.Range, ro *opt.ReadOptions) *dbIter {
+func (db *DB) newIterator(seq uint64, slice *util.Range, ro *opt.ReadOptions) *dbIter {
 	var islice *util.Range
 	if slice != nil {
 		islice = &util.Range{}
 		if slice.Start != nil {
-			islice.Start = makeInternalKey(nil, slice.Start, keyMaxSeq, keyTypeSeek)
+			islice.Start = newIkey(slice.Start, kMaxSeq, ktSeek)
 		}
 		if slice.Limit != nil {
-			islice.Limit = makeInternalKey(nil, slice.Limit, keyMaxSeq, keyTypeSeek)
+			islice.Limit = newIkey(slice.Limit, kMaxSeq, ktSeek)
 		}
 	}
-	rawIter := db.newRawIterator(auxm, auxt, islice, ro)
+	rawIter := db.newRawIterator(islice, ro)
 	iter := &dbIter{
 		db:     db,
 		icmp:   db.s.icmp,
@@ -187,7 +177,7 @@ func (i *dbIter) Seek(key []byte) bool {
 		return false
 	}
 
-	ikey := makeInternalKey(nil, key, i.seq, keyTypeSeek)
+	ikey := newIkey(key, i.seq, ktSeek)
 	if i.iter.Seek(ikey) {
 		i.dir = dirSOI
 		return i.next()
@@ -199,15 +189,15 @@ func (i *dbIter) Seek(key []byte) bool {
 
 func (i *dbIter) next() bool {
 	for {
-		if ukey, seq, kt, kerr := parseInternalKey(i.iter.Key()); kerr == nil {
+		if ukey, seq, kt, kerr := parseIkey(i.iter.Key()); kerr == nil {
 			i.sampleSeek()
 			if seq <= i.seq {
 				switch kt {
-				case keyTypeDel:
+				case ktDel:
 					// Skip deleted key.
 					i.key = append(i.key[:0], ukey...)
 					i.dir = dirForward
-				case keyTypeVal:
+				case ktVal:
 					if i.dir == dirSOI || i.icmp.uCompare(ukey, i.key) > 0 {
 						i.key = append(i.key[:0], ukey...)
 						i.value = append(i.value[:0], i.iter.Value()...)
@@ -250,13 +240,13 @@ func (i *dbIter) prev() bool {
 	del := true
 	if i.iter.Valid() {
 		for {
-			if ukey, seq, kt, kerr := parseInternalKey(i.iter.Key()); kerr == nil {
+			if ukey, seq, kt, kerr := parseIkey(i.iter.Key()); kerr == nil {
 				i.sampleSeek()
 				if seq <= i.seq {
 					if !del && i.icmp.uCompare(ukey, i.key) < 0 {
 						return true
 					}
-					del = (kt == keyTypeDel)
+					del = (kt == ktDel)
 					if !del {
 						i.key = append(i.key[:0], ukey...)
 						i.value = append(i.value[:0], i.iter.Value()...)
@@ -292,7 +282,7 @@ func (i *dbIter) Prev() bool {
 		return i.Last()
 	case dirForward:
 		for i.iter.Prev() {
-			if ukey, _, _, kerr := parseInternalKey(i.iter.Key()); kerr == nil {
+			if ukey, _, _, kerr := parseIkey(i.iter.Key()); kerr == nil {
 				i.sampleSeek()
 				if i.icmp.uCompare(ukey, i.key) < 0 {
 					goto cont