Translation update

* pr/909:
  Add Vilbrekin
  Correctly check whether parent directory is writable for current user. "&04" was checking if file is readable by others, while "&0200" checks if it's writable for current user.

CONTRIBUTING.md

@@ -12,7 +12,7 @@ least the following:
  - What operating system, operating system version and version of
    Syncthing you are running
 
- - The same for other connected nodes, where relevant
+ - The same for other connected devices, where relevant
 
  - Screenshot if the issue concerns something visible in the GUI
 
@@ -39,12 +39,24 @@ attention. In general, any open issues are fair game! Be prepared for a
 review](https://discourse.syncthing.net/t/733); it's all in the name of
 quality. :)
 
+You may be asked to "rebase" or "squash" your pull request before it's
+accepted. This means to make sure that the pull request:
+
+ - Is based on the latest commit on `master`.
+
+ - Only contains a single commit (exceptions to be made where
+   appropriate).
+
+ - Has a descriptive commit message, ending in `(fixes #123)` if it
+   resolves an existing issue.
+
 ## Licensing
 
-All contributions are made under the same MIT License as the rest of the
-project, except documentation which is licensed under the Creative
-Commons Attribution 4.0 International License. You retain the copyright
-to code you have written.
+All contributions are made under the same GPL license as the rest of the
+project, except documentation, user interface text and translation
+strings which are licensed under the Creative Commons Attribution 4.0
+International License. You retain the copyright to code you have
+written.
 
 When accepting your first contribution, the maintainer of the project
 will ensure that you are added to the CONTRIBUTORS file. You are welcome
@@ -90,4 +102,4 @@ Yes please!
 
 ## License
 
-MIT
+GPLv3

CONTRIBUTORS

@@ -5,12 +5,21 @@ Audrius Butkevicius <audrius.butkevicius@gmail.com>
 Arthur Axel fREW Schmidt <frew@afoolishmanifesto.com> <frioux@gmail.com>
 Ben Sidhom <bsidhom@gmail.com>
 Brandon Philips <brandon@ifup.org>
+Caleb Callaway <enlightened.despot@gmail.com>
+Chris Joel <chris@scriptolo.gy>
+Daniel Martí <mvdan@mvdan.cc>
+Felix Ableitner <me@nutomic.com>
+Felix Unterpaintner <bigbear2nd@gmail.com>
 Gilli Sigurdsson <gilli@vx.is>
 James Patterson <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
 Jens Diemer <github.com@jensdiemer.de> <git@jensdiemer.de>
+Jochen Voss <voss@seehuhn.de>
+Lode Hoste <zillode@zillode.be>
 Marcin Dziadus <dziadus.marcin@gmail.com>
 Michael Tilli <pyfisch@gmail.com>
 Philippe Schommers <philippe@schommers.be>
+Phill Luby <phill.luby@newredo.com>
 Ryan Sullivan <kayoticsully@gmail.com>
 Tully Robinson <tully@tojr.org>
 Veeti Paananen <veeti.paananen@rojekti.fi>
+Vil Brekin <vilbrekin@gmail.com>

Godeps/Godeps.json

@@ -1,15 +1,10 @@
 {
 	"ImportPath": "github.com/syncthing/syncthing",
-	"GoVersion": "go1.3.1",
+	"GoVersion": "go1.3.3",
 	"Packages": [
 		"./cmd/..."
 	],
 	"Deps": [
-		{
-			"ImportPath": "bitbucket.org/kardianos/osext",
-			"Comment": "null-13",
-			"Rev": "5d3ddcf53a508cc2f7404eaebf546ef2cb5cdb6e"
-		},
 		{
 			"ImportPath": "code.google.com/p/go.crypto/bcrypt",
 			"Comment": "null-216",
@@ -35,13 +30,25 @@
 			"Comment": "null-15",
 			"Rev": "12e4b4183793ac4b061921e7980845e750679fd0"
 		},
+		{
+			"ImportPath": "github.com/AudriusButkevicius/lfu-go",
+			"Rev": "164bcecceb92fd6037f4d18a8d97b495ec6ef669"
+		},
 		{
 			"ImportPath": "github.com/bkaradzic/go-lz4",
-			"Rev": "77e2ba877bde9da31213bec75dbbe197fa507c21"
+			"Rev": "93a831dcee242be64a9cc9803dda84af25932de7"
+		},
+		{
+			"ImportPath": "github.com/calmh/logger",
+			"Rev": "f50d32b313bec2933a3e1049f7416a29f3413d29"
+		},
+		{
+			"ImportPath": "github.com/calmh/osext",
+			"Rev": "9bf61584e5f1f172e8766ddc9022d9c401faaa5e"
 		},
 		{
 			"ImportPath": "github.com/calmh/xdr",
-			"Rev": "a597b63b87d6140f79084c8aab214b4d533833a1"
+			"Rev": "ec3d404f43731551258977b38dd72cf557d00398"
 		},
 		{
 			"ImportPath": "github.com/juju/ratelimit",
@@ -49,7 +56,7 @@
 		},
 		{
 			"ImportPath": "github.com/syndtr/goleveldb/leveldb",
-			"Rev": "2b99e8d4757bf06eeab1b0485d80b8ae1c088874"
+			"Rev": "0d8857b7ec571b0a6c9677d8e6c0a4ceeabd1d71"
 		},
 		{
 			"ImportPath": "github.com/vitrun/qart/coding",

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2012 Dave Grijalva
+
+Permission is hereby granted, free of charge, to any person obtaining a 
+copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation 
+the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the 
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included 
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 
+THE SOFTWARE.

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/README.md

@@ -0,0 +1,19 @@
+A simple LFU cache for golang.  Based on the paper [An O(1) algorithm for implementing the LFU cache eviction scheme](http://dhruvbird.com/lfu.pdf).
+
+Usage:
+
+```go
+import "github.com/dgrijalva/lfu-go"
+
+// Make a new thing
+c := lfu.New()
+
+// Set some values
+c.Set("myKey", myValue)
+
+// Retrieve some values
+myValue = c.Get("myKey")
+
+// Evict some values
+c.Evict(1)
+```

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/lfu.go

@@ -0,0 +1,156 @@
+package lfu
+
+import (
+	"container/list"
+	"sync"
+)
+
+type Eviction struct {
+	Key   string
+	Value interface{}
+}
+
+type Cache struct {
+	// If len > UpperBound, cache will automatically evict
+	// down to LowerBound.  If either value is 0, this behavior
+	// is disabled.
+	UpperBound      int
+	LowerBound      int
+	values          map[string]*cacheEntry
+	freqs           *list.List
+	len             int
+	lock            *sync.Mutex
+	EvictionChannel chan<- Eviction
+}
+
+type cacheEntry struct {
+	key      string
+	value    interface{}
+	freqNode *list.Element
+}
+
+type listEntry struct {
+	entries map[*cacheEntry]byte
+	freq    int
+}
+
+func New() *Cache {
+	c := new(Cache)
+	c.values = make(map[string]*cacheEntry)
+	c.freqs = list.New()
+	c.lock = new(sync.Mutex)
+	return c
+}
+
+func (c *Cache) Get(key string) interface{} {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	if e, ok := c.values[key]; ok {
+		c.increment(e)
+		return e.value
+	}
+	return nil
+}
+
+func (c *Cache) Set(key string, value interface{}) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	if e, ok := c.values[key]; ok {
+		// value already exists for key.  overwrite
+		e.value = value
+		c.increment(e)
+	} else {
+		// value doesn't exist.  insert
+		e := new(cacheEntry)
+		e.key = key
+		e.value = value
+		c.values[key] = e
+		c.increment(e)
+		c.len++
+		// bounds mgmt
+		if c.UpperBound > 0 && c.LowerBound > 0 {
+			if c.len > c.UpperBound {
+				c.evict(c.len - c.LowerBound)
+			}
+		}
+	}
+}
+
+func (c *Cache) Len() int {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return c.len
+}
+
+func (c *Cache) Evict(count int) int {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return c.evict(count)
+}
+
+func (c *Cache) evict(count int) int {
+	// No lock here so it can be called
+	// from within the lock (during Set)
+	var evicted int
+	for i := 0; i < count; {
+		if place := c.freqs.Front(); place != nil {
+			for entry, _ := range place.Value.(*listEntry).entries {
+				if i < count {
+					if c.EvictionChannel != nil {
+						c.EvictionChannel <- Eviction{
+							Key:   entry.key,
+							Value: entry.value,
+						}
+					}
+					delete(c.values, entry.key)
+					c.remEntry(place, entry)
+					evicted++
+					c.len--
+					i++
+				}
+			}
+		}
+	}
+	return evicted
+}
+
+func (c *Cache) increment(e *cacheEntry) {
+	currentPlace := e.freqNode
+	var nextFreq int
+	var nextPlace *list.Element
+	if currentPlace == nil {
+		// new entry
+		nextFreq = 1
+		nextPlace = c.freqs.Front()
+	} else {
+		// move up
+		nextFreq = currentPlace.Value.(*listEntry).freq + 1
+		nextPlace = currentPlace.Next()
+	}
+
+	if nextPlace == nil || nextPlace.Value.(*listEntry).freq != nextFreq {
+		// create a new list entry
+		li := new(listEntry)
+		li.freq = nextFreq
+		li.entries = make(map[*cacheEntry]byte)
+		if currentPlace != nil {
+			nextPlace = c.freqs.InsertAfter(li, currentPlace)
+		} else {
+			nextPlace = c.freqs.PushFront(li)
+		}
+	}
+	e.freqNode = nextPlace
+	nextPlace.Value.(*listEntry).entries[e] = 1
+	if currentPlace != nil {
+		// remove from current position
+		c.remEntry(currentPlace, e)
+	}
+}
+
+func (c *Cache) remEntry(place *list.Element, entry *cacheEntry) {
+	entries := place.Value.(*listEntry).entries
+	delete(entries, entry)
+	if len(entries) == 0 {
+		c.freqs.Remove(place)
+	}
+}

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/lfu_test.go

@@ -0,0 +1,68 @@
+package lfu
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestLFU(t *testing.T) {
+	c := New()
+	c.Set("a", "a")
+	if v := c.Get("a"); v != "a" {
+		t.Errorf("Value was not saved: %v != 'a'", v)
+	}
+	if l := c.Len(); l != 1 {
+		t.Errorf("Length was not updated: %v != 1", l)
+	}
+
+	c.Set("b", "b")
+	if v := c.Get("b"); v != "b" {
+		t.Errorf("Value was not saved: %v != 'b'", v)
+	}
+	if l := c.Len(); l != 2 {
+		t.Errorf("Length was not updated: %v != 2", l)
+	}
+
+	c.Get("a")
+	evicted := c.Evict(1)
+	if v := c.Get("a"); v != "a" {
+		t.Errorf("Value was improperly evicted: %v != 'a'", v)
+	}
+	if v := c.Get("b"); v != nil {
+		t.Errorf("Value was not evicted: %v", v)
+	}
+	if l := c.Len(); l != 1 {
+		t.Errorf("Length was not updated: %v != 1", l)
+	}
+	if evicted != 1 {
+		t.Errorf("Number of evicted items is wrong: %v != 1", evicted)
+	}
+}
+
+func TestBoundsMgmt(t *testing.T) {
+	c := New()
+	c.UpperBound = 10
+	c.LowerBound = 5
+
+	for i := 0; i < 100; i++ {
+		c.Set(fmt.Sprintf("%v", i), i)
+	}
+	if c.Len() > 10 {
+		t.Errorf("Bounds management failed to evict properly: %v", c.Len())
+	}
+}
+
+func TestEviction(t *testing.T) {
+	ch := make(chan Eviction, 1)
+
+	c := New()
+	c.EvictionChannel = ch
+	c.Set("a", "b")
+	c.Evict(1)
+
+	ev := <-ch
+
+	if ev.Key != "a" || ev.Value.(string) != "b" {
+		t.Error("Incorrect item")
+	}
+}

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/lz4-example/main.go

@@ -72,10 +72,18 @@ func main() {
 
 	if *decompress {
 		data, _ = ioutil.ReadAll(input)
-		data, _ = lz4.Decode(nil, data)
+		data, err = lz4.Decode(nil, data)
+		if err != nil {
+			fmt.Println("Failed to decode:", err)
+			return
+		}
 	} else {
 		data, _ = ioutil.ReadAll(input)
-		data, _ = lz4.Encode(nil, data)
+		data, err = lz4.Encode(nil, data)
+		if err != nil {
+			fmt.Println("Failed to encode:", err)
+			return
+		}
 	}
 
 	err = ioutil.WriteFile(args[1], data, 0644)

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/writer.go

@@ -121,7 +121,7 @@ func Encode(dst, src []byte) ([]byte, error) {
 	)
 
 	for {
-		if int(e.pos)+4 >= len(e.src) {
+		if int(e.pos)+12 >= len(e.src) {
 			e.writeLiterals(uint32(len(e.src))-e.anchor, 0, e.anchor)
 			return e.dst[:e.dpos], nil
 		}
@@ -158,7 +158,7 @@ func Encode(dst, src []byte) ([]byte, error) {
 		ref += minMatch
 		e.anchor = e.pos
 
-		for int(e.pos) < len(e.src) && e.src[e.pos] == e.src[ref] {
+		for int(e.pos) < len(e.src)-5 && e.src[e.pos] == e.src[ref] {
 			e.pos++
 			ref++
 		}

Godeps/_workspace/src/github.com/calmh/logger/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2013 Jakob Borg
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+of the Software, and to permit persons to whom the Software is furnished to do
+so, subject to the following conditions:
+
+- The above copyright notice and this permission notice shall be included in
+  all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Godeps/_workspace/src/github.com/calmh/logger/README.md

@@ -0,0 +1,14 @@
+logger
+======
+
+A small wrapper around `log` to provide log levels.
+
+Documentation
+-------------
+
+http://godoc.org/github.com/calmh/logger
+
+License
+-------
+
+MIT

Godeps/_workspace/src/github.com/calmh/logger/logger.go

@@ -1,6 +1,5 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
 
 // Package logger implements a standardized logger with callback functionality
 package logger
@@ -24,6 +23,7 @@ const (
 	NumLevels
 )
 
+// A MessageHandler is called with the log level and message text.
 type MessageHandler func(l LogLevel, msg string)
 
 type Logger struct {
@@ -32,6 +32,7 @@ type Logger struct {
 	mut      sync.Mutex
 }
 
+// The default logger logs to standard output with a time prefix.
 var DefaultLogger = New()
 
 func New() *Logger {
@@ -40,16 +41,20 @@ func New() *Logger {
 	}
 }
 
+// AddHandler registers a new MessageHandler to receive messages with the
+// specified log level or above.
 func (l *Logger) AddHandler(level LogLevel, h MessageHandler) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
 	l.handlers[level] = append(l.handlers[level], h)
 }
 
+// See log.SetFlags
 func (l *Logger) SetFlags(flag int) {
 	l.logger.SetFlags(flag)
 }
 
+// See log.SetPrefix
 func (l *Logger) SetPrefix(prefix string) {
 	l.logger.SetPrefix(prefix)
 }
@@ -60,6 +65,7 @@ func (l *Logger) callHandlers(level LogLevel, s string) {
 	}
 }
 
+// Debugln logs a line with a DEBUG prefix.
 func (l *Logger) Debugln(vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -68,6 +74,7 @@ func (l *Logger) Debugln(vals ...interface{}) {
 	l.callHandlers(LevelDebug, s)
 }
 
+// Debugf logs a formatted line with a DEBUG prefix.
 func (l *Logger) Debugf(format string, vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -75,6 +82,8 @@ func (l *Logger) Debugf(format string, vals ...interface{}) {
 	l.logger.Output(2, "DEBUG: "+s)
 	l.callHandlers(LevelDebug, s)
 }
+
+// Infoln logs a line with an INFO prefix.
 func (l *Logger) Infoln(vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -83,6 +92,7 @@ func (l *Logger) Infoln(vals ...interface{}) {
 	l.callHandlers(LevelInfo, s)
 }
 
+// Infof logs a formatted line with an INFO prefix.
 func (l *Logger) Infof(format string, vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -91,6 +101,7 @@ func (l *Logger) Infof(format string, vals ...interface{}) {
 	l.callHandlers(LevelInfo, s)
 }
 
+// Okln logs a line with an OK prefix.
 func (l *Logger) Okln(vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -99,6 +110,7 @@ func (l *Logger) Okln(vals ...interface{}) {
 	l.callHandlers(LevelOK, s)
 }
 
+// Okf logs a formatted line with an OK prefix.
 func (l *Logger) Okf(format string, vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -107,6 +119,7 @@ func (l *Logger) Okf(format string, vals ...interface{}) {
 	l.callHandlers(LevelOK, s)
 }
 
+// Warnln logs a formatted line with a WARNING prefix.
 func (l *Logger) Warnln(vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -115,6 +128,7 @@ func (l *Logger) Warnln(vals ...interface{}) {
 	l.callHandlers(LevelWarn, s)
 }
 
+// Warnf logs a formatted line with a WARNING prefix.
 func (l *Logger) Warnf(format string, vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -123,6 +137,8 @@ func (l *Logger) Warnf(format string, vals ...interface{}) {
 	l.callHandlers(LevelWarn, s)
 }
 
+// Fatalln logs a line with a FATAL prefix and exits the process with exit
+// code 1.
 func (l *Logger) Fatalln(vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -132,6 +148,8 @@ func (l *Logger) Fatalln(vals ...interface{}) {
 	os.Exit(1)
 }
 
+// Fatalf logs a formatted line with a FATAL prefix and exits the process with
+// exit code 1.
 func (l *Logger) Fatalf(format string, vals ...interface{}) {
 	l.mut.Lock()
 	defer l.mut.Unlock()
@@ -140,14 +158,3 @@ func (l *Logger) Fatalf(format string, vals ...interface{}) {
 	l.callHandlers(LevelFatal, s)
 	os.Exit(1)
 }
-
-func (l *Logger) FatalErr(err error) {
-	if err != nil {
-		l.mut.Lock()
-		defer l.mut.Unlock()
-		l.logger.SetFlags(l.logger.Flags() | log.Lshortfile)
-		l.logger.Output(2, "FATAL: "+err.Error())
-		l.callHandlers(LevelFatal, err.Error())
-		os.Exit(1)
-	}
-}

Godeps/_workspace/src/github.com/calmh/logger/logger_test.go

@@ -1,6 +1,5 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+// Copyright (C) 2014 Jakob Borg. All rights reserved. Use of this source code
+// is governed by an MIT-style license that can be found in the LICENSE file.
 
 package logger
 

Godeps/_workspace/src/github.com/calmh/osext/osext_procfs.go

@@ -2,12 +2,13 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build linux netbsd openbsd
+// +build linux netbsd openbsd solaris
 
 package osext
 
 import (
 	"errors"
+	"fmt"
 	"os"
 	"runtime"
 )
@@ -20,6 +21,8 @@ func executable() (string, error) {
 		return os.Readlink("/proc/curproc/exe")
 	case "openbsd":
 		return os.Readlink("/proc/curproc/file")
+	case "solaris":
+		return os.Readlink(fmt.Sprintf("/proc/%d/path/a.out", os.Getpid()))
 	}
 	return "", errors.New("ExecPath not implemented for " + runtime.GOOS)
 }

Godeps/_workspace/src/github.com/calmh/xdr/bench_test.go

@@ -33,11 +33,15 @@ var s = XDRBenchStruct{
 	S0:  "Hello World! String one.",
 	S1:  "Hello World! String two.",
 }
-var e = s.MarshalXDR()
+var e []byte
+
+func init() {
+	e, _ = s.MarshalXDR()
+}
 
 func BenchmarkThisMarshal(b *testing.B) {
 	for i := 0; i < b.N; i++ {
-		res = s.MarshalXDR()
+		res, _ = s.MarshalXDR()
 	}
 }
 

Godeps/_workspace/src/github.com/calmh/xdr/bench_xdr_test.go

@@ -72,15 +72,23 @@ func (o XDRBenchStruct) EncodeXDR(w io.Writer) (int, error) {
 	return o.encodeXDR(xw)
 }
 
-func (o XDRBenchStruct) MarshalXDR() []byte {
+func (o XDRBenchStruct) MarshalXDR() ([]byte, error) {
 	return o.AppendXDR(make([]byte, 0, 128))
 }
 
-func (o XDRBenchStruct) AppendXDR(bs []byte) []byte {
+func (o XDRBenchStruct) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o XDRBenchStruct) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	o.encodeXDR(xw)
-	return []byte(aw)
+	_, err := o.encodeXDR(xw)
+	return []byte(aw), err
 }
 
 func (o XDRBenchStruct) encodeXDR(xw *xdr.Writer) (int, error) {
@@ -88,13 +96,13 @@ func (o XDRBenchStruct) encodeXDR(xw *xdr.Writer) (int, error) {
 	xw.WriteUint32(o.I2)
 	xw.WriteUint16(o.I3)
 	xw.WriteUint8(o.I4)
-	if len(o.Bs0) > 128 {
-		return xw.Tot(), xdr.ErrElementSizeExceeded
+	if l := len(o.Bs0); l > 128 {
+		return xw.Tot(), xdr.ElementSizeExceeded("Bs0", l, 128)
 	}
 	xw.WriteBytes(o.Bs0)
 	xw.WriteBytes(o.Bs1)
-	if len(o.S0) > 128 {
-		return xw.Tot(), xdr.ErrElementSizeExceeded
+	if l := len(o.S0); l > 128 {
+		return xw.Tot(), xdr.ElementSizeExceeded("S0", l, 128)
 	}
 	xw.WriteString(o.S0)
 	xw.WriteString(o.S1)
@@ -150,15 +158,23 @@ func (o repeatReader) EncodeXDR(w io.Writer) (int, error) {
 	return o.encodeXDR(xw)
 }
 
-func (o repeatReader) MarshalXDR() []byte {
+func (o repeatReader) MarshalXDR() ([]byte, error) {
 	return o.AppendXDR(make([]byte, 0, 128))
 }
 
-func (o repeatReader) AppendXDR(bs []byte) []byte {
+func (o repeatReader) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o repeatReader) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	o.encodeXDR(xw)
-	return []byte(aw)
+	_, err := o.encodeXDR(xw)
+	return []byte(aw), err
 }
 
 func (o repeatReader) encodeXDR(xw *xdr.Writer) (int, error) {

Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go

@@ -53,15 +53,23 @@ func (o {{.TypeName}}) EncodeXDR(w io.Writer) (int, error) {
 	return o.encodeXDR(xw)
 }//+n
 
-func (o {{.TypeName}}) MarshalXDR() []byte {
+func (o {{.TypeName}}) MarshalXDR() ([]byte, error) {
 	return o.AppendXDR(make([]byte, 0, 128))
 }//+n
 
-func (o {{.TypeName}}) AppendXDR(bs []byte) []byte {
+func (o {{.TypeName}}) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}//+n
+
+func (o {{.TypeName}}) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	o.encodeXDR(xw)
-	return []byte(aw)
+	_, err := o.encodeXDR(xw)
+	return []byte(aw), err
 }//+n
 
 func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
@@ -71,8 +79,8 @@ func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
 				xw.Write{{$fieldInfo.Encoder}}({{$fieldInfo.Convert}}(o.{{$fieldInfo.Name}}))
 			{{else if $fieldInfo.IsBasic}}
 				{{if ge $fieldInfo.Max 1}}
-					if len(o.{{$fieldInfo.Name}}) > {{$fieldInfo.Max}} {
-						return xw.Tot(), xdr.ErrElementSizeExceeded
+					if l := len(o.{{$fieldInfo.Name}}); l > {{$fieldInfo.Max}} {
+						return xw.Tot(), xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", l, {{$fieldInfo.Max}})
 					}
 				{{end}}
 				xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}})
@@ -84,8 +92,8 @@ func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
 			{{end}}
 		{{else}}
 			{{if ge $fieldInfo.Max 1}}
-				if len(o.{{$fieldInfo.Name}}) > {{$fieldInfo.Max}} {
-					return xw.Tot(), xdr.ErrElementSizeExceeded
+				if l := len(o.{{$fieldInfo.Name}}); l > {{$fieldInfo.Max}} {
+					return xw.Tot(), xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", l, {{$fieldInfo.Max}})
 				}
 			{{end}}
 			xw.WriteUint32(uint32(len(o.{{$fieldInfo.Name}})))
@@ -135,7 +143,7 @@ func (o *{{.TypeName}}) decodeXDR(xr *xdr.Reader) error {
 			_{{$fieldInfo.Name}}Size := int(xr.ReadUint32())
 			{{if ge $fieldInfo.Max 1}}
 				if _{{$fieldInfo.Name}}Size > {{$fieldInfo.Max}} {
-					return xdr.ErrElementSizeExceeded
+					return xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", _{{$fieldInfo.Name}}Size, {{$fieldInfo.Max}})
 				}
 			{{end}}
 			o.{{$fieldInfo.Name}} = make([]{{$fieldInfo.FieldType}}, _{{$fieldInfo.Name}}Size)

Godeps/_workspace/src/github.com/calmh/xdr/encdec_test.go

@@ -24,9 +24,10 @@ type TestStruct struct {
 	UI32 uint32
 	I64  int64
 	UI64 uint64
-	BS   []byte
-	S    string
+	BS   []byte // max:1024
+	S    string // max:1024
 	C    Opaque
+	SS   []string // max:1024
 }
 
 type Opaque [32]byte
@@ -49,9 +50,12 @@ func (Opaque) Generate(rand *rand.Rand, size int) reflect.Value {
 
 func TestEncDec(t *testing.T) {
 	fn := func(t0 TestStruct) bool {
-		bs := t0.MarshalXDR()
+		bs, err := t0.MarshalXDR()
+		if err != nil {
+			t.Fatal(err)
+		}
 		var t1 TestStruct
-		err := t1.UnmarshalXDR(bs)
+		err = t1.UnmarshalXDR(bs)
 		if err != nil {
 			t.Fatal(err)
 		}

Godeps/_workspace/src/github.com/calmh/xdr/encdec_xdr_test.go

@@ -54,6 +54,14 @@ TestStruct Structure:
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |                            Opaque                             |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Number of SS                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                         Length of SS                          |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                     SS (variable length)                      \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 
 struct TestStruct {
@@ -66,9 +74,10 @@ struct TestStruct {
 	unsigned int UI32;
 	hyper I64;
 	unsigned hyper UI64;
-	opaque BS<>;
-	string S<>;
+	opaque BS<1024>;
+	string S<1024>;
 	Opaque C;
+	string SS<1024>;
 }
 
 */
@@ -78,15 +87,23 @@ func (o TestStruct) EncodeXDR(w io.Writer) (int, error) {
 	return o.encodeXDR(xw)
 }
 
-func (o TestStruct) MarshalXDR() []byte {
+func (o TestStruct) MarshalXDR() ([]byte, error) {
 	return o.AppendXDR(make([]byte, 0, 128))
 }
 
-func (o TestStruct) AppendXDR(bs []byte) []byte {
+func (o TestStruct) MustMarshalXDR() []byte {
+	bs, err := o.MarshalXDR()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}
+
+func (o TestStruct) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	o.encodeXDR(xw)
-	return []byte(aw)
+	_, err := o.encodeXDR(xw)
+	return []byte(aw), err
 }
 
 func (o TestStruct) encodeXDR(xw *xdr.Writer) (int, error) {
@@ -99,12 +116,25 @@ func (o TestStruct) encodeXDR(xw *xdr.Writer) (int, error) {
 	xw.WriteUint32(o.UI32)
 	xw.WriteUint64(uint64(o.I64))
 	xw.WriteUint64(o.UI64)
+	if l := len(o.BS); l > 1024 {
+		return xw.Tot(), xdr.ElementSizeExceeded("BS", l, 1024)
+	}
 	xw.WriteBytes(o.BS)
+	if l := len(o.S); l > 1024 {
+		return xw.Tot(), xdr.ElementSizeExceeded("S", l, 1024)
+	}
 	xw.WriteString(o.S)
 	_, err := o.C.encodeXDR(xw)
 	if err != nil {
 		return xw.Tot(), err
 	}
+	if l := len(o.SS); l > 1024 {
+		return xw.Tot(), xdr.ElementSizeExceeded("SS", l, 1024)
+	}
+	xw.WriteUint32(uint32(len(o.SS)))
+	for i := range o.SS {
+		xw.WriteString(o.SS[i])
+	}
 	return xw.Tot(), xw.Error()
 }
 
@@ -129,8 +159,16 @@ func (o *TestStruct) decodeXDR(xr *xdr.Reader) error {
 	o.UI32 = xr.ReadUint32()
 	o.I64 = int64(xr.ReadUint64())
 	o.UI64 = xr.ReadUint64()
-	o.BS = xr.ReadBytes()
-	o.S = xr.ReadString()
+	o.BS = xr.ReadBytesMax(1024)
+	o.S = xr.ReadStringMax(1024)
 	(&o.C).decodeXDR(xr)
+	_SSSize := int(xr.ReadUint32())
+	if _SSSize > 1024 {
+		return xdr.ElementSizeExceeded("SS", _SSSize, 1024)
+	}
+	o.SS = make([]string, _SSSize)
+	for i := range o.SS {
+		o.SS[i] = xr.ReadString()
+	}
 	return xr.Error()
 }

Godeps/_workspace/src/github.com/calmh/xdr/reader.go

@@ -5,14 +5,12 @@
 package xdr
 
 import (
-	"errors"
+	"fmt"
 	"io"
 	"reflect"
 	"unsafe"
 )
 
-var ErrElementSizeExceeded = errors.New("element size exceeded")
-
 type Reader struct {
 	r   io.Reader
 	err error
@@ -71,7 +69,7 @@ func (r *Reader) ReadBytesMaxInto(max int, dst []byte) []byte {
 		return nil
 	}
 	if max > 0 && l > max {
-		r.err = ErrElementSizeExceeded
+		r.err = ElementSizeExceeded("bytes field", l, max)
 		return nil
 	}
 
@@ -162,3 +160,7 @@ func (r *Reader) Error() error {
 	}
 	return XDRError{"read", r.err}
 }
+
+func ElementSizeExceeded(field string, size, limit int) error {
+	return fmt.Errorf("%s exceeds size limit; %d > %d", field, size, limit)
+}

Godeps/_workspace/src/github.com/calmh/xdr/xdr_test.go

@@ -5,6 +5,7 @@ package xdr
 
 import (
 	"bytes"
+	"strings"
 	"testing"
 	"testing/quick"
 )
@@ -60,7 +61,7 @@ func TestReadBytesMaxInto(t *testing.T) {
 				if read := len(bs); read != tot {
 					t.Errorf("Incorrect read bytes, wrote=%d, buf=%d, max=%d, read=%d", tot, tot+diff, max, read)
 				}
-			} else if r.err != ErrElementSizeExceeded {
+			} else if !strings.Contains(r.err.Error(), "exceeds size") {
 				t.Errorf("Unexpected non-ErrElementSizeExceeded error for wrote=%d, max=%d: %v", tot, max, r.err)
 			}
 		}
@@ -84,7 +85,7 @@ func TestReadStringMax(t *testing.T) {
 				if read != tot {
 					t.Errorf("Incorrect read bytes, wrote=%d, max=%d, read=%d", tot, max, read)
 				}
-			} else if r.err != ErrElementSizeExceeded {
+			} else if !strings.Contains(r.err.Error(), "exceeds size") {
 				t.Errorf("Unexpected non-ErrElementSizeExceeded error for wrote=%d, max=%d, read=%d: %v", tot, max, read, r.err)
 			}
 		}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache.go

@@ -128,7 +128,8 @@ const (
 type nodeState int
 
 const (
-	nodeEffective nodeState = iota
+	nodeZero nodeState = iota
+	nodeEffective
 	nodeEvicted
 	nodeDeleted
 )

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go

@@ -7,10 +7,8 @@
 package cache
 
 import (
-	"fmt"
 	"math/rand"
 	"runtime"
-	"strings"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -225,16 +223,16 @@ func TestLRUCache_Purge(t *testing.T) {
 }
 
 type testingCacheObjectCounter struct {
-	created  uint32
-	released uint32
+	created  uint
+	released uint
 }
 
 func (c *testingCacheObjectCounter) createOne() {
-	atomic.AddUint32(&c.created, 1)
+	c.created++
 }
 
 func (c *testingCacheObjectCounter) releaseOne() {
-	atomic.AddUint32(&c.released, 1)
+	c.released++
 }
 
 type testingCacheObject struct {
@@ -243,17 +241,75 @@ type testingCacheObject struct {
 
 	ns, key uint64
 
-	releaseCalled uint32
+	releaseCalled bool
 }
 
 func (x *testingCacheObject) Release() {
-	if atomic.CompareAndSwapUint32(&x.releaseCalled, 0, 1) {
+	if !x.releaseCalled {
+		x.releaseCalled = true
 		x.cnt.releaseOne()
 	} else {
 		x.t.Errorf("duplicate setfin NS#%d KEY#%s", x.ns, x.key)
 	}
 }
 
+func TestLRUCache_ConcurrentSetGet(t *testing.T) {
+	runtime.GOMAXPROCS(runtime.NumCPU())
+
+	seed := time.Now().UnixNano()
+	t.Logf("seed=%d", seed)
+
+	const (
+		N = 2000000
+		M = 4000
+		C = 3
+	)
+
+	var set, get uint32
+
+	wg := &sync.WaitGroup{}
+	c := NewLRUCache(M / 4)
+	for ni := uint64(0); ni < C; ni++ {
+		r0 := rand.New(rand.NewSource(seed + int64(ni)))
+		r1 := rand.New(rand.NewSource(seed + int64(ni) + 1))
+		ns := c.GetNamespace(ni)
+
+		wg.Add(2)
+		go func(ns Namespace, r *rand.Rand) {
+			for i := 0; i < N; i++ {
+				x := uint64(r.Int63n(M))
+				o := ns.Get(x, func() (int, interface{}) {
+					atomic.AddUint32(&set, 1)
+					return 1, x
+				})
+				if v := o.Value().(uint64); v != x {
+					t.Errorf("#%d invalid value, got=%d", x, v)
+				}
+				o.Release()
+			}
+			wg.Done()
+		}(ns, r0)
+		go func(ns Namespace, r *rand.Rand) {
+			for i := 0; i < N; i++ {
+				x := uint64(r.Int63n(M))
+				o := ns.Get(x, nil)
+				if o != nil {
+					atomic.AddUint32(&get, 1)
+					if v := o.Value().(uint64); v != x {
+						t.Errorf("#%d invalid value, got=%d", x, v)
+					}
+					o.Release()
+				}
+			}
+			wg.Done()
+		}(ns, r1)
+	}
+
+	wg.Wait()
+
+	t.Logf("set=%d get=%d", set, get)
+}
+
 func TestLRUCache_Finalizer(t *testing.T) {
 	const (
 		capacity   = 100
@@ -262,10 +318,6 @@ func TestLRUCache_Finalizer(t *testing.T) {
 		keymax     = 8000
 	)
 
-	runtime.GOMAXPROCS(runtime.NumCPU())
-	defer runtime.GOMAXPROCS(1)
-
-	wg := &sync.WaitGroup{}
 	cnt := &testingCacheObjectCounter{}
 
 	c := NewLRUCache(capacity)
@@ -273,38 +325,40 @@ func TestLRUCache_Finalizer(t *testing.T) {
 	type instance struct {
 		seed       int64
 		rnd        *rand.Rand
-		ns         uint64
-		effective  int32
+		nsid       uint64
+		ns         Namespace
+		effective  int
 		handles    []Handle
 		handlesMap map[uint64]int
 
 		delete          bool
 		purge           bool
 		zap             bool
-		wantDel         int32
-		delfinCalledAll int32
-		delfinCalledEff int32
-		purgefinCalled  int32
+		wantDel         int
+		delfinCalled    int
+		delfinCalledAll int
+		delfinCalledEff int
+		purgefinCalled  int
 	}
 
-	instanceGet := func(p *instance, ns Namespace, key uint64) {
-		h := ns.Get(key, func() (charge int, value interface{}) {
+	instanceGet := func(p *instance, key uint64) {
+		h := p.ns.Get(key, func() (charge int, value interface{}) {
 			to := &testingCacheObject{
 				t: t, cnt: cnt,
-				ns:  p.ns,
+				ns:  p.nsid,
 				key: key,
 			}
-			atomic.AddInt32(&p.effective, 1)
+			p.effective++
 			cnt.createOne()
 			return 1, releaserFunc{func() {
 				to.Release()
-				atomic.AddInt32(&p.effective, -1)
+				p.effective--
 			}, to}
 		})
 		p.handles = append(p.handles, h)
 		p.handlesMap[key] = p.handlesMap[key] + 1
 	}
-	instanceRelease := func(p *instance, ns Namespace, i int) {
+	instanceRelease := func(p *instance, i int) {
 		h := p.handles[i]
 		key := h.Value().(releaserFunc).value.(*testingCacheObject).key
 		if n := p.handlesMap[key]; n == 0 {
@@ -319,55 +373,71 @@ func TestLRUCache_Finalizer(t *testing.T) {
 		p.handles[len(p.handles) : len(p.handles)+1][0] = nil
 	}
 
-	seeds := make([]int64, goroutines)
-	instances := make([]instance, goroutines)
+	seed := time.Now().UnixNano()
+	t.Logf("seed=%d", seed)
+
+	instances := make([]*instance, goroutines)
 	for i := range instances {
-		p := &instances[i]
+		p := &instance{}
 		p.handlesMap = make(map[uint64]int)
-		if seeds[i] == 0 {
-			seeds[i] = time.Now().UnixNano()
-		}
-		p.seed = seeds[i]
+		p.seed = seed + int64(i)
 		p.rnd = rand.New(rand.NewSource(p.seed))
-		p.ns = uint64(i)
+		p.nsid = uint64(i)
+		p.ns = c.GetNamespace(p.nsid)
 		p.delete = i%6 == 0
 		p.purge = i%8 == 0
 		p.zap = i%12 == 0 || i%3 == 0
+		instances[i] = p
 	}
 
-	seedsStr := make([]string, len(seeds))
-	for i, seed := range seeds {
-		seedsStr[i] = fmt.Sprint(seed)
-	}
-	t.Logf("seeds := []int64{%s}", strings.Join(seedsStr, ", "))
-
-	// Get and release.
-	for i := range instances {
-		p := &instances[i]
-
-		wg.Add(1)
-		go func(p *instance) {
-			defer wg.Done()
-
-			ns := c.GetNamespace(p.ns)
-			for i := 0; i < iterations; i++ {
-				if len(p.handles) == 0 || p.rnd.Int()%2 == 0 {
-					instanceGet(p, ns, uint64(p.rnd.Intn(keymax)))
-				} else {
-					instanceRelease(p, ns, p.rnd.Intn(len(p.handles)))
+	runr := rand.New(rand.NewSource(seed - 1))
+	run := func(rnd *rand.Rand, x []*instance, init func(p *instance) bool, fn func(p *instance, i int) bool) {
+		var (
+			rx []*instance
+			rn []int
+		)
+		if init == nil {
+			rx = append([]*instance{}, x...)
+			rn = make([]int, len(x))
+		} else {
+			for _, p := range x {
+				if init(p) {
+					rx = append(rx, p)
+					rn = append(rn, 0)
 				}
 			}
-		}(p)
+		}
+		for len(rx) > 0 {
+			i := rand.Intn(len(rx))
+			if fn(rx[i], rn[i]) {
+				rn[i]++
+			} else {
+				rx = append(rx[:i], rx[i+1:]...)
+				rn = append(rn[:i], rn[i+1:]...)
+			}
+		}
 	}
-	wg.Wait()
+
+	// Get and release.
+	run(runr, instances, nil, func(p *instance, i int) bool {
+		if i < iterations {
+			if len(p.handles) == 0 || p.rnd.Int()%2 == 0 {
+				instanceGet(p, uint64(p.rnd.Intn(keymax)))
+			} else {
+				instanceRelease(p, p.rnd.Intn(len(p.handles)))
+			}
+			return true
+		} else {
+			return false
+		}
+	})
 
 	if used, cap := c.Used(), c.Capacity(); used > cap {
 		t.Errorf("Used > capacity, used=%d cap=%d", used, cap)
 	}
 
 	// Check effective objects.
-	for i := range instances {
-		p := &instances[i]
+	for i, p := range instances {
 		if int(p.effective) < len(p.handlesMap) {
 			t.Errorf("#%d effective objects < acquired handle, eo=%d ah=%d", i, p.effective, len(p.handlesMap))
 		}
@@ -377,103 +447,93 @@ func TestLRUCache_Finalizer(t *testing.T) {
 		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
 	}
 
-	// Delete and purge.
-	for i := range instances {
-		p := &instances[i]
+	// First delete.
+	run(runr, instances, func(p *instance) bool {
 		p.wantDel = p.effective
-
-		wg.Add(1)
-		go func(p *instance) {
-			defer wg.Done()
-
-			ns := c.GetNamespace(p.ns)
-
-			if p.delete {
-				for key := uint64(0); key < keymax; key++ {
-					_, wantExist := p.handlesMap[key]
-					gotExist := ns.Delete(key, func(exist, pending bool) {
-						atomic.AddInt32(&p.delfinCalledAll, 1)
-						if exist {
-							atomic.AddInt32(&p.delfinCalledEff, 1)
-						}
-					})
-					if !gotExist && wantExist {
-						t.Errorf("delete on NS#%d KEY#%d not found", p.ns, key)
-					}
-				}
-
-				var delfinCalled int
-				for key := uint64(0); key < keymax; key++ {
-					func(key uint64) {
-						gotExist := ns.Delete(key, func(exist, pending bool) {
-							if exist && !pending {
-								t.Errorf("delete fin on NS#%d KEY#%d exist and not pending for deletion", p.ns, key)
-							}
-							delfinCalled++
-						})
-						if gotExist {
-							t.Errorf("delete on NS#%d KEY#%d found", p.ns, key)
-						}
-					}(key)
-				}
-				if delfinCalled != keymax {
-					t.Errorf("(2) #%d not all delete fin called, diff=%d", p.ns, keymax-delfinCalled)
+		return p.delete
+	}, func(p *instance, i int) bool {
+		key := uint64(i)
+		if key < keymax {
+			_, wantExist := p.handlesMap[key]
+			gotExist := p.ns.Delete(key, func(exist, pending bool) {
+				p.delfinCalledAll++
+				if exist {
+					p.delfinCalledEff++
 				}
+			})
+			if !gotExist && wantExist {
+				t.Errorf("delete on NS#%d KEY#%d not found", p.nsid, key)
 			}
+			return true
+		} else {
+			return false
+		}
+	})
 
-			if p.purge {
-				ns.Purge(func(ns, key uint64) {
-					atomic.AddInt32(&p.purgefinCalled, 1)
-				})
+	// Second delete.
+	run(runr, instances, func(p *instance) bool {
+		p.delfinCalled = 0
+		return p.delete
+	}, func(p *instance, i int) bool {
+		key := uint64(i)
+		if key < keymax {
+			gotExist := p.ns.Delete(key, func(exist, pending bool) {
+				if exist && !pending {
+					t.Errorf("delete fin on NS#%d KEY#%d exist and not pending for deletion", p.nsid, key)
+				}
+				p.delfinCalled++
+			})
+			if gotExist {
+				t.Errorf("delete on NS#%d KEY#%d found", p.nsid, key)
 			}
-		}(p)
-	}
-	wg.Wait()
+			return true
+		} else {
+			if p.delfinCalled != keymax {
+				t.Errorf("(2) #%d not all delete fin called, diff=%d", p.ns, keymax-p.delfinCalled)
+			}
+			return false
+		}
+	})
+
+	// Purge.
+	run(runr, instances, func(p *instance) bool {
+		return p.purge
+	}, func(p *instance, i int) bool {
+		p.ns.Purge(func(ns, key uint64) {
+			p.purgefinCalled++
+		})
+		return false
+	})
 
 	if want := int(cnt.created - cnt.released); c.Size() != want {
 		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
 	}
 
 	// Release.
-	for i := range instances {
-		p := &instances[i]
-
-		if !p.zap {
-			wg.Add(1)
-			go func(p *instance) {
-				defer wg.Done()
-
-				ns := c.GetNamespace(p.ns)
-				for i := len(p.handles) - 1; i >= 0; i-- {
-					instanceRelease(p, ns, i)
-				}
-			}(p)
+	run(runr, instances, func(p *instance) bool {
+		return !p.zap
+	}, func(p *instance, i int) bool {
+		if len(p.handles) > 0 {
+			instanceRelease(p, len(p.handles)-1)
+			return true
+		} else {
+			return false
 		}
-	}
-	wg.Wait()
+	})
 
 	if want := int(cnt.created - cnt.released); c.Size() != want {
 		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
 	}
 
 	// Zap.
-	for i := range instances {
-		p := &instances[i]
-
-		if p.zap {
-			wg.Add(1)
-			go func(p *instance) {
-				defer wg.Done()
-
-				ns := c.GetNamespace(p.ns)
-				ns.Zap()
-
-				p.handles = nil
-				p.handlesMap = nil
-			}(p)
-		}
-	}
-	wg.Wait()
+	run(runr, instances, func(p *instance) bool {
+		return p.zap
+	}, func(p *instance, i int) bool {
+		p.ns.Zap()
+		p.handles = nil
+		p.handlesMap = nil
+		return false
+	})
 
 	if want := int(cnt.created - cnt.released); c.Size() != want {
 		t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
@@ -485,23 +545,21 @@ func TestLRUCache_Finalizer(t *testing.T) {
 
 	c.Purge(nil)
 
-	for i := range instances {
-		p := &instances[i]
-
+	for _, p := range instances {
 		if p.delete {
 			if p.delfinCalledAll != keymax {
-				t.Errorf("#%d not all delete fin called, purge=%v zap=%v diff=%d", p.ns, p.purge, p.zap, keymax-p.delfinCalledAll)
+				t.Errorf("#%d not all delete fin called, purge=%v zap=%v diff=%d", p.nsid, p.purge, p.zap, keymax-p.delfinCalledAll)
 			}
 			if p.delfinCalledEff != p.wantDel {
-				t.Errorf("#%d not all effective delete fin called, diff=%d", p.ns, p.wantDel-p.delfinCalledEff)
+				t.Errorf("#%d not all effective delete fin called, diff=%d", p.nsid, p.wantDel-p.delfinCalledEff)
 			}
 			if p.purge && p.purgefinCalled > 0 {
-				t.Errorf("#%d some purge fin called, delete=%v zap=%v n=%d", p.ns, p.delete, p.zap, p.purgefinCalled)
+				t.Errorf("#%d some purge fin called, delete=%v zap=%v n=%d", p.nsid, p.delete, p.zap, p.purgefinCalled)
 			}
 		} else {
 			if p.purge {
 				if p.purgefinCalled != p.wantDel {
-					t.Errorf("#%d not all purge fin called, delete=%v zap=%v diff=%d", p.ns, p.delete, p.zap, p.wantDel-p.purgefinCalled)
+					t.Errorf("#%d not all purge fin called, delete=%v zap=%v diff=%d", p.nsid, p.delete, p.zap, p.wantDel-p.purgefinCalled)
 				}
 			}
 		}
@@ -512,6 +570,56 @@ func TestLRUCache_Finalizer(t *testing.T) {
 	}
 }
 
+func BenchmarkLRUCache_Set(b *testing.B) {
+	c := NewLRUCache(0)
+	ns := c.GetNamespace(0)
+	b.ResetTimer()
+	for i := uint64(0); i < uint64(b.N); i++ {
+		set(ns, i, "", 1, nil)
+	}
+}
+
+func BenchmarkLRUCache_Get(b *testing.B) {
+	c := NewLRUCache(0)
+	ns := c.GetNamespace(0)
+	b.ResetTimer()
+	for i := uint64(0); i < uint64(b.N); i++ {
+		set(ns, i, "", 1, nil)
+	}
+	b.ResetTimer()
+	for i := uint64(0); i < uint64(b.N); i++ {
+		ns.Get(i, nil)
+	}
+}
+
+func BenchmarkLRUCache_Get2(b *testing.B) {
+	c := NewLRUCache(0)
+	ns := c.GetNamespace(0)
+	b.ResetTimer()
+	for i := uint64(0); i < uint64(b.N); i++ {
+		set(ns, i, "", 1, nil)
+	}
+	b.ResetTimer()
+	for i := uint64(0); i < uint64(b.N); i++ {
+		ns.Get(i, func() (charge int, value interface{}) {
+			return 0, nil
+		})
+	}
+}
+
+func BenchmarkLRUCache_Release(b *testing.B) {
+	c := NewLRUCache(0)
+	ns := c.GetNamespace(0)
+	handles := make([]Handle, b.N)
+	for i := uint64(0); i < uint64(b.N); i++ {
+		handles[i] = set(ns, i, "", 1, nil)
+	}
+	b.ResetTimer()
+	for _, h := range handles {
+		h.Release()
+	}
+}
+
 func BenchmarkLRUCache_SetRelease(b *testing.B) {
 	capacity := b.N / 100
 	if capacity <= 0 {
@@ -521,7 +629,7 @@ func BenchmarkLRUCache_SetRelease(b *testing.B) {
 	ns := c.GetNamespace(0)
 	b.ResetTimer()
 	for i := uint64(0); i < uint64(b.N); i++ {
-		set(ns, i, nil, 1, nil).Release()
+		set(ns, i, "", 1, nil).Release()
 	}
 }
 
@@ -538,10 +646,10 @@ func BenchmarkLRUCache_SetReleaseTwice(b *testing.B) {
 	nb := b.N - na
 
 	for i := uint64(0); i < uint64(na); i++ {
-		set(ns, i, nil, 1, nil).Release()
+		set(ns, i, "", 1, nil).Release()
 	}
 
 	for i := uint64(0); i < uint64(nb); i++ {
-		set(ns, i, nil, 1, nil).Release()
+		set(ns, i, "", 1, nil).Release()
 	}
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru_cache.go

@@ -13,6 +13,13 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
+// The LLRB implementation were taken from https://github.com/petar/GoLLRB.
+// Which contains the following header:
+//
+// Copyright 2010 Petar Maymounkov. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
 // lruCache represent a LRU cache state.
 type lruCache struct {
 	mu                sync.Mutex
@@ -74,11 +81,7 @@ func (c *lruCache) GetNamespace(id uint64) Namespace {
 		return ns
 	}
 
-	ns := &lruNs{
-		lru:   c,
-		id:    id,
-		table: make(map[uint64]*lruNode),
-	}
+	ns := &lruNs{lru: c, id: id}
 	c.table[id] = ns
 	return ns
 }
@@ -121,6 +124,9 @@ func (c *lruCache) Zap() {
 func (c *lruCache) evict() {
 	top := &c.recent
 	for n := c.recent.rPrev; c.used > c.capacity && n != top; {
+		if n.state != nodeEffective {
+			panic("evicting non effective node")
+		}
 		n.state = nodeEvicted
 		n.rRemove()
 		n.derefNB()
@@ -130,130 +136,271 @@ func (c *lruCache) evict() {
 }
 
 type lruNs struct {
-	lru   *lruCache
-	id    uint64
-	table map[uint64]*lruNode
-	state nsState
+	lru    *lruCache
+	id     uint64
+	rbRoot *lruNode
+	state  nsState
+}
+
+func (ns *lruNs) rbGetOrCreateNode(h *lruNode, key uint64) (hn, n *lruNode) {
+	if h == nil {
+		n = &lruNode{ns: ns, key: key}
+		return n, n
+	}
+
+	if key < h.key {
+		hn, n = ns.rbGetOrCreateNode(h.rbLeft, key)
+		if hn != nil {
+			h.rbLeft = hn
+		} else {
+			return nil, n
+		}
+	} else if key > h.key {
+		hn, n = ns.rbGetOrCreateNode(h.rbRight, key)
+		if hn != nil {
+			h.rbRight = hn
+		} else {
+			return nil, n
+		}
+	} else {
+		return nil, h
+	}
+
+	if rbIsRed(h.rbRight) && !rbIsRed(h.rbLeft) {
+		h = rbRotLeft(h)
+	}
+	if rbIsRed(h.rbLeft) && rbIsRed(h.rbLeft.rbLeft) {
+		h = rbRotRight(h)
+	}
+	if rbIsRed(h.rbLeft) && rbIsRed(h.rbRight) {
+		rbFlip(h)
+	}
+	return h, n
+}
+
+func (ns *lruNs) getOrCreateNode(key uint64) *lruNode {
+	hn, n := ns.rbGetOrCreateNode(ns.rbRoot, key)
+	if hn != nil {
+		ns.rbRoot = hn
+		ns.rbRoot.rbBlack = true
+	}
+	return n
+}
+
+func (ns *lruNs) rbGetNode(key uint64) *lruNode {
+	h := ns.rbRoot
+	for h != nil {
+		switch {
+		case key < h.key:
+			h = h.rbLeft
+		case key > h.key:
+			h = h.rbRight
+		default:
+			return h
+		}
+	}
+	return nil
+}
+
+func (ns *lruNs) getNode(key uint64) *lruNode {
+	return ns.rbGetNode(key)
+}
+
+func (ns *lruNs) rbDeleteNode(h *lruNode, key uint64) *lruNode {
+	if h == nil {
+		return nil
+	}
+
+	if key < h.key {
+		if h.rbLeft == nil { // key not present. Nothing to delete
+			return h
+		}
+		if !rbIsRed(h.rbLeft) && !rbIsRed(h.rbLeft.rbLeft) {
+			h = rbMoveLeft(h)
+		}
+		h.rbLeft = ns.rbDeleteNode(h.rbLeft, key)
+	} else {
+		if rbIsRed(h.rbLeft) {
+			h = rbRotRight(h)
+		}
+		// If @key equals @h.key and no right children at @h
+		if h.key == key && h.rbRight == nil {
+			return nil
+		}
+		if h.rbRight != nil && !rbIsRed(h.rbRight) && !rbIsRed(h.rbRight.rbLeft) {
+			h = rbMoveRight(h)
+		}
+		// If @key equals @h.key, and (from above) 'h.Right != nil'
+		if h.key == key {
+			var x *lruNode
+			h.rbRight, x = rbDeleteMin(h.rbRight)
+			if x == nil {
+				panic("logic")
+			}
+			x.rbLeft, h.rbLeft = h.rbLeft, nil
+			x.rbRight, h.rbRight = h.rbRight, nil
+			x.rbBlack = h.rbBlack
+			h = x
+		} else { // Else, @key is bigger than @h.key
+			h.rbRight = ns.rbDeleteNode(h.rbRight, key)
+		}
+	}
+
+	return rbFixup(h)
+}
+
+func (ns *lruNs) deleteNode(key uint64) {
+	ns.rbRoot = ns.rbDeleteNode(ns.rbRoot, key)
+	if ns.rbRoot != nil {
+		ns.rbRoot.rbBlack = true
+	}
+}
+
+func (ns *lruNs) rbIterateNodes(h *lruNode, pivot uint64, iter func(n *lruNode) bool) bool {
+	if h == nil {
+		return true
+	}
+	if h.key >= pivot {
+		if !ns.rbIterateNodes(h.rbLeft, pivot, iter) {
+			return false
+		}
+		if !iter(h) {
+			return false
+		}
+	}
+	return ns.rbIterateNodes(h.rbRight, pivot, iter)
+}
+
+func (ns *lruNs) iterateNodes(iter func(n *lruNode) bool) {
+	ns.rbIterateNodes(ns.rbRoot, 0, iter)
 }
 
 func (ns *lruNs) Get(key uint64, setf SetFunc) Handle {
 	ns.lru.mu.Lock()
+	defer ns.lru.mu.Unlock()
 
 	if ns.state != nsEffective {
-		ns.lru.mu.Unlock()
 		return nil
 	}
 
-	node, ok := ns.table[key]
-	if ok {
-		switch node.state {
-		case nodeEvicted:
-			// Insert to recent list.
-			node.state = nodeEffective
-			node.ref++
-			ns.lru.used += node.charge
-			ns.lru.evict()
-			fallthrough
-		case nodeEffective:
-			// Bump to front.
-			node.rRemove()
-			node.rInsert(&ns.lru.recent)
-		}
-		node.ref++
-	} else {
-		if setf == nil {
-			ns.lru.mu.Unlock()
+	var n *lruNode
+	if setf == nil {
+		n = ns.getNode(key)
+		if n == nil {
 			return nil
 		}
-
+	} else {
+		n = ns.getOrCreateNode(key)
+	}
+	switch n.state {
+	case nodeZero:
 		charge, value := setf()
 		if value == nil {
-			ns.lru.mu.Unlock()
+			ns.deleteNode(key)
 			return nil
 		}
-
-		node = &lruNode{
-			ns:     ns,
-			key:    key,
-			value:  value,
-			charge: charge,
-			ref:    1,
+		if charge < 0 {
+			charge = 0
 		}
-		ns.table[key] = node
+
+		n.value = value
+		n.charge = charge
+		n.state = nodeEvicted
 
 		ns.lru.size += charge
 		ns.lru.alive++
-		if charge > 0 {
-			node.ref++
-			node.rInsert(&ns.lru.recent)
-			ns.lru.used += charge
-			ns.lru.evict()
-		}
-	}
 
-	ns.lru.mu.Unlock()
-	return &lruHandle{node: node}
+		fallthrough
+	case nodeEvicted:
+		if n.charge == 0 {
+			break
+		}
+
+		// Insert to recent list.
+		n.state = nodeEffective
+		n.ref++
+		ns.lru.used += n.charge
+		ns.lru.evict()
+
+		fallthrough
+	case nodeEffective:
+		// Bump to front.
+		n.rRemove()
+		n.rInsert(&ns.lru.recent)
+	case nodeDeleted:
+		// Do nothing.
+	default:
+		panic("invalid state")
+	}
+	n.ref++
+
+	return &lruHandle{node: n}
 }
 
 func (ns *lruNs) Delete(key uint64, fin DelFin) bool {
 	ns.lru.mu.Lock()
+	defer ns.lru.mu.Unlock()
 
 	if ns.state != nsEffective {
 		if fin != nil {
 			fin(false, false)
 		}
-		ns.lru.mu.Unlock()
 		return false
 	}
 
-	node, exist := ns.table[key]
-	if !exist {
+	n := ns.getNode(key)
+	if n == nil {
 		if fin != nil {
 			fin(false, false)
 		}
-		ns.lru.mu.Unlock()
 		return false
+
 	}
 
-	switch node.state {
+	switch n.state {
+	case nodeEffective:
+		ns.lru.used -= n.charge
+		n.state = nodeDeleted
+		n.delfin = fin
+		n.rRemove()
+		n.derefNB()
+	case nodeEvicted:
+		n.state = nodeDeleted
+		n.delfin = fin
 	case nodeDeleted:
 		if fin != nil {
 			fin(true, true)
 		}
-		ns.lru.mu.Unlock()
 		return false
-	case nodeEffective:
-		ns.lru.used -= node.charge
-		node.state = nodeDeleted
-		node.delfin = fin
-		node.rRemove()
-		node.derefNB()
 	default:
-		node.state = nodeDeleted
-		node.delfin = fin
+		panic("invalid state")
 	}
 
-	ns.lru.mu.Unlock()
 	return true
 }
 
 func (ns *lruNs) purgeNB(fin PurgeFin) {
-	if ns.state != nsEffective {
-		return
-	}
-
-	for _, node := range ns.table {
-		switch node.state {
-		case nodeDeleted:
-		case nodeEffective:
-			ns.lru.used -= node.charge
-			node.state = nodeDeleted
-			node.purgefin = fin
-			node.rRemove()
-			node.derefNB()
-		default:
-			node.state = nodeDeleted
-			node.purgefin = fin
+	if ns.state == nsEffective {
+		var nodes []*lruNode
+		ns.iterateNodes(func(n *lruNode) bool {
+			nodes = append(nodes, n)
+			return true
+		})
+		for _, n := range nodes {
+			switch n.state {
+			case nodeEffective:
+				ns.lru.used -= n.charge
+				n.state = nodeDeleted
+				n.purgefin = fin
+				n.rRemove()
+				n.derefNB()
+			case nodeEvicted:
+				n.state = nodeDeleted
+				n.purgefin = fin
+			case nodeDeleted:
+			default:
+				panic("invalid state")
+			}
 		}
 	}
 }
@@ -265,22 +412,22 @@ func (ns *lruNs) Purge(fin PurgeFin) {
 }
 
 func (ns *lruNs) zapNB() {
-	if ns.state != nsEffective {
-		return
-	}
+	if ns.state == nsEffective {
+		ns.state = nsZapped
 
-	ns.state = nsZapped
+		ns.iterateNodes(func(n *lruNode) bool {
+			if n.state == nodeEffective {
+				ns.lru.used -= n.charge
+				n.rRemove()
+			}
+			ns.lru.size -= n.charge
+			n.state = nodeDeleted
+			n.fin()
 
-	for _, node := range ns.table {
-		if node.state == nodeEffective {
-			ns.lru.used -= node.charge
-			node.rRemove()
-		}
-		ns.lru.size -= node.charge
-		node.state = nodeDeleted
-		node.fin()
+			return true
+		})
+		ns.rbRoot = nil
 	}
-	ns.table = nil
 }
 
 func (ns *lruNs) Zap() {
@@ -293,7 +440,9 @@ func (ns *lruNs) Zap() {
 type lruNode struct {
 	ns *lruNs
 
-	rNext, rPrev *lruNode
+	rNext, rPrev    *lruNode
+	rbLeft, rbRight *lruNode
+	rbBlack         bool
 
 	key      uint64
 	value    interface{}
@@ -330,8 +479,10 @@ func (n *lruNode) fin() {
 		r.Release()
 	}
 	if n.purgefin != nil {
+		if n.delfin != nil {
+			panic("conflicting delete and purge fin")
+		}
 		n.purgefin(n.ns.id, n.key)
-		n.delfin = nil
 		n.purgefin = nil
 	} else if n.delfin != nil {
 		n.delfin(true, false)
@@ -344,7 +495,7 @@ func (n *lruNode) derefNB() {
 	if n.ref == 0 {
 		if n.ns.state == nsEffective {
 			// Remove elemement.
-			delete(n.ns.table, n.key)
+			n.ns.deleteNode(n.key)
 			n.ns.lru.size -= n.charge
 			n.ns.lru.alive--
 			n.fin()
@@ -380,3 +531,92 @@ func (h *lruHandle) Release() {
 	h.node.deref()
 	h.node = nil
 }
+
+func rbIsRed(h *lruNode) bool {
+	if h == nil {
+		return false
+	}
+	return !h.rbBlack
+}
+
+func rbRotLeft(h *lruNode) *lruNode {
+	x := h.rbRight
+	if x.rbBlack {
+		panic("rotating a black link")
+	}
+	h.rbRight = x.rbLeft
+	x.rbLeft = h
+	x.rbBlack = h.rbBlack
+	h.rbBlack = false
+	return x
+}
+
+func rbRotRight(h *lruNode) *lruNode {
+	x := h.rbLeft
+	if x.rbBlack {
+		panic("rotating a black link")
+	}
+	h.rbLeft = x.rbRight
+	x.rbRight = h
+	x.rbBlack = h.rbBlack
+	h.rbBlack = false
+	return x
+}
+
+func rbFlip(h *lruNode) {
+	h.rbBlack = !h.rbBlack
+	h.rbLeft.rbBlack = !h.rbLeft.rbBlack
+	h.rbRight.rbBlack = !h.rbRight.rbBlack
+}
+
+func rbMoveLeft(h *lruNode) *lruNode {
+	rbFlip(h)
+	if rbIsRed(h.rbRight.rbLeft) {
+		h.rbRight = rbRotRight(h.rbRight)
+		h = rbRotLeft(h)
+		rbFlip(h)
+	}
+	return h
+}
+
+func rbMoveRight(h *lruNode) *lruNode {
+	rbFlip(h)
+	if rbIsRed(h.rbLeft.rbLeft) {
+		h = rbRotRight(h)
+		rbFlip(h)
+	}
+	return h
+}
+
+func rbFixup(h *lruNode) *lruNode {
+	if rbIsRed(h.rbRight) {
+		h = rbRotLeft(h)
+	}
+
+	if rbIsRed(h.rbLeft) && rbIsRed(h.rbLeft.rbLeft) {
+		h = rbRotRight(h)
+	}
+
+	if rbIsRed(h.rbLeft) && rbIsRed(h.rbRight) {
+		rbFlip(h)
+	}
+
+	return h
+}
+
+func rbDeleteMin(h *lruNode) (hn, n *lruNode) {
+	if h == nil {
+		return nil, nil
+	}
+	if h.rbLeft == nil {
+		return nil, h
+	}
+
+	if !rbIsRed(h.rbLeft) && !rbIsRed(h.rbLeft.rbLeft) {
+		h = rbMoveLeft(h)
+	}
+
+	h.rbLeft, n = rbDeleteMin(h.rbLeft)
+
+	return rbFixup(h), n
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go

@@ -260,7 +260,7 @@ func (db *DB) memCompaction() {
 		return c.commit(db.journalFile.Num(), db.frozenSeq)
 	}, nil)
 
-	db.logf("mem@flush commited F·%d T·%v", len(c.rec.addedTables), stats.duration)
+	db.logf("mem@flush committed F·%d T·%v", len(c.rec.addedTables), stats.duration)
 
 	for _, r := range c.rec.addedTables {
 		stats.write += r.size
@@ -478,7 +478,7 @@ func (db *DB) tableCompaction(c *compaction, noTrivial bool) {
 	}, nil)
 
 	resultSize := int(stats[1].write)
-	db.logf("table@compaction commited F%s S%s D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), dropCnt, stats[1].duration)
+	db.logf("table@compaction committed F%s S%s D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), dropCnt, stats[1].duration)
 
 	// Save compaction stats
 	for i := range stats {

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_iter.go

@@ -321,9 +321,13 @@ func (i *dbIter) Release() {
 }
 
 func (i *dbIter) SetReleaser(releaser util.Releaser) {
-	if i.dir != dirReleased {
-		i.releaser = releaser
+	if i.dir == dirReleased {
+		panic(util.ErrReleased)
 	}
+	if i.releaser != nil && releaser != nil {
+		panic(util.ErrHasReleaser)
+	}
+	i.releaser = releaser
 }
 
 func (i *dbIter) Error() error {

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_snapshot.go

@@ -71,7 +71,7 @@ func (db *DB) releaseSnapshot(elem *snapshotElement) {
 func (db *DB) minSeq() uint64 {
 	db.snapsMu.Lock()
 	defer db.snapsMu.Unlock()
-	elem := db.snapsRoot.prev
+	elem := db.snapsRoot.next
 	if elem != &db.snapsRoot {
 		return elem.seq
 	}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_test.go

@@ -1125,6 +1125,45 @@ func TestDb_Snapshot(t *testing.T) {
 	})
 }
 
+func TestDb_SnapshotList(t *testing.T) {
+	db := &DB{}
+	db.initSnapshot()
+	e0a := db.acquireSnapshot()
+	e0b := db.acquireSnapshot()
+	db.seq = 1
+	e1 := db.acquireSnapshot()
+	db.seq = 2
+	e2 := db.acquireSnapshot()
+
+	if db.minSeq() != 0 {
+		t.Fatalf("invalid sequence number, got=%d", db.minSeq())
+	}
+	db.releaseSnapshot(e0a)
+	if db.minSeq() != 0 {
+		t.Fatalf("invalid sequence number, got=%d", db.minSeq())
+	}
+	db.releaseSnapshot(e2)
+	if db.minSeq() != 0 {
+		t.Fatalf("invalid sequence number, got=%d", db.minSeq())
+	}
+	db.releaseSnapshot(e0b)
+	if db.minSeq() != 1 {
+		t.Fatalf("invalid sequence number, got=%d", db.minSeq())
+	}
+	e2 = db.acquireSnapshot()
+	if db.minSeq() != 1 {
+		t.Fatalf("invalid sequence number, got=%d", db.minSeq())
+	}
+	db.releaseSnapshot(e1)
+	if db.minSeq() != 2 {
+		t.Fatalf("invalid sequence number, got=%d", db.minSeq())
+	}
+	db.releaseSnapshot(e2)
+	if db.minSeq() != 2 {
+		t.Fatalf("invalid sequence number, got=%d", db.minSeq())
+	}
+}
+
 func TestDb_HiddenValuesAreRemoved(t *testing.T) {
 	trun(t, func(h *dbHarness) {
 		s := h.db.s
@@ -1210,7 +1249,7 @@ func TestDb_DeletionMarkers2(t *testing.T) {
 }
 
 func TestDb_CompactionTableOpenError(t *testing.T) {
-	h := newDbHarnessWopt(t, &opt.Options{MaxOpenFiles: 0})
+	h := newDbHarnessWopt(t, &opt.Options{CachedOpenFiles: -1})
 	defer h.close()
 
 	im := 10
@@ -1577,11 +1616,7 @@ func TestDb_BloomFilter(t *testing.T) {
 		return fmt.Sprintf("key%06d", i)
 	}
 
-	const (
-		n              = 10000
-		indexOverheat  = 19898
-		filterOverheat = 19799
-	)
+	const n = 10000
 
 	// Populate multiple layers
 	for i := 0; i < n; i++ {
@@ -1605,7 +1640,7 @@ func TestDb_BloomFilter(t *testing.T) {
 	cnt := int(h.stor.ReadCounter())
 	t.Logf("lookup of %d present keys yield %d sstable I/O reads", n, cnt)
 
-	if min, max := n+indexOverheat+filterOverheat, n+indexOverheat+filterOverheat+2*n/100; cnt < min || cnt > max {
+	if min, max := n, n+2*n/100; cnt < min || cnt > max {
 		t.Errorf("num of sstable I/O reads of present keys not in range of %d - %d, got %d", min, max, cnt)
 	}
 
@@ -1616,7 +1651,7 @@ func TestDb_BloomFilter(t *testing.T) {
 	}
 	cnt = int(h.stor.ReadCounter())
 	t.Logf("lookup of %d missing keys yield %d sstable I/O reads", n, cnt)
-	if max := 3*n/100 + indexOverheat + filterOverheat; cnt > max {
+	if max := 3 * n / 100; cnt > max {
 		t.Errorf("num of sstable I/O reads of missing keys was more than %d, got %d", max, cnt)
 	}
 
@@ -1888,3 +1923,102 @@ func TestDb_LeveldbIssue200(t *testing.T) {
 	iter.Next()
 	assertBytes(t, []byte("5"), iter.Key())
 }
+
+func TestDb_GoleveldbIssue74(t *testing.T) {
+	h := newDbHarnessWopt(t, &opt.Options{
+		WriteBuffer: 1 * opt.MiB,
+	})
+	defer h.close()
+
+	const n, dur = 10000, 5 * time.Second
+
+	runtime.GOMAXPROCS(runtime.NumCPU())
+
+	until := time.Now().Add(dur)
+	wg := new(sync.WaitGroup)
+	wg.Add(2)
+	var done uint32
+	go func() {
+		var i int
+		defer func() {
+			t.Logf("WRITER DONE #%d", i)
+			atomic.StoreUint32(&done, 1)
+			wg.Done()
+		}()
+
+		b := new(Batch)
+		for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ {
+			iv := fmt.Sprintf("VAL%010d", i)
+			for k := 0; k < n; k++ {
+				key := fmt.Sprintf("KEY%06d", k)
+				b.Put([]byte(key), []byte(key+iv))
+				b.Put([]byte(fmt.Sprintf("PTR%06d", k)), []byte(key))
+			}
+			h.write(b)
+
+			b.Reset()
+			snap := h.getSnapshot()
+			iter := snap.NewIterator(util.BytesPrefix([]byte("PTR")), nil)
+			var k int
+			for ; iter.Next(); k++ {
+				ptrKey := iter.Key()
+				key := iter.Value()
+
+				if _, err := snap.Get(ptrKey, nil); err != nil {
+					t.Fatalf("WRITER #%d snapshot.Get %q: %v", i, ptrKey, err)
+				}
+				if value, err := snap.Get(key, nil); err != nil {
+					t.Fatalf("WRITER #%d snapshot.Get %q: %v", i, key, err)
+				} else if string(value) != string(key)+iv {
+					t.Fatalf("WRITER #%d snapshot.Get %q got invalid value, want %q got %q", i, key, string(key)+iv, value)
+				}
+
+				b.Delete(key)
+				b.Delete(ptrKey)
+			}
+			h.write(b)
+			iter.Release()
+			snap.Release()
+			if k != n {
+				t.Fatalf("#%d %d != %d", i, k, n)
+			}
+		}
+		t.Logf("writer done after %d iterations", i)
+	}()
+	go func() {
+		var i int
+		defer func() {
+			t.Logf("READER DONE #%d", i)
+			atomic.StoreUint32(&done, 1)
+			wg.Done()
+		}()
+		for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ {
+			snap := h.getSnapshot()
+			iter := snap.NewIterator(util.BytesPrefix([]byte("PTR")), nil)
+			var prevValue string
+			var k int
+			for ; iter.Next(); k++ {
+				ptrKey := iter.Key()
+				key := iter.Value()
+
+				if _, err := snap.Get(ptrKey, nil); err != nil {
+					t.Fatalf("READER #%d snapshot.Get %q: %v", i, ptrKey, err)
+				}
+
+				if value, err := snap.Get(key, nil); err != nil {
+					t.Fatalf("READER #%d snapshot.Get %q: %v", i, key, err)
+				} else if prevValue != "" && string(value) != string(key)+prevValue {
+					t.Fatalf("READER #%d snapshot.Get %q got invalid value, want %q got %q", i, key, string(key)+prevValue, value)
+				} else {
+					prevValue = string(value[len(key):])
+				}
+			}
+			iter.Release()
+			snap.Release()
+			if k > 0 && k != n {
+				t.Fatalf("#%d %d != %d", i, k, n)
+			}
+		}
+	}()
+	wg.Wait()
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/external_test.go

@@ -21,7 +21,7 @@ var _ = testutil.Defer(func() {
 			BlockRestartInterval: 5,
 			BlockSize:            50,
 			Compression:          opt.NoCompression,
-			MaxOpenFiles:         0,
+			CachedOpenFiles:      -1,
 			Strict:               opt.StrictAll,
 			WriteBuffer:          1000,
 		}
@@ -40,18 +40,17 @@ var _ = testutil.Defer(func() {
 		})
 
 		Describe("read test", func() {
-			testutil.AllKeyValueTesting(nil, func(kv testutil.KeyValue) testutil.DB {
+			testutil.AllKeyValueTesting(nil, nil, func(kv testutil.KeyValue) testutil.DB {
 				// Building the DB.
 				db := newTestingDB(o, nil, nil)
 				kv.IterateShuffled(nil, func(i int, key, value []byte) {
 					err := db.TestPut(key, value)
 					Expect(err).NotTo(HaveOccurred())
 				})
-				testutil.Defer("teardown", func() {
-					db.TestClose()
-				})
 
 				return db
+			}, func(db testutil.DB) {
+				db.(*testingDB).TestClose()
 			})
 		})
 	})

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/array_iter.go

@@ -40,13 +40,19 @@ type basicArrayIterator struct {
 	util.BasicReleaser
 	array BasicArray
 	pos   int
+	err   error
 }
 
 func (i *basicArrayIterator) Valid() bool {
-	return i.pos >= 0 && i.pos < i.array.Len()
+	return i.pos >= 0 && i.pos < i.array.Len() && !i.Released()
 }
 
 func (i *basicArrayIterator) First() bool {
+	if i.Released() {
+		i.err = ErrIterReleased
+		return false
+	}
+
 	if i.array.Len() == 0 {
 		i.pos = -1
 		return false
@@ -56,6 +62,11 @@ func (i *basicArrayIterator) First() bool {
 }
 
 func (i *basicArrayIterator) Last() bool {
+	if i.Released() {
+		i.err = ErrIterReleased
+		return false
+	}
+
 	n := i.array.Len()
 	if n == 0 {
 		i.pos = 0
@@ -66,6 +77,11 @@ func (i *basicArrayIterator) Last() bool {
 }
 
 func (i *basicArrayIterator) Seek(key []byte) bool {
+	if i.Released() {
+		i.err = ErrIterReleased
+		return false
+	}
+
 	n := i.array.Len()
 	if n == 0 {
 		i.pos = 0
@@ -79,6 +95,11 @@ func (i *basicArrayIterator) Seek(key []byte) bool {
 }
 
 func (i *basicArrayIterator) Next() bool {
+	if i.Released() {
+		i.err = ErrIterReleased
+		return false
+	}
+
 	i.pos++
 	if n := i.array.Len(); i.pos >= n {
 		i.pos = n
@@ -88,6 +109,11 @@ func (i *basicArrayIterator) Next() bool {
 }
 
 func (i *basicArrayIterator) Prev() bool {
+	if i.Released() {
+		i.err = ErrIterReleased
+		return false
+	}
+
 	i.pos--
 	if i.pos < 0 {
 		i.pos = -1
@@ -96,7 +122,7 @@ func (i *basicArrayIterator) Prev() bool {
 	return true
 }
 
-func (i *basicArrayIterator) Error() error { return nil }
+func (i *basicArrayIterator) Error() error { return i.err }
 
 type arrayIterator struct {
 	basicArrayIterator

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter.go

@@ -26,9 +26,10 @@ type indexedIterator struct {
 	strict    bool
 	strictGet bool
 
-	data Iterator
-	err  error
-	errf func(err error)
+	data   Iterator
+	err    error
+	errf   func(err error)
+	closed bool
 }
 
 func (i *indexedIterator) setData() {
@@ -50,6 +51,15 @@ func (i *indexedIterator) clearData() {
 	i.data = nil
 }
 
+func (i *indexedIterator) indexErr() {
+	if err := i.index.Error(); err != nil {
+		if i.errf != nil {
+			i.errf(err)
+		}
+		i.err = err
+	}
+}
+
 func (i *indexedIterator) dataErr() bool {
 	if i.errf != nil {
 		if err := i.data.Error(); err != nil {
@@ -72,9 +82,13 @@ func (i *indexedIterator) Valid() bool {
 func (i *indexedIterator) First() bool {
 	if i.err != nil {
 		return false
+	} else if i.Released() {
+		i.err = ErrIterReleased
+		return false
 	}
 
 	if !i.index.First() {
+		i.indexErr()
 		i.clearData()
 		return false
 	}
@@ -85,9 +99,13 @@ func (i *indexedIterator) First() bool {
 func (i *indexedIterator) Last() bool {
 	if i.err != nil {
 		return false
+	} else if i.Released() {
+		i.err = ErrIterReleased
+		return false
 	}
 
 	if !i.index.Last() {
+		i.indexErr()
 		i.clearData()
 		return false
 	}
@@ -105,9 +123,13 @@ func (i *indexedIterator) Last() bool {
 func (i *indexedIterator) Seek(key []byte) bool {
 	if i.err != nil {
 		return false
+	} else if i.Released() {
+		i.err = ErrIterReleased
+		return false
 	}
 
 	if !i.index.Seek(key) {
+		i.indexErr()
 		i.clearData()
 		return false
 	}
@@ -125,6 +147,9 @@ func (i *indexedIterator) Seek(key []byte) bool {
 func (i *indexedIterator) Next() bool {
 	if i.err != nil {
 		return false
+	} else if i.Released() {
+		i.err = ErrIterReleased
+		return false
 	}
 
 	switch {
@@ -136,6 +161,7 @@ func (i *indexedIterator) Next() bool {
 		fallthrough
 	case i.data == nil:
 		if !i.index.Next() {
+			i.indexErr()
 			return false
 		}
 		i.setData()
@@ -147,6 +173,9 @@ func (i *indexedIterator) Next() bool {
 func (i *indexedIterator) Prev() bool {
 	if i.err != nil {
 		return false
+	} else if i.Released() {
+		i.err = ErrIterReleased
+		return false
 	}
 
 	switch {
@@ -158,6 +187,7 @@ func (i *indexedIterator) Prev() bool {
 		fallthrough
 	case i.data == nil:
 		if !i.index.Prev() {
+			i.indexErr()
 			return false
 		}
 		i.setData()

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/iter.go

@@ -14,6 +14,10 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
+var (
+	ErrIterReleased = errors.New("leveldb/iterator: iterator released")
+)
+
 // IteratorSeeker is the interface that wraps the 'seeks method'.
 type IteratorSeeker interface {
 	// First moves the iterator to the first key/value pair. If the iterator
@@ -100,28 +104,13 @@ type ErrorCallbackSetter interface {
 }
 
 type emptyIterator struct {
-	releaser util.Releaser
-	released bool
-	err      error
+	util.BasicReleaser
+	err error
 }
 
 func (i *emptyIterator) rErr() {
-	if i.err == nil && i.released {
-		i.err = errors.New("leveldb/iterator: iterator released")
-	}
-}
-
-func (i *emptyIterator) Release() {
-	if i.releaser != nil {
-		i.releaser.Release()
-		i.releaser = nil
-	}
-	i.released = true
-}
-
-func (i *emptyIterator) SetReleaser(releaser util.Releaser) {
-	if !i.released {
-		i.releaser = releaser
+	if i.err == nil && i.Released() {
+		i.err = ErrIterReleased
 	}
 }
 

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter.go

@@ -7,16 +7,10 @@
 package iterator
 
 import (
-	"errors"
-
 	"github.com/syndtr/goleveldb/leveldb/comparer"
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
-var (
-	ErrIterReleased = errors.New("leveldb/iterator: iterator released")
-)
-
 type dir int
 
 const (
@@ -274,9 +268,13 @@ func (i *mergedIterator) Release() {
 }
 
 func (i *mergedIterator) SetReleaser(releaser util.Releaser) {
-	if i.dir != dirReleased {
-		i.releaser = releaser
+	if i.dir == dirReleased {
+		panic(util.ErrReleased)
 	}
+	if i.releaser != nil && releaser != nil {
+		panic(util.ErrHasReleaser)
+	}
+	i.releaser = releaser
 }
 
 func (i *mergedIterator) Error() error {

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go

@@ -8,6 +8,7 @@
 package memdb
 
 import (
+	"errors"
 	"math/rand"
 	"sync"
 
@@ -17,7 +18,8 @@ import (
 )
 
 var (
-	ErrNotFound = util.ErrNotFound
+	ErrNotFound     = util.ErrNotFound
+	ErrIterReleased = errors.New("leveldb/memdb: iterator released")
 )
 
 const tMaxHeight = 12
@@ -29,6 +31,7 @@ type dbIter struct {
 	node       int
 	forward    bool
 	key, value []byte
+	err        error
 }
 
 func (i *dbIter) fill(checkStart, checkLimit bool) bool {
@@ -59,6 +62,11 @@ func (i *dbIter) Valid() bool {
 }
 
 func (i *dbIter) First() bool {
+	if i.Released() {
+		i.err = ErrIterReleased
+		return false
+	}
+
 	i.forward = true
 	i.p.mu.RLock()
 	defer i.p.mu.RUnlock()
@@ -71,9 +79,11 @@ func (i *dbIter) First() bool {
 }
 
 func (i *dbIter) Last() bool {
-	if i.p == nil {
+	if i.Released() {
+		i.err = ErrIterReleased
 		return false
 	}
+
 	i.forward = false
 	i.p.mu.RLock()
 	defer i.p.mu.RUnlock()
@@ -86,9 +96,11 @@ func (i *dbIter) Last() bool {
 }
 
 func (i *dbIter) Seek(key []byte) bool {
-	if i.p == nil {
+	if i.Released() {
+		i.err = ErrIterReleased
 		return false
 	}
+
 	i.forward = true
 	i.p.mu.RLock()
 	defer i.p.mu.RUnlock()
@@ -100,9 +112,11 @@ func (i *dbIter) Seek(key []byte) bool {
 }
 
 func (i *dbIter) Next() bool {
-	if i.p == nil {
+	if i.Released() {
+		i.err = ErrIterReleased
 		return false
 	}
+
 	if i.node == 0 {
 		if !i.forward {
 			return i.First()
@@ -117,9 +131,11 @@ func (i *dbIter) Next() bool {
 }
 
 func (i *dbIter) Prev() bool {
-	if i.p == nil {
+	if i.Released() {
+		i.err = ErrIterReleased
 		return false
 	}
+
 	if i.node == 0 {
 		if i.forward {
 			return i.Last()
@@ -141,10 +157,10 @@ func (i *dbIter) Value() []byte {
 	return i.value
 }
 
-func (i *dbIter) Error() error { return nil }
+func (i *dbIter) Error() error { return i.err }
 
 func (i *dbIter) Release() {
-	if i.p != nil {
+	if !i.Released() {
 		i.p = nil
 		i.node = 0
 		i.key = nil

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb_test.go

@@ -129,7 +129,7 @@ var _ = testutil.Defer(func() {
 				}
 
 				return db
-			})
+			}, nil, nil)
 		})
 	})
 })

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go

@@ -24,7 +24,7 @@ const (
 	DefaultBlockRestartInterval = 16
 	DefaultBlockSize            = 4 * KiB
 	DefaultCompressionType      = SnappyCompression
-	DefaultMaxOpenFiles         = 1000
+	DefaultCachedOpenFiles      = 500
 	DefaultWriteBuffer          = 4 * MiB
 )
 
@@ -125,6 +125,13 @@ type Options struct {
 	// The default value is 4KiB.
 	BlockSize int
 
+	// CachedOpenFiles defines number of open files to kept around when not
+	// in-use, the counting includes still in-use files.
+	// Set this to negative value to disable caching.
+	//
+	// The default value is 500.
+	CachedOpenFiles int
+
 	// Comparer defines a total ordering over the space of []byte keys: a 'less
 	// than' relationship. The same comparison algorithm must be used for reads
 	// and writes over the lifetime of the DB.
@@ -165,13 +172,6 @@ type Options struct {
 	// The default value is nil.
 	Filter filter.Filter
 
-	// MaxOpenFiles defines maximum number of open files to kept around
-	// (cached). This is not an hard limit, actual open files may exceed
-	// the defined value.
-	//
-	// The default value is 1000.
-	MaxOpenFiles int
-
 	// Strict defines the DB strict level.
 	Strict Strict
 
@@ -213,6 +213,15 @@ func (o *Options) GetBlockSize() int {
 	return o.BlockSize
 }
 
+func (o *Options) GetCachedOpenFiles() int {
+	if o == nil || o.CachedOpenFiles == 0 {
+		return DefaultCachedOpenFiles
+	} else if o.CachedOpenFiles < 0 {
+		return 0
+	}
+	return o.CachedOpenFiles
+}
+
 func (o *Options) GetComparer() comparer.Comparer {
 	if o == nil || o.Comparer == nil {
 		return comparer.DefaultComparer
@@ -248,13 +257,6 @@ func (o *Options) GetFilter() filter.Filter {
 	return o.Filter
 }
 
-func (o *Options) GetMaxOpenFiles() int {
-	if o == nil || o.MaxOpenFiles <= 0 {
-		return DefaultMaxOpenFiles
-	}
-	return o.MaxOpenFiles
-}
-
 func (o *Options) GetStrict(strict Strict) bool {
 	if o == nil || o.Strict == 0 {
 		return DefaultStrict&strict != 0

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go

@@ -58,7 +58,7 @@ func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) {
 		storLock: storLock,
 	}
 	s.setOptions(o)
-	s.tops = newTableOps(s, s.o.GetMaxOpenFiles())
+	s.tops = newTableOps(s, s.o.GetCachedOpenFiles())
 	s.setVersion(&version{s: s})
 	s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock D·DeletedEntry L·Level Q·SeqNum T·TimeElapsed")
 	return

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go

@@ -147,7 +147,7 @@ func (s *session) fillRecord(r *sessionRecord, snapshot bool) {
 	}
 }
 
-// Mark if record has been commited, this will update session state;
+// Mark if record has been committed, this will update session state;
 // need external synchronization.
 func (s *session) recordCommited(r *sessionRecord) {
 	if r.has(recJournalNum) {

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go

@@ -7,7 +7,9 @@
 package leveldb
 
 import (
+	"fmt"
 	"sort"
+	"sync"
 	"sync/atomic"
 
 	"github.com/syndtr/goleveldb/leveldb/cache"
@@ -276,6 +278,8 @@ type tOps struct {
 	cache   cache.Cache
 	cacheNS cache.Namespace
 	bpool   *util.BufferPool
+	mu      sync.Mutex
+	closed  bool
 }
 
 // Creates an empty table and returns table writer.
@@ -322,9 +326,42 @@ func (t *tOps) createFrom(src iterator.Iterator) (f *tFile, n int, err error) {
 	return
 }
 
+type trWrapper struct {
+	*table.Reader
+	t   *tOps
+	ref int
+}
+
+func (w *trWrapper) Release() {
+	if w.ref != 0 && !w.t.closed {
+		panic(fmt.Sprintf("BUG: invalid ref %d, refer to issue #72", w.ref))
+	}
+	w.Reader.Release()
+}
+
+type trCacheHandleWrapper struct {
+	cache.Handle
+	t        *tOps
+	released bool
+}
+
+func (w *trCacheHandleWrapper) Release() {
+	w.t.mu.Lock()
+	defer w.t.mu.Unlock()
+
+	if !w.released {
+		w.released = true
+		w.Value().(*trWrapper).ref--
+	}
+	w.Handle.Release()
+}
+
 // Opens table. It returns a cache handle, which should
 // be released after use.
 func (t *tOps) open(f *tFile) (ch cache.Handle, err error) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
 	num := f.file.Num()
 	ch = t.cacheNS.Get(num, func() (charge int, value interface{}) {
 		var r storage.Reader
@@ -337,11 +374,13 @@ func (t *tOps) open(f *tFile) (ch cache.Handle, err error) {
 		if bc := t.s.o.GetBlockCache(); bc != nil {
 			bcacheNS = bc.GetNamespace(num)
 		}
-		return 1, table.NewReader(r, int64(f.size), bcacheNS, t.bpool, t.s.o)
+		return 1, &trWrapper{table.NewReader(r, int64(f.size), bcacheNS, t.bpool, t.s.o), t, 0}
 	})
 	if ch == nil && err == nil {
 		err = ErrClosed
 	}
+	ch.Value().(*trWrapper).ref++
+	ch = &trCacheHandleWrapper{ch, t, false}
 	return
 }
 
@@ -353,7 +392,7 @@ func (t *tOps) find(f *tFile, key []byte, ro *opt.ReadOptions) (rkey, rvalue []b
 		return nil, nil, err
 	}
 	defer ch.Release()
-	return ch.Value().(*table.Reader).Find(key, ro)
+	return ch.Value().(*trWrapper).Find(key, ro)
 }
 
 // Returns approximate offset of the given key.
@@ -363,7 +402,7 @@ func (t *tOps) offsetOf(f *tFile, key []byte) (offset uint64, err error) {
 		return
 	}
 	defer ch.Release()
-	offset_, err := ch.Value().(*table.Reader).OffsetOf(key)
+	offset_, err := ch.Value().(*trWrapper).OffsetOf(key)
 	return uint64(offset_), err
 }
 
@@ -373,7 +412,7 @@ func (t *tOps) newIterator(f *tFile, slice *util.Range, ro *opt.ReadOptions) ite
 	if err != nil {
 		return iterator.NewEmptyIterator(err)
 	}
-	iter := ch.Value().(*table.Reader).NewIterator(slice, ro)
+	iter := ch.Value().(*trWrapper).NewIterator(slice, ro)
 	iter.SetReleaser(ch)
 	return iter
 }
@@ -381,6 +420,9 @@ func (t *tOps) newIterator(f *tFile, slice *util.Range, ro *opt.ReadOptions) ite
 // Removes table from persistent storage. It waits until
 // no one use the the table.
 func (t *tOps) remove(f *tFile) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
 	num := f.file.Num()
 	t.cacheNS.Delete(num, func(exist, pending bool) {
 		if !pending {
@@ -399,6 +441,10 @@ func (t *tOps) remove(f *tFile) {
 // Closes the table ops instance. It will close all tables,
 // regadless still used or not.
 func (t *tOps) close() {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	t.closed = true
 	t.cache.Zap()
 	t.bpool.Close()
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/block_test.go

@@ -59,7 +59,7 @@ var _ = testutil.Defer(func() {
 						// Make block.
 						br := Build(kv, restartInterval)
 						// Do testing.
-						testutil.KeyValueTesting(nil, br, kv.Clone())
+						testutil.KeyValueTesting(nil, kv.Clone(), br, nil, nil)
 					}
 
 					Describe(Text(), Test)

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/reader.go

@@ -13,6 +13,7 @@ import (
 	"io"
 	"sort"
 	"strings"
+	"sync"
 
 	"code.google.com/p/snappy-go/snappy"
 
@@ -25,8 +26,9 @@ import (
 )
 
 var (
-	ErrNotFound     = util.ErrNotFound
-	ErrIterReleased = errors.New("leveldb/table: iterator released")
+	ErrNotFound       = util.ErrNotFound
+	ErrReaderReleased = errors.New("leveldb/table: reader released")
+	ErrIterReleased   = errors.New("leveldb/table: iterator released")
 )
 
 func max(x, y int) int {
@@ -134,9 +136,7 @@ func (b *block) newIterator(slice *util.Range, inclLimit bool, cache util.Releas
 }
 
 func (b *block) Release() {
-	if b.tr.bpool != nil {
-		b.tr.bpool.Put(b.data)
-	}
+	b.tr.bpool.Put(b.data)
 	b.tr = nil
 	b.data = nil
 }
@@ -439,7 +439,7 @@ func (i *blockIter) Value() []byte {
 }
 
 func (i *blockIter) Release() {
-	if i.dir > dirReleased {
+	if i.dir != dirReleased {
 		i.block = nil
 		i.prevNode = nil
 		i.prevKeys = nil
@@ -458,9 +458,13 @@ func (i *blockIter) Release() {
 }
 
 func (i *blockIter) SetReleaser(releaser util.Releaser) {
-	if i.dir > dirReleased {
-		i.releaser = releaser
+	if i.dir == dirReleased {
+		panic(util.ErrReleased)
 	}
+	if i.releaser != nil && releaser != nil {
+		panic(util.ErrHasReleaser)
+	}
+	i.releaser = releaser
 }
 
 func (i *blockIter) Valid() bool {
@@ -495,9 +499,7 @@ func (b *filterBlock) contains(offset uint64, key []byte) bool {
 }
 
 func (b *filterBlock) Release() {
-	if b.tr.bpool != nil {
-		b.tr.bpool.Put(b.data)
-	}
+	b.tr.bpool.Put(b.data)
 	b.tr = nil
 	b.data = nil
 }
@@ -519,15 +521,17 @@ func (i *indexIter) Get() iterator.Iterator {
 	if n == 0 {
 		return iterator.NewEmptyIterator(errors.New("leveldb/table: Reader: invalid table (bad data block handle)"))
 	}
+
 	var slice *util.Range
 	if i.slice != nil && (i.blockIter.isFirst() || i.blockIter.isLast()) {
 		slice = i.slice
 	}
-	return i.blockIter.block.tr.getDataIter(dataBH, slice, i.checksum, i.fillCache)
+	return i.blockIter.block.tr.getDataIterErr(dataBH, slice, i.checksum, i.fillCache)
 }
 
 // Reader is a table reader.
 type Reader struct {
+	mu     sync.RWMutex
 	reader io.ReaderAt
 	cache  cache.Namespace
 	err    error
@@ -540,6 +544,8 @@ type Reader struct {
 
 	dataEnd           int64
 	indexBH, filterBH blockHandle
+	indexBlock        *block
+	filterBlock       *filterBlock
 }
 
 func verifyChecksum(data []byte) bool {
@@ -688,6 +694,20 @@ func (r *Reader) readFilterBlockCached(bh blockHandle, fillCache bool) (*filterB
 	return b, b, err
 }
 
+func (r *Reader) getIndexBlock(fillCache bool) (b *block, rel util.Releaser, err error) {
+	if r.indexBlock == nil {
+		return r.readBlockCached(r.indexBH, true, fillCache)
+	}
+	return r.indexBlock, util.NoopReleaser{}, nil
+}
+
+func (r *Reader) getFilterBlock(fillCache bool) (*filterBlock, util.Releaser, error) {
+	if r.filterBlock == nil {
+		return r.readFilterBlockCached(r.filterBH, fillCache)
+	}
+	return r.filterBlock, util.NoopReleaser{}, nil
+}
+
 func (r *Reader) getDataIter(dataBH blockHandle, slice *util.Range, checksum, fillCache bool) iterator.Iterator {
 	b, rel, err := r.readBlockCached(dataBH, checksum, fillCache)
 	if err != nil {
@@ -696,6 +716,17 @@ func (r *Reader) getDataIter(dataBH blockHandle, slice *util.Range, checksum, fi
 	return b.newIterator(slice, false, rel)
 }
 
+func (r *Reader) getDataIterErr(dataBH blockHandle, slice *util.Range, checksum, fillCache bool) iterator.Iterator {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
+	if r.err != nil {
+		return iterator.NewEmptyIterator(r.err)
+	}
+
+	return r.getDataIter(dataBH, slice, checksum, fillCache)
+}
+
 // NewIterator creates an iterator from the table.
 //
 // Slice allows slicing the iterator to only contains keys in the given
@@ -708,22 +739,25 @@ func (r *Reader) getDataIter(dataBH blockHandle, slice *util.Range, checksum, fi
 //
 // Also read Iterator documentation of the leveldb/iterator package.
 func (r *Reader) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
 	if r.err != nil {
 		return iterator.NewEmptyIterator(r.err)
 	}
 
 	fillCache := !ro.GetDontFillCache()
-	b, rel, err := r.readBlockCached(r.indexBH, true, fillCache)
+	indexBlock, rel, err := r.getIndexBlock(fillCache)
 	if err != nil {
 		return iterator.NewEmptyIterator(err)
 	}
 	index := &indexIter{
-		blockIter: b.newIterator(slice, true, rel),
+		blockIter: indexBlock.newIterator(slice, true, rel),
 		slice:     slice,
 		checksum:  ro.GetStrict(opt.StrictBlockChecksum),
 		fillCache: !ro.GetDontFillCache(),
 	}
-	return iterator.NewIndexedIterator(index, r.strictIter || ro.GetStrict(opt.StrictIterator), false)
+	return iterator.NewIndexedIterator(index, r.strictIter || ro.GetStrict(opt.StrictIterator), true)
 }
 
 // Find finds key/value pair whose key is greater than or equal to the
@@ -733,12 +767,15 @@ func (r *Reader) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.It
 // The caller should not modify the contents of the returned slice, but
 // it is safe to modify the contents of the argument after Find returns.
 func (r *Reader) Find(key []byte, ro *opt.ReadOptions) (rkey, value []byte, err error) {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
 	if r.err != nil {
 		err = r.err
 		return
 	}
 
-	indexBlock, rel, err := r.readBlockCached(r.indexBH, true, true)
+	indexBlock, rel, err := r.getIndexBlock(true)
 	if err != nil {
 		return
 	}
@@ -759,7 +796,7 @@ func (r *Reader) Find(key []byte, ro *opt.ReadOptions) (rkey, value []byte, err
 		return
 	}
 	if r.filter != nil {
-		filterBlock, rel, ferr := r.readFilterBlockCached(r.filterBH, true)
+		filterBlock, rel, ferr := r.getFilterBlock(true)
 		if ferr == nil {
 			if !filterBlock.contains(dataBH.offset, key) {
 				rel.Release()
@@ -779,9 +816,13 @@ func (r *Reader) Find(key []byte, ro *opt.ReadOptions) (rkey, value []byte, err
 	}
 	// Don't use block buffer, no need to copy the buffer.
 	rkey = data.Key()
-	// Use block buffer, and since the buffer will be recycled, the buffer
-	// need to be copied.
-	value = append([]byte{}, data.Value()...)
+	if r.bpool == nil {
+		value = data.Value()
+	} else {
+		// Use block buffer, and since the buffer will be recycled, the buffer
+		// need to be copied.
+		value = append([]byte{}, data.Value()...)
+	}
 	return
 }
 
@@ -791,6 +832,9 @@ func (r *Reader) Find(key []byte, ro *opt.ReadOptions) (rkey, value []byte, err
 // The caller should not modify the contents of the returned slice, but
 // it is safe to modify the contents of the argument after Get returns.
 func (r *Reader) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
 	if r.err != nil {
 		err = r.err
 		return
@@ -808,6 +852,9 @@ func (r *Reader) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error)
 //
 // It is safe to modify the contents of the argument after Get returns.
 func (r *Reader) OffsetOf(key []byte) (offset int64, err error) {
+	r.mu.RLock()
+	defer r.mu.RUnlock()
+
 	if r.err != nil {
 		err = r.err
 		return
@@ -840,12 +887,24 @@ func (r *Reader) OffsetOf(key []byte) (offset int64, err error) {
 // Release implements util.Releaser.
 // It also close the file if it is an io.Closer.
 func (r *Reader) Release() {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
 	if closer, ok := r.reader.(io.Closer); ok {
 		closer.Close()
 	}
+	if r.indexBlock != nil {
+		r.indexBlock.Release()
+		r.indexBlock = nil
+	}
+	if r.filterBlock != nil {
+		r.filterBlock.Release()
+		r.filterBlock = nil
+	}
 	r.reader = nil
 	r.cache = nil
 	r.bpool = nil
+	r.err = ErrReaderReleased
 }
 
 // NewReader creates a new initialized table reader for the file.
@@ -853,9 +912,6 @@ func (r *Reader) Release() {
 //
 // The returned table reader instance is goroutine-safe.
 func NewReader(f io.ReaderAt, size int64, cache cache.Namespace, bpool *util.BufferPool, o *opt.Options) *Reader {
-	if bpool == nil {
-		bpool = util.NewBufferPool(o.GetBlockSize() + blockTrailerLen)
-	}
 	r := &Reader{
 		reader:     f,
 		cache:      cache,
@@ -930,5 +986,22 @@ func NewReader(f io.ReaderAt, size int64, cache cache.Namespace, bpool *util.Buf
 	}
 	metaIter.Release()
 	metaBlock.Release()
+
+	// Cache index and filter block locally, since we don't have global cache.
+	if cache == nil {
+		r.indexBlock, r.err = r.readBlock(r.indexBH, true)
+		if r.err != nil {
+			return r
+		}
+		if r.filter != nil {
+			r.filterBlock, err = r.readFilterBlock(r.filterBH)
+			if err != nil {
+				// Don't use filter then.
+				r.filter = nil
+				r.filterBH = blockHandle{}
+			}
+		}
+	}
+
 	return r
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table_test.go

@@ -104,11 +104,11 @@ var _ = testutil.Defer(func() {
 					if body != nil {
 						body(db.(tableWrapper).Reader)
 					}
-					testutil.KeyValueTesting(nil, db, *kv)
+					testutil.KeyValueTesting(nil, *kv, db, nil, nil)
 				}
 			}
 
-			testutil.AllKeyValueTesting(nil, Build)
+			testutil.AllKeyValueTesting(nil, Build, nil, nil)
 			Describe("with one key per block", Test(testutil.KeyValue_Generate(nil, 9, 1, 10, 512, 512), func(r *Reader) {
 				It("should have correct blocks number", func() {
 					indexBlock, err := r.readBlock(r.indexBH, true)

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/kvtest.go

@@ -16,13 +16,22 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
-func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
+func KeyValueTesting(rnd *rand.Rand, kv KeyValue, p DB, setup func(KeyValue) DB, teardown func(DB)) {
 	if rnd == nil {
 		rnd = NewRand()
 	}
 
-	if db, ok := p.(Find); ok {
-		It("Should find all keys with Find", func() {
+	if p == nil {
+		BeforeEach(func() {
+			p = setup(kv)
+		})
+		AfterEach(func() {
+			teardown(p)
+		})
+	}
+
+	It("Should find all keys with Find", func() {
+		if db, ok := p.(Find); ok {
 			ShuffledIndex(nil, kv.Len(), 1, func(i int) {
 				key_, key, value := kv.IndexInexact(i)
 
@@ -38,9 +47,11 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
 				Expect(rkey).Should(Equal(key))
 				Expect(rvalue).Should(Equal(value), "Value for key %q (%q)", key_, key)
 			})
-		})
+		}
+	})
 
-		It("Should return error if the key is not present", func() {
+	It("Should return error if the key is not present", func() {
+		if db, ok := p.(Find); ok {
 			var key []byte
 			if kv.Len() > 0 {
 				key_, _ := kv.Index(kv.Len() - 1)
@@ -49,11 +60,11 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
 			rkey, _, err := db.TestFind(key)
 			Expect(err).Should(HaveOccurred(), "Find for key %q yield key %q", key, rkey)
 			Expect(err).Should(Equal(util.ErrNotFound))
-		})
-	}
+		}
+	})
 
-	if db, ok := p.(Get); ok {
-		It("Should only find exact key with Get", func() {
+	It("Should only find exact key with Get", func() {
+		if db, ok := p.(Get); ok {
 			ShuffledIndex(nil, kv.Len(), 1, func(i int) {
 				key_, key, value := kv.IndexInexact(i)
 
@@ -69,11 +80,11 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
 					Expect(err).Should(Equal(util.ErrNotFound))
 				}
 			})
-		})
-	}
+		}
+	})
 
-	if db, ok := p.(NewIterator); ok {
-		TestIter := func(r *util.Range, _kv KeyValue) {
+	TestIter := func(r *util.Range, _kv KeyValue) {
+		if db, ok := p.(NewIterator); ok {
 			iter := db.TestNewIterator(r)
 			Expect(iter.Error()).ShouldNot(HaveOccurred())
 
@@ -84,45 +95,48 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
 
 			DoIteratorTesting(&t)
 		}
+	}
 
-		It("Should iterates and seeks correctly", func(done Done) {
-			TestIter(nil, kv.Clone())
-			done <- true
-		}, 3.0)
+	It("Should iterates and seeks correctly", func(done Done) {
+		TestIter(nil, kv.Clone())
+		done <- true
+	}, 3.0)
 
-		RandomIndex(rnd, kv.Len(), kv.Len(), func(i int) {
-			type slice struct {
-				r            *util.Range
-				start, limit int
-			}
+	RandomIndex(rnd, kv.Len(), kv.Len(), func(i int) {
+		type slice struct {
+			r            *util.Range
+			start, limit int
+		}
 
-			key_, _, _ := kv.IndexInexact(i)
-			for _, x := range []slice{
-				{&util.Range{Start: key_, Limit: nil}, i, kv.Len()},
-				{&util.Range{Start: nil, Limit: key_}, 0, i},
-			} {
-				It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", x.start, x.limit), func(done Done) {
-					TestIter(x.r, kv.Slice(x.start, x.limit))
-					done <- true
-				}, 3.0)
-			}
-		})
-
-		RandomRange(rnd, kv.Len(), kv.Len(), func(start, limit int) {
-			It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", start, limit), func(done Done) {
-				r := kv.Range(start, limit)
-				TestIter(&r, kv.Slice(start, limit))
+		key_, _, _ := kv.IndexInexact(i)
+		for _, x := range []slice{
+			{&util.Range{Start: key_, Limit: nil}, i, kv.Len()},
+			{&util.Range{Start: nil, Limit: key_}, 0, i},
+		} {
+			It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", x.start, x.limit), func(done Done) {
+				TestIter(x.r, kv.Slice(x.start, x.limit))
 				done <- true
 			}, 3.0)
-		})
-	}
+		}
+	})
+
+	RandomRange(rnd, kv.Len(), kv.Len(), func(start, limit int) {
+		It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", start, limit), func(done Done) {
+			r := kv.Range(start, limit)
+			TestIter(&r, kv.Slice(start, limit))
+			done <- true
+		}, 3.0)
+	})
 }
 
-func AllKeyValueTesting(rnd *rand.Rand, body func(kv KeyValue) DB) {
+func AllKeyValueTesting(rnd *rand.Rand, body, setup func(KeyValue) DB, teardown func(DB)) {
 	Test := func(kv *KeyValue) func() {
 		return func() {
-			db := body(*kv)
-			KeyValueTesting(rnd, db, *kv)
+			var p DB
+			if body != nil {
+				p = body(*kv)
+			}
+			KeyValueTesting(rnd, *kv, p, setup, teardown)
 		}
 	}
 

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go

@@ -54,6 +54,10 @@ func (p *BufferPool) poolNum(n int) int {
 
 // Get returns buffer with length of n.
 func (p *BufferPool) Get(n int) []byte {
+	if p == nil {
+		return make([]byte, n)
+	}
+
 	atomic.AddUint32(&p.get, 1)
 
 	poolNum := p.poolNum(n)
@@ -145,6 +149,10 @@ func (p *BufferPool) Get(n int) []byte {
 
 // Put adds given buffer to the pool.
 func (p *BufferPool) Put(b []byte) {
+	if p == nil {
+		return
+	}
+
 	atomic.AddUint32(&p.put, 1)
 
 	pool := p.pool[p.poolNum(cap(b))]
@@ -156,6 +164,10 @@ func (p *BufferPool) Put(b []byte) {
 }
 
 func (p *BufferPool) Close() {
+	if p == nil {
+		return
+	}
+
 	select {
 	case p.close <- struct{}{}:
 	default:
@@ -163,6 +175,10 @@ func (p *BufferPool) Close() {
 }
 
 func (p *BufferPool) String() string {
+	if p == nil {
+		return "<nil>"
+	}
+
 	return fmt.Sprintf("BufferPool{B·%d Z·%v Zm·%v Zh·%v G·%d P·%d H·%d <·%d =·%d >·%d M·%d}",
 		p.baseline0, p.size, p.sizeMiss, p.sizeHalf, p.get, p.put, p.half, p.less, p.equal, p.greater, p.miss)
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/range.go

@@ -25,6 +25,7 @@ func BytesPrefix(prefix []byte) *Range {
 			limit = make([]byte, i+1)
 			copy(limit, prefix)
 			limit[i] = c + 1
+			break
 		}
 	}
 	return &Range{prefix, limit}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/util.go

@@ -12,7 +12,9 @@ import (
 )
 
 var (
-	ErrNotFound = errors.New("leveldb: not found")
+	ErrNotFound    = errors.New("leveldb: not found")
+	ErrReleased    = errors.New("leveldb: resource already relesed")
+	ErrHasReleaser = errors.New("leveldb: releaser already defined")
 )
 
 // Releaser is the interface that wraps the basic Release method.
@@ -27,23 +29,46 @@ type ReleaseSetter interface {
 	// SetReleaser associates the given releaser to the resources. The
 	// releaser will be called once coresponding resources released.
 	// Calling SetReleaser with nil will clear the releaser.
+	//
+	// This will panic if a releaser already present or coresponding
+	// resource is already released. Releaser should be cleared first
+	// before assigned a new one.
 	SetReleaser(releaser Releaser)
 }
 
 // BasicReleaser provides basic implementation of Releaser and ReleaseSetter.
 type BasicReleaser struct {
 	releaser Releaser
+	released bool
+}
+
+// Released returns whether Release method already called.
+func (r *BasicReleaser) Released() bool {
+	return r.released
 }
 
 // Release implements Releaser.Release.
 func (r *BasicReleaser) Release() {
-	if r.releaser != nil {
-		r.releaser.Release()
-		r.releaser = nil
+	if !r.released {
+		if r.releaser != nil {
+			r.releaser.Release()
+			r.releaser = nil
+		}
+		r.released = true
 	}
 }
 
 // SetReleaser implements ReleaseSetter.SetReleaser.
 func (r *BasicReleaser) SetReleaser(releaser Releaser) {
+	if r.released {
+		panic(ErrReleased)
+	}
+	if r.releaser != nil && releaser != nil {
+		panic(ErrHasReleaser)
+	}
 	r.releaser = releaser
 }
+
+type NoopReleaser struct{}
+
+func (NoopReleaser) Release() {}

LICENSE

@@ -1,19 +1,674 @@
-Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
 
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to
-deal in the Software without restriction, including without limitation the
-rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
-sell copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
 
-- The above copyright notice and this permission notice shall be included in
-  all copies or substantial portions of the Software.
+                            Preamble
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-IN THE SOFTWARE.
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<http://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<http://www.gnu.org/philosophy/why-not-lgpl.html>.

README.md

@@ -3,14 +3,14 @@ syncthing
 
 [![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing.svg?style=flat-square)](http://build.syncthing.net/job/syncthing/lastBuild/)
 [![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/syncthing)
-[![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat-square)](http://opensource.org/licenses/MIT)
+[![GPL License](http://img.shields.io/badge/license-GPL-blue.svg?style=flat-square)](http://opensource.org/licenses/GPL-3.0)
 
 This is the `syncthing` project. The following are the project goals:
 
- 1. Define a protocol for synchronization of a file repository between a
-    number of collaborating nodes. The protocol should be well defined,
-    unambiguous, easily understood, free to use, efficient, secure and
-    language neutral. This is the [Block Exchange
+ 1. Define a protocol for synchronization of a folder between a number of
+    collaborating devices. The protocol should be well defined, unambiguous,
+    easily understood, free to use, efficient, secure and language neutral.
+    This is the [Block Exchange
     Protocol](https://github.com/syncthing/syncthing/blob/master/protocol/PROTOCOL.md).
 
  2. Provide the reference implementation to demonstrate the usability of
@@ -56,5 +56,6 @@ All documentation and protocol specifications are licensed
 under the [Creative Commons Attribution 4.0 International
 License](http://creativecommons.org/licenses/by/4.0/).
 
-All code is licensed under the [MIT
-License](https://github.com/syncthing/syncthing/blob/master/LICENSE).
+All code is licensed under the
+[GPL](https://github.com/syncthing/syncthing/blob/master/LICENSE), v3 or
+later.

auto/auto_test.go

@@ -1,23 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
-
-package auto_test
-
-import (
-	"bytes"
-	"testing"
-
-	"github.com/syncthing/syncthing/auto"
-)
-
-func TestAssets(t *testing.T) {
-	assets := auto.Assets()
-	idx, ok := assets["index.html"]
-	if !ok {
-		t.Fatal("No index.html in compiled in assets")
-	}
-	if !bytes.Contains(idx, []byte("<html")) {
-		t.Fatal("No html in index.html")
-	}
-}

auto/doc.go

@@ -1,6 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
-
-// Package auto contains auto generated files for web assets.
-package auto

beacon/debug.go

@@ -1,17 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
-
-package beacon
-
-import (
-	"os"
-	"strings"
-
-	"github.com/syncthing/syncthing/logger"
-)
-
-var (
-	debug = strings.Contains(os.Getenv("STTRACE"), "beacon") || os.Getenv("STTRACE") == "all"
-	l     = logger.DefaultLogger
-)

beacon/doc.go

@@ -1,6 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
-
-// Package beacon implements an UDP broadcast beacon
-package beacon

build.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 // +build ignore
 
@@ -24,6 +35,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"time"
 )
 
 var (
@@ -31,6 +43,7 @@ var (
 	goarch    string
 	goos      string
 	noupgrade bool
+	version   string
 )
 
 const minGoVersion = 1.3
@@ -53,6 +66,7 @@ func main() {
 	flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
 	flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
 	flag.BoolVar(&noupgrade, "no-upgrade", false, "Disable upgrade functionality")
+	flag.StringVar(&version, "version", getVersion(), "Set compiled in version string")
 	flag.Parse()
 
 	switch goarch {
@@ -67,75 +81,62 @@ func main() {
 
 	checkRequiredGoVersion()
 
-	if check() != nil {
-		setup()
-	}
-
 	if flag.NArg() == 0 {
 		install("./cmd/...")
 		return
 	}
 
-	switch flag.Arg(0) {
-	case "install":
-		pkg := "./cmd/..."
-		if flag.NArg() > 2 {
-			pkg = flag.Arg(1)
+	for _, cmd := range flag.Args() {
+		switch cmd {
+		case "setup":
+			setup()
+
+		case "install":
+			pkg := "./cmd/..."
+			install(pkg)
+
+		case "build":
+			pkg := "./cmd/syncthing"
+			var tags []string
+			if noupgrade {
+				tags = []string{"noupgrade"}
+			}
+			build(pkg, tags)
+
+		case "test":
+			pkg := "./..."
+			test(pkg)
+
+		case "assets":
+			assets()
+
+		case "xdr":
+			xdr()
+
+		case "translate":
+			translate()
+
+		case "transifex":
+			transifex()
+
+		case "deps":
+			deps()
+
+		case "tar":
+			buildTar()
+
+		case "zip":
+			buildZip()
+
+		case "clean":
+			clean()
+
+		default:
+			log.Fatalf("Unknown command %q", cmd)
 		}
-		install(pkg)
-
-	case "build":
-		pkg := "./cmd/syncthing"
-		if flag.NArg() > 2 {
-			pkg = flag.Arg(1)
-		}
-		var tags []string
-		if noupgrade {
-			tags = []string{"noupgrade"}
-		}
-		build(pkg, tags)
-
-	case "test":
-		pkg := "./..."
-		if flag.NArg() > 2 {
-			pkg = flag.Arg(1)
-		}
-		test(pkg)
-
-	case "assets":
-		assets()
-
-	case "xdr":
-		xdr()
-
-	case "translate":
-		translate()
-
-	case "transifex":
-		transifex()
-
-	case "deps":
-		deps()
-
-	case "tar":
-		buildTar()
-
-	case "zip":
-		buildZip()
-
-	case "clean":
-		clean()
-
-	default:
-		log.Fatalf("Unknown command %q", flag.Arg(0))
 	}
 }
 
-func check() error {
-	_, err := exec.LookPath("godep")
-	return err
-}
-
 func checkRequiredGoVersion() {
 	ver := run("go", "version")
 	re := regexp.MustCompile(`go version go(\d+\.\d+)`)
@@ -163,24 +164,25 @@ func setup() {
 }
 
 func test(pkg string) {
-	runPrint("godep", "go", "test", "-short", "-timeout", "10s", pkg)
+	setBuildEnv()
+	runPrint("go", "test", "-short", "-timeout", "10s", pkg)
 }
 
 func install(pkg string) {
 	os.Setenv("GOBIN", "./bin")
 	setBuildEnv()
-	runPrint("godep", "go", "install", "-ldflags", ldflags(), pkg)
+	runPrint("go", "install", "-v", "-ldflags", ldflags(), pkg)
 }
 
 func build(pkg string, tags []string) {
 	rmr("syncthing", "syncthing.exe")
-	args := []string{"go", "build", "-ldflags", ldflags()}
+	args := []string{"build", "-ldflags", ldflags()}
 	if len(tags) > 0 {
 		args = append(args, "-tags", strings.Join(tags, ","))
 	}
 	args = append(args, pkg)
 	setBuildEnv()
-	runPrint("godep", args...)
+	runPrint("go", args...)
 }
 
 func buildTar() {
@@ -230,14 +232,22 @@ func setBuildEnv() {
 	if goarch == "386" {
 		os.Setenv("GO386", "387")
 	}
+	wd, err := os.Getwd()
+	if err != nil {
+		log.Println("Warning: can't determine current dir:", err)
+		log.Println("Build might not work as expected")
+	}
+	os.Setenv("GOPATH", fmt.Sprintf("%s%c%s", filepath.Join(wd, "Godeps", "_workspace"), os.PathListSeparator, os.Getenv("GOPATH")))
+	log.Println("GOPATH=" + os.Getenv("GOPATH"))
 }
 
 func assets() {
-	runPipe("auto/gui.files.go", "godep", "go", "run", "cmd/genassets/main.go", "gui")
+	setBuildEnv()
+	runPipe("internal/auto/gui.files.go", "go", "run", "cmd/genassets/main.go", "gui")
 }
 
 func xdr() {
-	for _, f := range []string{"discover/packets", "files/leveldb", "protocol/message"} {
+	for _, f := range []string{"internal/discover/packets", "internal/files/leveldb", "internal/protocol/message"} {
 		runPipe(f+"_xdr.go", "go", "run", "./Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go", "--", f+".go")
 	}
 }
@@ -273,7 +283,7 @@ func clean() {
 func ldflags() string {
 	var b bytes.Buffer
 	b.WriteString("-w")
-	b.WriteString(fmt.Sprintf(" -X main.Version %s", version()))
+	b.WriteString(fmt.Sprintf(" -X main.Version %s", version))
 	b.WriteString(fmt.Sprintf(" -X main.BuildStamp %d", buildStamp()))
 	b.WriteString(fmt.Sprintf(" -X main.BuildUser %s", buildUser()))
 	b.WriteString(fmt.Sprintf(" -X main.BuildHost %s", buildHost()))
@@ -291,8 +301,11 @@ func rmr(paths ...string) {
 	}
 }
 
-func version() string {
-	v := run("git", "describe", "--always", "--dirty")
+func getVersion() string {
+	v, err := runError("git", "describe", "--always", "--dirty")
+	if err != nil {
+		return "unknown-dev"
+	}
 	v = versionRe.ReplaceAllFunc(v, func(s []byte) []byte {
 		s[0] = '+'
 		return s
@@ -301,7 +314,10 @@ func version() string {
 }
 
 func buildStamp() int64 {
-	bs := run("git", "show", "-s", "--format=%ct")
+	bs, err := runError("git", "show", "-s", "--format=%ct")
+	if err != nil {
+		return time.Now().Unix()
+	}
 	s, _ := strconv.ParseInt(string(bs), 10, 64)
 	return s
 }
@@ -338,12 +354,11 @@ func buildArch() string {
 }
 
 func archiveName() string {
-	return fmt.Sprintf("syncthing-%s-%s", buildArch(), version())
+	return fmt.Sprintf("syncthing-%s-%s", buildArch(), version)
 }
 
 func run(cmd string, args ...string) []byte {
-	ecmd := exec.Command(cmd, args...)
-	bs, err := ecmd.CombinedOutput()
+	bs, err := runError(cmd, args...)
 	if err != nil {
 		log.Println(cmd, strings.Join(args, " "))
 		log.Println(string(bs))
@@ -352,6 +367,15 @@ func run(cmd string, args ...string) []byte {
 	return bytes.TrimSpace(bs)
 }
 
+func runError(cmd string, args ...string) ([]byte, error) {
+	ecmd := exec.Command(cmd, args...)
+	bs, err := ecmd.CombinedOutput()
+	if err != nil {
+		return nil, err
+	}
+	return bytes.TrimSpace(bs), nil
+}
+
 func runPrint(cmd string, args ...string) {
 	log.Println(cmd, strings.Join(args, " "))
 	ecmd := exec.Command(cmd, args...)

build.sh

@@ -12,9 +12,9 @@ case "${1:-default}" in
 		;;
 
 	test)
-		ulimit -t 60 || true
-		ulimit -d 512000 || true
-		ulimit -m 512000 || true
+		ulimit -t 60 &>/dev/null || true
+		ulimit -d 512000 &>/dev/null || true
+		ulimit -m 512000 &>/dev/null || true
 
 		go run build.go "$1"
 		;;
@@ -57,6 +57,9 @@ case "${1:-default}" in
 		go run build.go -goos freebsd -goarch amd64 tar
 		go run build.go -goos freebsd -goarch 386 tar
 
+		go run build.go -goos openbsd -goarch amd64 tar
+		go run build.go -goos openbsd -goarch 386 tar
+
 		go run build.go -goos darwin -goarch amd64 tar
 
 		go run build.go -goos windows -goarch amd64 zip
@@ -68,9 +71,9 @@ case "${1:-default}" in
 		;;
 
 	test-cov)
-		ulimit -t 60 || true
-		ulimit -d 512000 || true
-		ulimit -m 512000 || true
+		ulimit -t 60 &>/dev/null || true
+		ulimit -d 512000 &>/dev/null || true
+		ulimit -m 512000 &>/dev/null || true
 
 		go get github.com/axw/gocov/gocov
 		go get github.com/AlekSi/gocov-xml

check-contrib.sh

@@ -8,6 +8,8 @@ missing-contribs() {
 
 no-docs-typos() {
 	# Commits that are known to not change code
+	grep -v 63bd0136fb40a91efaa279cb4b4159d82e8e6904 |\
+	grep -v 4e2feb6fbc791bb8a2daf0ab8efb10775d66343e |\
 	grep -v f2459ef3319b2f060dbcdacd0c35a1788a94b8bd |\
 	grep -v b61f418bf2d1f7d5a9d7088a20a2a448e5e66801 |\
 	grep -v f0621207e3953711f9ab86d99724f1d0faac45b1 |\
@@ -24,6 +26,19 @@ print-missing-copyright() {
 	find . -name \*.go | xargs grep -L 'Copyright (C)' | grep -v Godeps
 }
 
+print-line-blame() {
+	for f in $(find . -name \*.go | grep -v Godep) gui/app.js gui/index.html ; do
+		git blame --line-porcelain $f | grep author-mail
+	done | sort | uniq -c | sort -n
+}
+echo Author emails missing in CONTRIBUTORS:
 print-missing-contribs
-print-missing-copyright
+echo
+
+echo Files missing copyright notice:
+print-missing-copyright
+echo
+
+echo Blame lines per author:
+print-line-blame
 

cmd/genassets/main.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 // +build ignore
 

cmd/stevents/main.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 package main
 

cmd/stindex/main.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 package main
 
@@ -10,8 +21,8 @@ import (
 	"log"
 	"os"
 
-	"github.com/syncthing/syncthing/files"
-	"github.com/syncthing/syncthing/protocol"
+	"github.com/syncthing/syncthing/internal/files"
+	"github.com/syncthing/syncthing/internal/protocol"
 	"github.com/syndtr/goleveldb/leveldb"
 )
 
@@ -19,8 +30,8 @@ func main() {
 	log.SetFlags(0)
 	log.SetOutput(os.Stdout)
 
-	repo := flag.String("repo", "default", "Repository ID")
-	node := flag.String("node", "", "Node ID (blank for global)")
+	folder := flag.String("folder", "default", "Folder ID")
+	device := flag.String("device", "", "Device ID (blank for global)")
 	flag.Parse()
 
 	db, err := leveldb.OpenFile(flag.Arg(0), nil)
@@ -28,10 +39,10 @@ func main() {
 		log.Fatal(err)
 	}
 
-	fs := files.NewSet(*repo, db)
+	fs := files.NewSet(*folder, db)
 
-	if *node == "" {
-		log.Printf("*** Global index for repo %q", *repo)
+	if *device == "" {
+		log.Printf("*** Global index for folder %q", *folder)
 		fs.WithGlobalTruncated(func(fi protocol.FileIntf) bool {
 			f := fi.(protocol.FileInfoTruncated)
 			fmt.Println(f)
@@ -39,11 +50,11 @@ func main() {
 			return true
 		})
 	} else {
-		n, err := protocol.NodeIDFromString(*node)
+		n, err := protocol.DeviceIDFromString(*device)
 		if err != nil {
 			log.Fatal(err)
 		}
-		log.Printf("*** Have index for repo %q node %q", *repo, n)
+		log.Printf("*** Have index for folder %q device %q", *folder, n)
 		fs.WithHaveTruncated(n, func(fi protocol.FileIntf) bool {
 			f := fi.(protocol.FileInfoTruncated)
 			fmt.Println(f)

cmd/syncthing/debug.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 package main
 

cmd/syncthing/gui.go

@@ -1,23 +1,30 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 package main
 
 import (
-	"bytes"
 	"crypto/tls"
-	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"math/rand"
 	"mime"
 	"net"
 	"net/http"
 	"os"
 	"path/filepath"
-	"reflect"
 	"runtime"
 	"strconv"
 	"strings"
@@ -25,13 +32,15 @@ import (
 	"time"
 
 	"code.google.com/p/go.crypto/bcrypt"
-	"github.com/syncthing/syncthing/auto"
-	"github.com/syncthing/syncthing/config"
-	"github.com/syncthing/syncthing/events"
-	"github.com/syncthing/syncthing/logger"
-	"github.com/syncthing/syncthing/model"
-	"github.com/syncthing/syncthing/protocol"
-	"github.com/syncthing/syncthing/upgrade"
+	"github.com/calmh/logger"
+	"github.com/syncthing/syncthing/internal/auto"
+	"github.com/syncthing/syncthing/internal/config"
+	"github.com/syncthing/syncthing/internal/discover"
+	"github.com/syncthing/syncthing/internal/events"
+	"github.com/syncthing/syncthing/internal/model"
+	"github.com/syncthing/syncthing/internal/osutil"
+	"github.com/syncthing/syncthing/internal/protocol"
+	"github.com/syncthing/syncthing/internal/upgrade"
 	"github.com/vitrun/qart/qr"
 )
 
@@ -44,15 +53,10 @@ var (
 	configInSync = true
 	guiErrors    = []guiError{}
 	guiErrorsMut sync.Mutex
-	apiKey       string
 	modt         = time.Now().UTC().Format(http.TimeFormat)
 	eventSub     *events.BufferedSubscription
 )
 
-const (
-	unchangedPassword = "--password-unchanged--"
-)
-
 func init() {
 	l.AddHandler(logger.LevelWarn, showGuiError)
 	sub := events.Default.Subscribe(events.AllEvents)
@@ -60,39 +64,32 @@ func init() {
 }
 
 func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) error {
-	var listener net.Listener
 	var err error
-	if cfg.UseTLS {
-		cert, err := loadCert(confDir, "https-")
-		if err != nil {
-			l.Infoln("Loading HTTPS certificate:", err)
-			l.Infoln("Creating new HTTPS certificate")
-			newCertificate(confDir, "https-")
-			cert, err = loadCert(confDir, "https-")
-		}
-		if err != nil {
-			return err
-		}
-		tlsCfg := &tls.Config{
-			Certificates: []tls.Certificate{cert},
-			ServerName:   "syncthing",
-		}
-		listener, err = tls.Listen("tcp", cfg.Address, tlsCfg)
-		if err != nil {
-			return err
-		}
-	} else {
-		listener, err = net.Listen("tcp", cfg.Address)
-		if err != nil {
-			return err
-		}
+
+	cert, err := loadCert(confDir, "https-")
+	if err != nil {
+		l.Infoln("Loading HTTPS certificate:", err)
+		l.Infoln("Creating new HTTPS certificate")
+		newCertificate(confDir, "https-")
+		cert, err = loadCert(confDir, "https-")
+	}
+	if err != nil {
+		return err
+	}
+	tlsCfg := &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		ServerName:   "syncthing",
 	}
 
-	apiKey = cfg.APIKey
-	loadCsrfTokens()
+	rawListener, err := net.Listen("tcp", cfg.Address)
+	if err != nil {
+		return err
+	}
+	listener := &DowngradingListener{rawListener, tlsCfg}
 
 	// The GET handlers
 	getRestMux := http.NewServeMux()
+	getRestMux.HandleFunc("/rest/ping", restPing)
 	getRestMux.HandleFunc("/rest/completion", withModel(m, restGetCompletion))
 	getRestMux.HandleFunc("/rest/config", restGetConfig)
 	getRestMux.HandleFunc("/rest/config/sync", restGetConfigInSync)
@@ -100,26 +97,28 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro
 	getRestMux.HandleFunc("/rest/discovery", restGetDiscovery)
 	getRestMux.HandleFunc("/rest/errors", restGetErrors)
 	getRestMux.HandleFunc("/rest/events", restGetEvents)
+	getRestMux.HandleFunc("/rest/ignores", withModel(m, restGetIgnores))
 	getRestMux.HandleFunc("/rest/lang", restGetLang)
 	getRestMux.HandleFunc("/rest/model", withModel(m, restGetModel))
-	getRestMux.HandleFunc("/rest/model/version", withModel(m, restGetModelVersion))
 	getRestMux.HandleFunc("/rest/need", withModel(m, restGetNeed))
-	getRestMux.HandleFunc("/rest/nodeid", restGetNodeID)
+	getRestMux.HandleFunc("/rest/deviceid", restGetDeviceID)
 	getRestMux.HandleFunc("/rest/report", withModel(m, restGetReport))
 	getRestMux.HandleFunc("/rest/system", restGetSystem)
 	getRestMux.HandleFunc("/rest/upgrade", restGetUpgrade)
 	getRestMux.HandleFunc("/rest/version", restGetVersion)
-	getRestMux.HandleFunc("/rest/stats/node", withModel(m, restGetNodeStats))
+	getRestMux.HandleFunc("/rest/stats/device", withModel(m, restGetDeviceStats))
 
 	// Debug endpoints, not for general use
 	getRestMux.HandleFunc("/rest/debug/peerCompletion", withModel(m, restGetPeerCompletion))
 
 	// The POST handlers
 	postRestMux := http.NewServeMux()
+	postRestMux.HandleFunc("/rest/ping", restPing)
 	postRestMux.HandleFunc("/rest/config", withModel(m, restPostConfig))
 	postRestMux.HandleFunc("/rest/discovery/hint", restPostDiscoveryHint)
 	postRestMux.HandleFunc("/rest/error", restPostError)
 	postRestMux.HandleFunc("/rest/error/clear", restClearErrors)
+	postRestMux.HandleFunc("/rest/ignores", withModel(m, restPostIgnores))
 	postRestMux.HandleFunc("/rest/model/override", withModel(m, restPostOverride))
 	postRestMux.HandleFunc("/rest/reset", restPostReset)
 	postRestMux.HandleFunc("/rest/restart", restPostRestart)
@@ -141,17 +140,32 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro
 
 	// Wrap everything in CSRF protection. The /rest prefix should be
 	// protected, other requests will grant cookies.
-	handler := csrfMiddleware("/rest", mux)
+	handler := csrfMiddleware("/rest", cfg.APIKey, mux)
 
 	// Add our version as a header to responses
 	handler = withVersionMiddleware(handler)
 
 	// Wrap everything in basic auth, if user/password is set.
-	if len(cfg.User) > 0 {
-		handler = basicAuthMiddleware(cfg.User, cfg.Password, handler)
+	if len(cfg.User) > 0 && len(cfg.Password) > 0 {
+		handler = basicAuthAndSessionMiddleware(cfg, handler)
 	}
 
-	go http.Serve(listener, handler)
+	// Redirect to HTTPS if we are supposed to
+	if cfg.UseTLS {
+		handler = redirectToHTTPSMiddleware(handler)
+	}
+
+	srv := http.Server{
+		Handler:     handler,
+		ReadTimeout: 2 * time.Second,
+	}
+
+	go func() {
+		err := srv.Serve(listener)
+		if err != nil {
+			panic(err)
+		}
+	}()
 	return nil
 }
 
@@ -168,6 +182,23 @@ func getPostHandler(get, post http.Handler) http.Handler {
 	})
 }
 
+func redirectToHTTPSMiddleware(h http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Add a generous access-control-allow-origin header since we may be
+		// redirecting REST requests over protocols
+		w.Header().Add("Access-Control-Allow-Origin", "*")
+
+		if r.TLS == nil {
+			// Redirect HTTP requests to HTTPS
+			r.URL.Host = r.Host
+			r.URL.Scheme = "https"
+			http.Redirect(w, r, r.URL.String(), http.StatusFound)
+		} else {
+			h.ServeHTTP(w, r)
+		}
+	})
+}
+
 func noCacheMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Cache-Control", "no-cache")
@@ -188,65 +219,62 @@ func withModel(m *model.Model, h func(m *model.Model, w http.ResponseWriter, r *
 	}
 }
 
+func restPing(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	json.NewEncoder(w).Encode(map[string]string{
+		"ping": "pong",
+	})
+}
+
 func restGetVersion(w http.ResponseWriter, r *http.Request) {
-	w.Write([]byte(Version))
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	json.NewEncoder(w).Encode(map[string]string{
+		"version":     Version,
+		"longVersion": LongVersion,
+		"os":          runtime.GOOS,
+		"arch":        runtime.GOARCH,
+	})
 }
 
 func restGetCompletion(m *model.Model, w http.ResponseWriter, r *http.Request) {
 	var qs = r.URL.Query()
-	var repo = qs.Get("repo")
-	var nodeStr = qs.Get("node")
+	var folder = qs.Get("folder")
+	var deviceStr = qs.Get("device")
 
-	node, err := protocol.NodeIDFromString(nodeStr)
+	device, err := protocol.DeviceIDFromString(deviceStr)
 	if err != nil {
 		http.Error(w, err.Error(), 500)
 		return
 	}
 
 	res := map[string]float64{
-		"completion": m.Completion(node, repo),
+		"completion": m.Completion(device, folder),
 	}
 
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	json.NewEncoder(w).Encode(res)
 }
 
-func restGetModelVersion(m *model.Model, w http.ResponseWriter, r *http.Request) {
-	var qs = r.URL.Query()
-	var repo = qs.Get("repo")
-	var res = make(map[string]interface{})
-
-	res["version"] = m.LocalVersion(repo)
-
-	w.Header().Set("Content-Type", "application/json; charset=utf-8")
-	json.NewEncoder(w).Encode(res)
-}
-
 func restGetModel(m *model.Model, w http.ResponseWriter, r *http.Request) {
 	var qs = r.URL.Query()
-	var repo = qs.Get("repo")
+	var folder = qs.Get("folder")
 	var res = make(map[string]interface{})
 
-	for _, cr := range cfg.Repositories {
-		if cr.ID == repo {
-			res["invalid"] = cr.Invalid
-			break
-		}
-	}
+	res["invalid"] = cfg.Folders()[folder].Invalid
 
-	globalFiles, globalDeleted, globalBytes := m.GlobalSize(repo)
+	globalFiles, globalDeleted, globalBytes := m.GlobalSize(folder)
 	res["globalFiles"], res["globalDeleted"], res["globalBytes"] = globalFiles, globalDeleted, globalBytes
 
-	localFiles, localDeleted, localBytes := m.LocalSize(repo)
+	localFiles, localDeleted, localBytes := m.LocalSize(folder)
 	res["localFiles"], res["localDeleted"], res["localBytes"] = localFiles, localDeleted, localBytes
 
-	needFiles, needBytes := m.NeedSize(repo)
+	needFiles, needBytes := m.NeedSize(folder)
 	res["needFiles"], res["needBytes"] = needFiles, needBytes
 
 	res["inSyncFiles"], res["inSyncBytes"] = globalFiles-needFiles, globalBytes-needBytes
 
-	res["state"], res["stateChanged"] = m.State(repo)
-	res["version"] = m.LocalVersion(repo)
+	res["state"], res["stateChanged"] = m.State(folder)
+	res["version"] = m.CurrentLocalVersion(folder) + m.RemoteLocalVersion(folder)
 
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	json.NewEncoder(w).Encode(res)
@@ -254,15 +282,15 @@ func restGetModel(m *model.Model, w http.ResponseWriter, r *http.Request) {
 
 func restPostOverride(m *model.Model, w http.ResponseWriter, r *http.Request) {
 	var qs = r.URL.Query()
-	var repo = qs.Get("repo")
-	go m.Override(repo)
+	var folder = qs.Get("folder")
+	go m.Override(folder)
 }
 
 func restGetNeed(m *model.Model, w http.ResponseWriter, r *http.Request) {
 	var qs = r.URL.Query()
-	var repo = qs.Get("repo")
+	var folder = qs.Get("folder")
 
-	files := m.NeedFilesRepo(repo)
+	files := m.NeedFolderFilesLimited(folder, 100, 2500) // max 100 files or 2500 blocks
 
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	json.NewEncoder(w).Encode(files)
@@ -274,19 +302,15 @@ func restGetConnections(m *model.Model, w http.ResponseWriter, r *http.Request)
 	json.NewEncoder(w).Encode(res)
 }
 
-func restGetNodeStats(m *model.Model, w http.ResponseWriter, r *http.Request) {
-	var res = m.NodeStatistics()
+func restGetDeviceStats(m *model.Model, w http.ResponseWriter, r *http.Request) {
+	var res = m.DeviceStatistics()
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	json.NewEncoder(w).Encode(res)
 }
 
 func restGetConfig(w http.ResponseWriter, r *http.Request) {
-	encCfg := cfg
-	if encCfg.GUI.Password != "" {
-		encCfg.GUI.Password = unchangedPassword
-	}
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
-	json.NewEncoder(w).Encode(encCfg)
+	json.NewEncoder(w).Encode(cfg.Raw())
 }
 
 func restPostConfig(m *model.Model, w http.ResponseWriter, r *http.Request) {
@@ -297,51 +321,22 @@ func restPostConfig(m *model.Model, w http.ResponseWriter, r *http.Request) {
 		http.Error(w, err.Error(), 500)
 		return
 	} else {
-		if newCfg.GUI.Password == "" {
-			// Leave it empty
-		} else if newCfg.GUI.Password == unchangedPassword {
-			newCfg.GUI.Password = cfg.GUI.Password
-		} else {
-			hash, err := bcrypt.GenerateFromPassword([]byte(newCfg.GUI.Password), 0)
-			if err != nil {
-				l.Warnln("bcrypting password:", err)
-				http.Error(w, err.Error(), 500)
-				return
-			} else {
-				newCfg.GUI.Password = string(hash)
-			}
-		}
-
-		// Figure out if any changes require a restart
-
-		if len(cfg.Repositories) != len(newCfg.Repositories) {
-			configInSync = false
-		} else {
-			om := cfg.RepoMap()
-			nm := newCfg.RepoMap()
-			for id := range om {
-				if !reflect.DeepEqual(om[id], nm[id]) {
-					configInSync = false
-					break
+		if newCfg.GUI.Password != cfg.GUI().Password {
+			if newCfg.GUI.Password != "" {
+				hash, err := bcrypt.GenerateFromPassword([]byte(newCfg.GUI.Password), 0)
+				if err != nil {
+					l.Warnln("bcrypting password:", err)
+					http.Error(w, err.Error(), 500)
+					return
+				} else {
+					newCfg.GUI.Password = string(hash)
 				}
 			}
 		}
 
-		if len(cfg.Nodes) != len(newCfg.Nodes) {
-			configInSync = false
-		} else {
-			om := cfg.NodeMap()
-			nm := newCfg.NodeMap()
-			for k := range om {
-				if _, ok := nm[k]; !ok {
-					// A node was removed and another added
-					configInSync = false
-					break
-				}
-			}
-		}
+		// Start or stop usage reporting as appropriate
 
-		if newCfg.Options.URAccepted > cfg.Options.URAccepted {
+		if curAcc := cfg.Options().URAccepted; newCfg.Options.URAccepted > curAcc {
 			// UR was enabled
 			newCfg.Options.URAccepted = usageReportVersion
 			err := sendUsageReport(m)
@@ -349,20 +344,17 @@ func restPostConfig(m *model.Model, w http.ResponseWriter, r *http.Request) {
 				l.Infoln("Usage report:", err)
 			}
 			go usageReportingLoop(m)
-		} else if newCfg.Options.URAccepted < cfg.Options.URAccepted {
+		} else if newCfg.Options.URAccepted < curAcc {
 			// UR was disabled
 			newCfg.Options.URAccepted = -1
 			stopUsageReporting()
 		}
 
-		if !reflect.DeepEqual(cfg.Options, newCfg.Options) || !reflect.DeepEqual(cfg.GUI, newCfg.GUI) {
-			configInSync = false
-		}
-
 		// Activate and save
 
-		cfg = newCfg
-		saveConfig()
+		configInSync = !config.ChangeRequiresRestart(cfg.Raw(), newCfg)
+		cfg.Replace(newCfg)
+		cfg.Save()
 	}
 }
 
@@ -377,8 +369,8 @@ func restPostRestart(w http.ResponseWriter, r *http.Request) {
 }
 
 func restPostReset(w http.ResponseWriter, r *http.Request) {
-	flushResponse(`{"ok": "resetting repos"}`, w)
-	resetRepositories()
+	flushResponse(`{"ok": "resetting folders"}`, w)
+	resetFolders()
 	go restart()
 }
 
@@ -400,13 +392,14 @@ func restGetSystem(w http.ResponseWriter, r *http.Request) {
 	var m runtime.MemStats
 	runtime.ReadMemStats(&m)
 
+	tilde, _ := osutil.ExpandTilde("~")
 	res := make(map[string]interface{})
 	res["myID"] = myID.String()
 	res["goroutines"] = runtime.NumGoroutine()
 	res["alloc"] = m.Alloc
 	res["sys"] = m.Sys - m.HeapReleased
-	res["tilde"] = expandTilde("~")
-	if cfg.Options.GlobalAnnEnabled && discoverer != nil {
+	res["tilde"] = tilde
+	if cfg.Options().GlobalAnnEnabled && discoverer != nil {
 		res["extAnnounceOK"] = discoverer.ExtAnnounceOK()
 	}
 	cpuUsageLock.RLock()
@@ -424,7 +417,7 @@ func restGetSystem(w http.ResponseWriter, r *http.Request) {
 func restGetErrors(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	guiErrorsMut.Lock()
-	json.NewEncoder(w).Encode(guiErrors)
+	json.NewEncoder(w).Encode(map[string][]guiError{"errors": guiErrors})
 	guiErrorsMut.Unlock()
 }
 
@@ -451,15 +444,25 @@ func showGuiError(l logger.LogLevel, err string) {
 
 func restPostDiscoveryHint(w http.ResponseWriter, r *http.Request) {
 	var qs = r.URL.Query()
-	var node = qs.Get("node")
+	var device = qs.Get("device")
 	var addr = qs.Get("addr")
-	if len(node) != 0 && len(addr) != 0 && discoverer != nil {
-		discoverer.Hint(node, []string{addr})
+	if len(device) != 0 && len(addr) != 0 && discoverer != nil {
+		discoverer.Hint(device, []string{addr})
 	}
 }
 
 func restGetDiscovery(w http.ResponseWriter, r *http.Request) {
-	json.NewEncoder(w).Encode(discoverer.All())
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	registry := discoverer.All()
+
+	// Device ids can't be marshalled as keys so we need to manually
+	// rebuild this map using strings.
+	devices := make(map[string][]discover.CacheEntry, len(registry))
+	for device, _ := range registry {
+		devices[device.String()] = registry[device]
+	}
+
+	json.NewEncoder(w).Encode(devices)
 }
 
 func restGetReport(m *model.Model, w http.ResponseWriter, r *http.Request) {
@@ -467,6 +470,41 @@ func restGetReport(m *model.Model, w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(reportData(m))
 }
 
+func restGetIgnores(m *model.Model, w http.ResponseWriter, r *http.Request) {
+	qs := r.URL.Query()
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+
+	ignores, err := m.GetIgnores(qs.Get("folder"))
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+	json.NewEncoder(w).Encode(map[string][]string{
+		"ignore": ignores,
+	})
+}
+
+func restPostIgnores(m *model.Model, w http.ResponseWriter, r *http.Request) {
+	qs := r.URL.Query()
+
+	var data map[string][]string
+	err := json.NewDecoder(r.Body).Decode(&data)
+	r.Body.Close()
+
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	err = m.SetIgnores(qs.Get("folder"), data["ignore"])
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	restGetIgnores(m, w, r)
+}
+
 func restGetEvents(w http.ResponseWriter, r *http.Request) {
 	qs := r.URL.Query()
 	sinceStr := qs.Get("since")
@@ -504,10 +542,10 @@ func restGetUpgrade(w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(res)
 }
 
-func restGetNodeID(w http.ResponseWriter, r *http.Request) {
+func restGetDeviceID(w http.ResponseWriter, r *http.Request) {
 	qs := r.URL.Query()
 	idStr := qs.Get("id")
-	id, err := protocol.NodeIDFromString(idStr)
+	id, err := protocol.DeviceIDFromString(idStr)
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	if err == nil {
 		json.NewEncoder(w).Encode(map[string]string{
@@ -547,15 +585,17 @@ func restPostUpgrade(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 
-		restPostRestart(w, r)
+		flushResponse(`{"ok": "restarting"}`, w)
+		l.Infoln("Upgrading")
+		stop <- exitUpgrading
 	}
 }
 
 func restPostScan(m *model.Model, w http.ResponseWriter, r *http.Request) {
 	qs := r.URL.Query()
-	repo := qs.Get("repo")
+	folder := qs.Get("folder")
 	sub := qs.Get("sub")
-	err := m.ScanRepoSub(repo, sub)
+	err := m.ScanFolderSub(folder, sub)
 	if err != nil {
 		http.Error(w, err.Error(), 500)
 	}
@@ -578,77 +618,27 @@ func restGetPeerCompletion(m *model.Model, w http.ResponseWriter, r *http.Reques
 	tot := map[string]float64{}
 	count := map[string]float64{}
 
-	for _, repo := range cfg.Repositories {
-		for _, node := range repo.NodeIDs() {
-			nodeStr := node.String()
-			if m.ConnectedTo(node) {
-				tot[nodeStr] += m.Completion(node, repo.ID)
+	for _, folder := range cfg.Folders() {
+		for _, device := range folder.DeviceIDs() {
+			deviceStr := device.String()
+			if m.ConnectedTo(device) {
+				tot[deviceStr] += m.Completion(device, folder.ID)
 			} else {
-				tot[nodeStr] = 0
+				tot[deviceStr] = 0
 			}
-			count[nodeStr]++
+			count[deviceStr]++
 		}
 	}
 
 	comp := map[string]int{}
-	for node := range tot {
-		comp[node] = int(tot[node] / count[node])
+	for device := range tot {
+		comp[device] = int(tot[device] / count[device])
 	}
 
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	json.NewEncoder(w).Encode(comp)
 }
 
-func basicAuthMiddleware(username string, passhash string, next http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if validAPIKey(r.Header.Get("X-API-Key")) {
-			next.ServeHTTP(w, r)
-			return
-		}
-
-		error := func() {
-			time.Sleep(time.Duration(rand.Intn(100)+100) * time.Millisecond)
-			w.Header().Set("WWW-Authenticate", "Basic realm=\"Authorization Required\"")
-			http.Error(w, "Not Authorized", http.StatusUnauthorized)
-		}
-
-		hdr := r.Header.Get("Authorization")
-		if !strings.HasPrefix(hdr, "Basic ") {
-			error()
-			return
-		}
-
-		hdr = hdr[6:]
-		bs, err := base64.StdEncoding.DecodeString(hdr)
-		if err != nil {
-			error()
-			return
-		}
-
-		fields := bytes.SplitN(bs, []byte(":"), 2)
-		if len(fields) != 2 {
-			error()
-			return
-		}
-
-		if string(fields[0]) != username {
-			error()
-			return
-		}
-
-		if err := bcrypt.CompareHashAndPassword([]byte(passhash), fields[1]); err != nil {
-			error()
-			return
-		}
-
-		next.ServeHTTP(w, r)
-	})
-}
-
-func validAPIKey(k string) bool {
-	return len(apiKey) > 0 && k == apiKey
-}
-
 func embeddedStatic(assetDir string) http.Handler {
 	assets := auto.Assets()
 

cmd/syncthing/gui_auth.go

@@ -0,0 +1,101 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"encoding/base64"
+	"math/rand"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"code.google.com/p/go.crypto/bcrypt"
+	"github.com/syncthing/syncthing/internal/config"
+)
+
+var (
+	sessions    = make(map[string]bool)
+	sessionsMut sync.Mutex
+)
+
+func basicAuthAndSessionMiddleware(cfg config.GUIConfiguration, next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if cfg.APIKey != "" && r.Header.Get("X-API-Key") == cfg.APIKey {
+			next.ServeHTTP(w, r)
+			return
+		}
+
+		cookie, err := r.Cookie("sessionid")
+		if err == nil && cookie != nil {
+			sessionsMut.Lock()
+			_, ok := sessions[cookie.Value]
+			sessionsMut.Unlock()
+			if ok {
+				next.ServeHTTP(w, r)
+				return
+			}
+		}
+
+		error := func() {
+			time.Sleep(time.Duration(rand.Intn(100)+100) * time.Millisecond)
+			w.Header().Set("WWW-Authenticate", "Basic realm=\"Authorization Required\"")
+			http.Error(w, "Not Authorized", http.StatusUnauthorized)
+		}
+
+		hdr := r.Header.Get("Authorization")
+		if !strings.HasPrefix(hdr, "Basic ") {
+			error()
+			return
+		}
+
+		hdr = hdr[6:]
+		bs, err := base64.StdEncoding.DecodeString(hdr)
+		if err != nil {
+			error()
+			return
+		}
+
+		fields := bytes.SplitN(bs, []byte(":"), 2)
+		if len(fields) != 2 {
+			error()
+			return
+		}
+
+		if string(fields[0]) != cfg.User {
+			error()
+			return
+		}
+
+		if err := bcrypt.CompareHashAndPassword([]byte(cfg.Password), fields[1]); err != nil {
+			error()
+			return
+		}
+
+		sessionid := randomString(32)
+		sessionsMut.Lock()
+		sessions[sessionid] = true
+		sessionsMut.Unlock()
+		http.SetCookie(w, &http.Cookie{
+			Name:   "sessionid",
+			Value:  sessionid,
+			MaxAge: 0,
+		})
+
+		next.ServeHTTP(w, r)
+	})
+}

cmd/syncthing/gui_csrf.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 package main
 
@@ -16,7 +27,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/syncthing/syncthing/osutil"
+	"github.com/syncthing/syncthing/internal/osutil"
 )
 
 var csrfTokens []string
@@ -25,10 +36,11 @@ var csrfMut sync.Mutex
 // Check for CSRF token on /rest/ URLs. If a correct one is not given, reject
 // the request with 403. For / and /index.html, set a new CSRF cookie if none
 // is currently set.
-func csrfMiddleware(prefix string, next http.Handler) http.Handler {
+func csrfMiddleware(prefix, apiKey string, next http.Handler) http.Handler {
+	loadCsrfTokens()
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// Allow requests carrying a valid API key
-		if validAPIKey(r.Header.Get("X-API-Key")) {
+		if apiKey != "" && r.Header.Get("X-API-Key") == apiKey {
 			next.ServeHTTP(w, r)
 			return
 		}
@@ -76,13 +88,7 @@ func validCsrfToken(token string) bool {
 }
 
 func newCsrfToken() string {
-	bs := make([]byte, 30)
-	_, err := rand.Reader.Read(bs)
-	if err != nil {
-		l.Fatalln(err)
-	}
-
-	token := base64.StdEncoding.EncodeToString(bs)
+	token := randomString(30)
 
 	csrfMut.Lock()
 	csrfTokens = append(csrfTokens, token)
@@ -134,3 +140,13 @@ func loadCsrfTokens() {
 		csrfTokens = append(csrfTokens, s.Text())
 	}
 }
+
+func randomString(len int) string {
+	bs := make([]byte, len)
+	_, err := rand.Reader.Read(bs)
+	if err != nil {
+		l.Fatalln(err)
+	}
+
+	return base64.StdEncoding.EncodeToString(bs)
+}

cmd/syncthing/gui_solaris.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 //+build solaris
 

cmd/syncthing/gui_unix.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 //+build !windows,!solaris
 

cmd/syncthing/gui_windows.go

@@ -0,0 +1,63 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
+
+//+build windows
+
+package main
+
+import (
+	"syscall"
+	"time"
+)
+
+func init() {
+	go trackCPUUsage()
+}
+
+func trackCPUUsage() {
+	handle, err := syscall.GetCurrentProcess()
+	if err != nil {
+		l.Warnln("Cannot track CPU usage:", err)
+		return
+	}
+
+	var ctime, etime, ktime, utime syscall.Filetime
+	err = syscall.GetProcessTimes(handle, &ctime, &etime, &ktime, &utime)
+	if err != nil {
+		l.Warnln("Cannot track CPU usage:", err)
+		return
+	}
+
+	prevTime := ctime.Nanoseconds()
+	prevUsage := ktime.Nanoseconds() + utime.Nanoseconds() // Always overflows
+
+	for _ = range time.NewTicker(time.Second).C {
+		err := syscall.GetProcessTimes(handle, &ctime, &etime, &ktime, &utime)
+		if err != nil {
+			continue
+		}
+
+		curTime := time.Now().UnixNano()
+		timeDiff := curTime - prevTime
+		curUsage := ktime.Nanoseconds() + utime.Nanoseconds()
+		usageDiff := curUsage - prevUsage
+		cpuUsageLock.Lock()
+		copy(cpuUsagePercent[1:], cpuUsagePercent[0:])
+		cpuUsagePercent[0] = 100 * float64(usageDiff) / float64(timeDiff)
+		cpuUsageLock.Unlock()
+		prevTime = curTime
+		prevUsage = curUsage
+	}
+}

cmd/syncthing/heapprof.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 package main
 

cmd/syncthing/limitedreader.go

@@ -0,0 +1,35 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"io"
+
+	"github.com/juju/ratelimit"
+)
+
+type limitedReader struct {
+	r      io.Reader
+	bucket *ratelimit.Bucket
+}
+
+func (r *limitedReader) Read(buf []byte) (int, error) {
+	n, err := r.r.Read(buf)
+	if r.bucket != nil {
+		r.bucket.Wait(int64(n))
+	}
+	return n, err
+}

cmd/syncthing/limitedwriter.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 package main
 

cmd/syncthing/main_test.go

@@ -1,7 +1,119 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
-package main_test
+package main
 
-// Empty test file to generate 0% coverage rather than no coverage
+import (
+	"os"
+	"testing"
+
+	"github.com/syncthing/syncthing/internal/config"
+	"github.com/syncthing/syncthing/internal/files"
+	"github.com/syncthing/syncthing/internal/model"
+	"github.com/syncthing/syncthing/internal/protocol"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+func TestSanityCheck(t *testing.T) {
+	fcfg := config.FolderConfiguration{
+		ID:   "folder",
+		Path: "testdata/testfolder",
+	}
+	cfg := config.Wrap("/tmp/test", config.Configuration{
+		Folders: []config.FolderConfiguration{fcfg},
+	})
+
+	for _, file := range []string{".stfolder", "testfolder", "testfolder/.stfolder"} {
+		_, err := os.Stat("testdata/" + file)
+		if err == nil {
+			t.Error("Found unexpected file")
+		}
+	}
+
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+
+	// Case 1 - new folder, directory and marker created
+
+	m := model.NewModel(cfg, "device", "syncthing", "dev", db)
+	sanityCheckFolders(cfg, m)
+
+	if cfg.Folders()["folder"].Invalid != "" {
+		t.Error("Unexpected error", cfg.Folders()["folder"].Invalid)
+	}
+
+	s, err := os.Stat("testdata/testfolder")
+	if err != nil || !s.IsDir() {
+		t.Error(err)
+	}
+
+	_, err = os.Stat("testdata/testfolder/.stfolder")
+	if err != nil {
+		t.Error(err)
+	}
+
+	os.Remove("testdata/testfolder/.stfolder")
+	os.Remove("testdata/testfolder/")
+
+	// Case 2 - new folder, marker created
+
+	fcfg.Path = "testdata/"
+	cfg = config.Wrap("/tmp/test", config.Configuration{
+		Folders: []config.FolderConfiguration{fcfg},
+	})
+
+	m = model.NewModel(cfg, "device", "syncthing", "dev", db)
+	sanityCheckFolders(cfg, m)
+
+	if cfg.Folders()["folder"].Invalid != "" {
+		t.Error("Unexpected error", cfg.Folders()["folder"].Invalid)
+	}
+
+	_, err = os.Stat("testdata/.stfolder")
+	if err != nil {
+		t.Error(err)
+	}
+
+	os.Remove("testdata/.stfolder")
+
+	// Case 3 - marker missing
+
+	set := files.NewSet("folder", db)
+	set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
+		{Name: "dummyfile"},
+	})
+
+	m = model.NewModel(cfg, "device", "syncthing", "dev", db)
+	sanityCheckFolders(cfg, m)
+
+	if cfg.Folders()["folder"].Invalid != "folder marker missing" {
+		t.Error("Incorrect error")
+	}
+
+	// Case 4 - path missing
+
+	fcfg.Path = "testdata/testfolder"
+	cfg = config.Wrap("/tmp/test", config.Configuration{
+		Folders: []config.FolderConfiguration{fcfg},
+	})
+
+	m = model.NewModel(cfg, "device", "syncthing", "dev", db)
+	sanityCheckFolders(cfg, m)
+
+	if cfg.Folders()["folder"].Invalid != "folder path missing" {
+		t.Error("Incorrect error")
+	}
+}

cmd/syncthing/memsize_darwin.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 package main
 

cmd/syncthing/memsize_linux.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 package main
 

cmd/syncthing/memsize_solaris.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 // +build solaris
 

cmd/syncthing/memsize_unimpl.go

@@ -1,8 +1,19 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
-// +build freebsd
+// +build freebsd openbsd
 
 package main
 

cmd/syncthing/memsize_windows.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 package main
 

cmd/syncthing/monitor.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 package main
 
@@ -11,10 +22,13 @@ import (
 	"os/exec"
 	"os/signal"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"sync"
 	"syscall"
 	"time"
+
+	"github.com/syncthing/syncthing/internal/osutil"
 )
 
 var (
@@ -30,8 +44,35 @@ const (
 
 func monitorMain() {
 	os.Setenv("STNORESTART", "yes")
+	os.Setenv("STMONITORED", "yes")
 	l.SetPrefix("[monitor] ")
 
+	var err error
+	var dst io.Writer = os.Stdout
+
+	if logFile != "" {
+		var fileDst io.Writer
+
+		fileDst, err = os.Create(logFile)
+		if err != nil {
+			l.Fatalln("log file:", err)
+		}
+
+		if runtime.GOOS == "windows" {
+			// Translate line breaks to Windows standard
+			fileDst = osutil.ReplacingWriter{
+				Writer: fileDst,
+				From:   '\n',
+				To:     []byte{'\r', '\n'},
+			}
+		}
+
+		// Log to both stdout and file.
+		dst = io.MultiWriter(dst, fileDst)
+
+		l.Infof(`Log output saved to file "%s"`, logFile)
+	}
+
 	args := os.Args
 	var restarts [countRestarts]time.Time
 
@@ -52,12 +93,12 @@ func monitorMain() {
 
 		stderr, err := cmd.StderrPipe()
 		if err != nil {
-			l.Fatalln(err)
+			l.Fatalln("stderr:", err)
 		}
 
 		stdout, err := cmd.StdoutPipe()
 		if err != nil {
-			l.Fatalln(err)
+			l.Fatalln("stdout:", err)
 		}
 
 		l.Infoln("Starting syncthing")
@@ -66,13 +107,17 @@ func monitorMain() {
 			l.Fatalln(err)
 		}
 
+		// Let the next child process know that this is not the first time
+		// it's starting up.
+		os.Setenv("STRESTART", "yes")
+
 		stdoutMut.Lock()
 		stdoutFirstLines = make([]string, 0, 10)
 		stdoutLastLines = make([]string, 0, 50)
 		stdoutMut.Unlock()
 
-		go copyStderr(stderr)
-		go copyStdout(stdout)
+		go copyStderr(stderr, dst)
+		go copyStdout(stdout, dst)
 
 		exit := make(chan error)
 
@@ -86,10 +131,26 @@ func monitorMain() {
 			cmd.Process.Kill()
 			<-exit
 			return
-		case <-exit:
+
+		case err = <-exit:
 			if err == nil {
 				// Successfull exit indicates an intentional shutdown
 				return
+			} else if exiterr, ok := err.(*exec.ExitError); ok {
+				if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
+					switch status.ExitStatus() {
+					case exitUpgrading:
+						// Restart the monitor process to release the .old
+						// binary as part of the upgrade process.
+						l.Infoln("Restarting monitor...")
+						os.Setenv("STNORESTART", "")
+						err := exec.Command(args[0], args[1:]...).Start()
+						if err != nil {
+							l.Warnln("restart:", err)
+						}
+						return
+					}
+				}
 			}
 		}
 
@@ -98,21 +159,18 @@ func monitorMain() {
 	}
 }
 
-func copyStderr(stderr io.ReadCloser) {
+func copyStderr(stderr io.ReadCloser, dst io.Writer) {
 	br := bufio.NewReader(stderr)
 
 	var panicFd *os.File
 	for {
 		line, err := br.ReadString('\n')
 		if err != nil {
-			if err != io.EOF {
-				l.Warnln("stderr:", err)
-			}
 			return
 		}
 
 		if panicFd == nil {
-			os.Stderr.WriteString(line)
+			dst.Write([]byte(line))
 
 			if strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:") {
 				panicFd, err = os.Create(filepath.Join(confDir, time.Now().Format("panic-20060102-150405.log")))
@@ -122,7 +180,7 @@ func copyStderr(stderr io.ReadCloser) {
 				}
 
 				l.Warnf("Panic detected, writing to \"%s\"", panicFd.Name())
-				l.Warnln("Please create an issue at https://github.com/syncting/syncthing/issues/ with the panic log attached")
+				l.Warnln("Please create an issue at https://github.com/syncthing/syncthing/issues/ with the panic log attached")
 
 				stdoutMut.Lock()
 				for _, line := range stdoutFirstLines {
@@ -132,7 +190,10 @@ func copyStderr(stderr io.ReadCloser) {
 				for _, line := range stdoutLastLines {
 					panicFd.WriteString(line)
 				}
+				stdoutMut.Unlock()
 			}
+
+			panicFd.WriteString("Panic at " + time.Now().Format(time.RFC3339) + "\n")
 		}
 
 		if panicFd != nil {
@@ -141,27 +202,25 @@ func copyStderr(stderr io.ReadCloser) {
 	}
 }
 
-func copyStdout(stderr io.ReadCloser) {
-	br := bufio.NewReader(stderr)
+func copyStdout(stdout io.ReadCloser, dst io.Writer) {
+	br := bufio.NewReader(stdout)
 	for {
 		line, err := br.ReadString('\n')
 		if err != nil {
-			if err != io.EOF {
-				l.Warnln("stdout:", err)
-			}
 			return
 		}
 
 		stdoutMut.Lock()
 		if len(stdoutFirstLines) < cap(stdoutFirstLines) {
 			stdoutFirstLines = append(stdoutFirstLines, line)
+		} else {
+			if l := len(stdoutLastLines); l == cap(stdoutLastLines) {
+				stdoutLastLines = stdoutLastLines[:l-1]
+			}
+			stdoutLastLines = append(stdoutLastLines, line)
 		}
-		if l := len(stdoutLastLines); l == cap(stdoutLastLines) {
-			stdoutLastLines = stdoutLastLines[:l-1]
-		}
-		stdoutLastLines = append(stdoutLastLines, line)
 		stdoutMut.Unlock()
 
-		os.Stdout.WriteString(line)
+		dst.Write([]byte(line))
 	}
 }

cmd/syncthing/openurl_unix.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 // +build !windows
 

cmd/syncthing/openurl_windows.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 // +build windows
 

cmd/syncthing/perfstats_unix.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 // +build !solaris,!windows
 
@@ -12,6 +23,8 @@ import (
 	"runtime"
 	"syscall"
 	"time"
+
+	"github.com/syncthing/syncthing/internal/protocol"
 )
 
 func init() {
@@ -30,6 +43,7 @@ func savePerfStats(file string) {
 	var prevTime int64
 	var rusage syscall.Rusage
 	var memstats runtime.MemStats
+	var prevIn, prevOut uint64
 
 	t0 := time.Now()
 	for t := range time.NewTicker(250 * time.Millisecond).C {
@@ -41,11 +55,18 @@ func savePerfStats(file string) {
 		cpuUsagePercent := 100 * float64(usageDiff) / float64(timeDiff)
 		prevTime = curTime
 		prevUsage = curUsage
+		in, out := protocol.TotalInOut()
+		var inRate, outRate float64
+		if timeDiff > 0 {
+			inRate = float64(in-prevIn) / (float64(timeDiff) / 1e9)    // bytes per second
+			outRate = float64(out-prevOut) / (float64(timeDiff) / 1e9) // bytes per second
+		}
+		prevIn, prevOut = in, out
 
 		runtime.ReadMemStats(&memstats)
 
 		startms := int(t.Sub(t0).Seconds() * 1000)
 
-		fmt.Fprintf(fd, "%d\t%f\t%d\t%d\n", startms, cpuUsagePercent, memstats.Alloc, memstats.Sys-memstats.HeapReleased)
+		fmt.Fprintf(fd, "%d\t%f\t%d\t%d\t%.0f\t%.0f\n", startms, cpuUsagePercent, memstats.Alloc, memstats.Sys-memstats.HeapReleased, inRate, outRate)
 	}
 }

cmd/syncthing/tls.go

@@ -1,10 +1,22 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 package main
 
 import (
+	"bufio"
 	"crypto/rand"
 	"crypto/rsa"
 	"crypto/sha256"
@@ -13,8 +25,10 @@ import (
 	"crypto/x509/pkix"
 	"encoding/binary"
 	"encoding/pem"
+	"io"
 	"math/big"
 	mr "math/rand"
+	"net"
 	"os"
 	"path/filepath"
 	"time"
@@ -42,7 +56,9 @@ func newCertificate(dir string, prefix string) {
 	l.Infoln("Generating RSA key and certificate...")
 
 	priv, err := rsa.GenerateKey(rand.Reader, tlsRSABits)
-	l.FatalErr(err)
+	if err != nil {
+		l.Fatalln("generate key:", err)
+	}
 
 	notBefore := time.Now()
 	notAfter := time.Date(2049, 12, 31, 23, 59, 59, 0, time.UTC)
@@ -61,15 +77,71 @@ func newCertificate(dir string, prefix string) {
 	}
 
 	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
-	l.FatalErr(err)
+	if err != nil {
+		l.Fatalln("create cert:", err)
+	}
 
 	certOut, err := os.Create(filepath.Join(dir, prefix+"cert.pem"))
-	l.FatalErr(err)
-	pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
-	certOut.Close()
+	if err != nil {
+		l.Fatalln("save cert:", err)
+	}
+	err = pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
+	if err != nil {
+		l.Fatalln("save cert:", err)
+	}
+	err = certOut.Close()
+	if err != nil {
+		l.Fatalln("save cert:", err)
+	}
 
 	keyOut, err := os.OpenFile(filepath.Join(dir, prefix+"key.pem"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
-	l.FatalErr(err)
-	pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
-	keyOut.Close()
+	if err != nil {
+		l.Fatalln("save key:", err)
+	}
+	err = pem.Encode(keyOut, &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(priv)})
+	if err != nil {
+		l.Fatalln("save key:", err)
+	}
+	err = keyOut.Close()
+	if err != nil {
+		l.Fatalln("save key:", err)
+	}
+}
+
+type DowngradingListener struct {
+	net.Listener
+	TLSConfig *tls.Config
+}
+
+type WrappedConnection struct {
+	io.Reader
+	net.Conn
+}
+
+func (l *DowngradingListener) Accept() (net.Conn, error) {
+	conn, err := l.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+
+	br := bufio.NewReader(conn)
+	bs, err := br.Peek(1)
+	if err != nil {
+		// We hit a read error here, but the Accept() call succeeded so we must not return an error.
+		// We return the connection as is and let whoever tries to use it deal with the error.
+		return conn, nil
+	}
+
+	wrapper := &WrappedConnection{br, conn}
+
+	// 0x16 is the first byte of a TLS handshake
+	if bs[0] == 0x16 {
+		return tls.Server(wrapper, l.TLSConfig), nil
+	}
+
+	return wrapper, nil
+}
+
+func (c *WrappedConnection) Read(b []byte) (n int, err error) {
+	return c.Reader.Read(b)
 }

cmd/syncthing/usage.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 package main
 
@@ -36,9 +47,8 @@ func usageFor(fs *flag.FlagSet, usage string, extra string) func() {
 			var opt = "  -" + f.Name
 
 			if f.DefValue != "false" {
-				opt += "=" + fmt.Sprintf("%q", f.DefValue)
+				opt += "=" + fmt.Sprintf(`"%s"`, f.DefValue)
 			}
-
 			options = append(options, []string{opt, f.Usage})
 		})
 

cmd/syncthing/usage_report.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 package main
 
@@ -15,7 +26,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/syncthing/syncthing/model"
+	"github.com/syncthing/syncthing/internal/model"
 )
 
 // Current version number of the usage report, for acceptance purposes. If
@@ -31,13 +42,13 @@ func reportData(m *model.Model) map[string]interface{} {
 	res["version"] = Version
 	res["longVersion"] = LongVersion
 	res["platform"] = runtime.GOOS + "-" + runtime.GOARCH
-	res["numRepos"] = len(cfg.Repositories)
-	res["numNodes"] = len(cfg.Nodes)
+	res["numFolders"] = len(cfg.Folders())
+	res["numDevices"] = len(cfg.Devices())
 
 	var totFiles, maxFiles int
 	var totBytes, maxBytes int64
-	for _, repo := range cfg.Repositories {
-		files, _, bytes := m.GlobalSize(repo.ID)
+	for folderID := range cfg.Folders() {
+		files, _, bytes := m.GlobalSize(folderID)
 		totFiles += files
 		totBytes += bytes
 		if files > maxFiles {
@@ -49,9 +60,9 @@ func reportData(m *model.Model) map[string]interface{} {
 	}
 
 	res["totFiles"] = totFiles
-	res["repoMaxFiles"] = maxFiles
+	res["folderMaxFiles"] = maxFiles
 	res["totMiB"] = totBytes / 1024 / 1024
-	res["repoMaxMiB"] = maxBytes / 1024 / 1024
+	res["folderMaxMiB"] = maxBytes / 1024 / 1024
 
 	var mem runtime.MemStats
 	runtime.ReadMemStats(&mem)

cmd/todos/main.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 // +build ignore
 

cmd/transifexdl/main.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 // +build ignore
 
@@ -52,12 +63,11 @@ func main() {
 	var langs []string
 	for code, stat := range stats {
 		code = strings.Replace(code, "_", "-", 1)
-		if !curValidLangs[code] {
-			if pct := 100 * stat.Translated / (stat.Translated + stat.Untranslated); pct < 95 {
-				log.Printf("Skipping language %q (too low completion ratio %d%%)", code, pct)
-				os.Remove("lang-" + code + ".json")
-				continue
-			}
+		pct := 100 * stat.Translated / (stat.Translated + stat.Untranslated)
+		if pct < 75 || !curValidLangs[code] && pct < 95 {
+			log.Printf("Skipping language %q (too low completion ratio %d%%)", code, pct)
+			os.Remove("lang-" + code + ".json")
+			continue
 		}
 
 		langs = append(langs, code)

cmd/translate/main.go

@@ -1,6 +1,17 @@
 // Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
 
 // +build ignore
 

config/config.go

@@ -1,522 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
-
-// Package config implements reading and writing of the syncthing configuration file.
-package config
-
-import (
-	"encoding/xml"
-	"fmt"
-	"io"
-	"os"
-	"reflect"
-	"sort"
-	"strconv"
-
-	"code.google.com/p/go.crypto/bcrypt"
-	"github.com/syncthing/syncthing/logger"
-	"github.com/syncthing/syncthing/protocol"
-)
-
-var l = logger.DefaultLogger
-
-type Configuration struct {
-	Version      int                       `xml:"version,attr" default:"3"`
-	Repositories []RepositoryConfiguration `xml:"repository"`
-	Nodes        []NodeConfiguration       `xml:"node"`
-	GUI          GUIConfiguration          `xml:"gui"`
-	Options      OptionsConfiguration      `xml:"options"`
-	XMLName      xml.Name                  `xml:"configuration" json:"-"`
-}
-
-type RepositoryConfiguration struct {
-	ID              string                        `xml:"id,attr"`
-	Directory       string                        `xml:"directory,attr"`
-	Nodes           []RepositoryNodeConfiguration `xml:"node"`
-	ReadOnly        bool                          `xml:"ro,attr"`
-	RescanIntervalS int                           `xml:"rescanIntervalS,attr" default:"60"`
-	IgnorePerms     bool                          `xml:"ignorePerms,attr"`
-	Invalid         string                        `xml:"-"` // Set at runtime when there is an error, not saved
-	Versioning      VersioningConfiguration       `xml:"versioning"`
-
-	nodeIDs []protocol.NodeID
-}
-
-type VersioningConfiguration struct {
-	Type   string `xml:"type,attr"`
-	Params map[string]string
-}
-
-type InternalVersioningConfiguration struct {
-	Type   string          `xml:"type,attr,omitempty"`
-	Params []InternalParam `xml:"param"`
-}
-
-type InternalParam struct {
-	Key string `xml:"key,attr"`
-	Val string `xml:"val,attr"`
-}
-
-func (c *VersioningConfiguration) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
-	var tmp InternalVersioningConfiguration
-	tmp.Type = c.Type
-	for k, v := range c.Params {
-		tmp.Params = append(tmp.Params, InternalParam{k, v})
-	}
-
-	return e.EncodeElement(tmp, start)
-
-}
-
-func (c *VersioningConfiguration) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
-	var tmp InternalVersioningConfiguration
-	err := d.DecodeElement(&tmp, &start)
-	if err != nil {
-		return err
-	}
-
-	c.Type = tmp.Type
-	c.Params = make(map[string]string, len(tmp.Params))
-	for _, p := range tmp.Params {
-		c.Params[p.Key] = p.Val
-	}
-	return nil
-}
-
-func (r *RepositoryConfiguration) NodeIDs() []protocol.NodeID {
-	if r.nodeIDs == nil {
-		for _, n := range r.Nodes {
-			r.nodeIDs = append(r.nodeIDs, n.NodeID)
-		}
-	}
-	return r.nodeIDs
-}
-
-type NodeConfiguration struct {
-	NodeID      protocol.NodeID `xml:"id,attr"`
-	Name        string          `xml:"name,attr,omitempty"`
-	Addresses   []string        `xml:"address,omitempty"`
-	Compression bool            `xml:"compression,attr"`
-	CertName    string          `xml:"certName,attr,omitempty"`
-}
-
-type RepositoryNodeConfiguration struct {
-	NodeID protocol.NodeID `xml:"id,attr"`
-
-	Deprecated_Name      string   `xml:"name,attr,omitempty" json:"-"`
-	Deprecated_Addresses []string `xml:"address,omitempty" json:"-"`
-}
-
-type OptionsConfiguration struct {
-	ListenAddress      []string `xml:"listenAddress" default:"0.0.0.0:22000"`
-	GlobalAnnServer    string   `xml:"globalAnnounceServer" default:"announce.syncthing.net:22026"`
-	GlobalAnnEnabled   bool     `xml:"globalAnnounceEnabled" default:"true"`
-	LocalAnnEnabled    bool     `xml:"localAnnounceEnabled" default:"true"`
-	LocalAnnPort       int      `xml:"localAnnouncePort" default:"21025"`
-	LocalAnnMCAddr     string   `xml:"localAnnounceMCAddr" default:"[ff32::5222]:21026"`
-	ParallelRequests   int      `xml:"parallelRequests" default:"16"`
-	MaxSendKbps        int      `xml:"maxSendKbps"`
-	ReconnectIntervalS int      `xml:"reconnectionIntervalS" default:"60"`
-	StartBrowser       bool     `xml:"startBrowser" default:"true"`
-	UPnPEnabled        bool     `xml:"upnpEnabled" default:"true"`
-	UPnPLease          int      `xml:"upnpLeaseMinutes" default:"0"`
-	UPnPRenewal        int      `xml:"upnpRenewalMinutes" default:"30"`
-	URAccepted         int      `xml:"urAccepted"` // Accepted usage reporting version; 0 for off (undecided), -1 for off (permanently)
-
-	Deprecated_RescanIntervalS int    `xml:"rescanIntervalS,omitempty" json:"-"`
-	Deprecated_UREnabled       bool   `xml:"urEnabled,omitempty" json:"-"`
-	Deprecated_URDeclined      bool   `xml:"urDeclined,omitempty" json:"-"`
-	Deprecated_ReadOnly        bool   `xml:"readOnly,omitempty" json:"-"`
-	Deprecated_GUIEnabled      bool   `xml:"guiEnabled,omitempty" json:"-"`
-	Deprecated_GUIAddress      string `xml:"guiAddress,omitempty" json:"-"`
-}
-
-type GUIConfiguration struct {
-	Enabled  bool   `xml:"enabled,attr" default:"true"`
-	Address  string `xml:"address" default:"127.0.0.1:8080"`
-	User     string `xml:"user,omitempty"`
-	Password string `xml:"password,omitempty"`
-	UseTLS   bool   `xml:"tls,attr"`
-	APIKey   string `xml:"apikey,omitempty"`
-}
-
-func (cfg *Configuration) NodeMap() map[protocol.NodeID]NodeConfiguration {
-	m := make(map[protocol.NodeID]NodeConfiguration, len(cfg.Nodes))
-	for _, n := range cfg.Nodes {
-		m[n.NodeID] = n
-	}
-	return m
-}
-
-func (cfg *Configuration) GetNodeConfiguration(nodeid protocol.NodeID) *NodeConfiguration {
-	for i, node := range cfg.Nodes {
-		if node.NodeID == nodeid {
-			return &cfg.Nodes[i]
-		}
-	}
-	return nil
-}
-
-func (cfg *Configuration) RepoMap() map[string]RepositoryConfiguration {
-	m := make(map[string]RepositoryConfiguration, len(cfg.Repositories))
-	for _, r := range cfg.Repositories {
-		m[r.ID] = r
-	}
-	return m
-}
-
-func setDefaults(data interface{}) error {
-	s := reflect.ValueOf(data).Elem()
-	t := s.Type()
-
-	for i := 0; i < s.NumField(); i++ {
-		f := s.Field(i)
-		tag := t.Field(i).Tag
-
-		v := tag.Get("default")
-		if len(v) > 0 {
-			switch f.Interface().(type) {
-			case string:
-				f.SetString(v)
-
-			case int:
-				i, err := strconv.ParseInt(v, 10, 64)
-				if err != nil {
-					return err
-				}
-				f.SetInt(i)
-
-			case bool:
-				f.SetBool(v == "true")
-
-			case []string:
-				// We don't do anything with string slices here. Any default
-				// we set will be appended to by the XML decoder, so we fill
-				// those after decoding.
-
-			default:
-				panic(f.Type())
-			}
-		}
-	}
-	return nil
-}
-
-// fillNilSlices sets default value on slices that are still nil.
-func fillNilSlices(data interface{}) error {
-	s := reflect.ValueOf(data).Elem()
-	t := s.Type()
-
-	for i := 0; i < s.NumField(); i++ {
-		f := s.Field(i)
-		tag := t.Field(i).Tag
-
-		v := tag.Get("default")
-		if len(v) > 0 {
-			switch f.Interface().(type) {
-			case []string:
-				if f.IsNil() {
-					rv := reflect.MakeSlice(reflect.TypeOf([]string{}), 1, 1)
-					rv.Index(0).SetString(v)
-					f.Set(rv)
-				}
-			}
-		}
-	}
-	return nil
-}
-
-func Save(wr io.Writer, cfg Configuration) error {
-	e := xml.NewEncoder(wr)
-	e.Indent("", "    ")
-	err := e.Encode(cfg)
-	if err != nil {
-		return err
-	}
-	_, err = wr.Write([]byte("\n"))
-	return err
-}
-
-func uniqueStrings(ss []string) []string {
-	var m = make(map[string]bool, len(ss))
-	for _, s := range ss {
-		m[s] = true
-	}
-
-	var us = make([]string, 0, len(m))
-	for k := range m {
-		us = append(us, k)
-	}
-
-	return us
-}
-
-func Load(rd io.Reader, myID protocol.NodeID) (Configuration, error) {
-	var cfg Configuration
-
-	setDefaults(&cfg)
-	setDefaults(&cfg.Options)
-	setDefaults(&cfg.GUI)
-
-	var err error
-	if rd != nil {
-		err = xml.NewDecoder(rd).Decode(&cfg)
-	}
-
-	fillNilSlices(&cfg.Options)
-
-	cfg.Options.ListenAddress = uniqueStrings(cfg.Options.ListenAddress)
-
-	// Initialize an empty slice for repositories if the config has none
-	if cfg.Repositories == nil {
-		cfg.Repositories = []RepositoryConfiguration{}
-	}
-
-	// Check for missing, bad or duplicate repository ID:s
-	var seenRepos = map[string]*RepositoryConfiguration{}
-	var uniqueCounter int
-	for i := range cfg.Repositories {
-		repo := &cfg.Repositories[i]
-
-		if len(repo.Directory) == 0 {
-			repo.Invalid = "no directory configured"
-			continue
-		}
-
-		if repo.ID == "" {
-			repo.ID = "default"
-		}
-
-		if seen, ok := seenRepos[repo.ID]; ok {
-			l.Warnf("Multiple repositories with ID %q; disabling", repo.ID)
-
-			seen.Invalid = "duplicate repository ID"
-			if seen.ID == repo.ID {
-				uniqueCounter++
-				seen.ID = fmt.Sprintf("%s~%d", repo.ID, uniqueCounter)
-			}
-			repo.Invalid = "duplicate repository ID"
-			uniqueCounter++
-			repo.ID = fmt.Sprintf("%s~%d", repo.ID, uniqueCounter)
-		} else {
-			seenRepos[repo.ID] = repo
-		}
-	}
-
-	if cfg.Options.Deprecated_URDeclined {
-		cfg.Options.URAccepted = -1
-	}
-	cfg.Options.Deprecated_URDeclined = false
-	cfg.Options.Deprecated_UREnabled = false
-
-	// Upgrade to v2 configuration if appropriate
-	if cfg.Version == 1 {
-		convertV1V2(&cfg)
-	}
-
-	// Upgrade to v3 configuration if appropriate
-	if cfg.Version == 2 {
-		convertV2V3(&cfg)
-	}
-
-	// Upgrade to v4 configuration if appropriate
-	if cfg.Version == 3 {
-		convertV3V4(&cfg)
-	}
-
-	// Hash old cleartext passwords
-	if len(cfg.GUI.Password) > 0 && cfg.GUI.Password[0] != '$' {
-		hash, err := bcrypt.GenerateFromPassword([]byte(cfg.GUI.Password), 0)
-		if err != nil {
-			l.Warnln("bcrypting password:", err)
-		} else {
-			cfg.GUI.Password = string(hash)
-		}
-	}
-
-	// Build a list of available nodes
-	existingNodes := make(map[protocol.NodeID]bool)
-	existingNodes[myID] = true
-	for _, node := range cfg.Nodes {
-		existingNodes[node.NodeID] = true
-	}
-
-	// Ensure this node is present in all relevant places
-	me := cfg.GetNodeConfiguration(myID)
-	if me == nil {
-		myName, _ := os.Hostname()
-		cfg.Nodes = append(cfg.Nodes, NodeConfiguration{
-			NodeID: myID,
-			Name:   myName,
-		})
-	}
-	sort.Sort(NodeConfigurationList(cfg.Nodes))
-	// Ensure that any loose nodes are not present in the wrong places
-	// Ensure that there are no duplicate nodes
-	for i := range cfg.Repositories {
-		cfg.Repositories[i].Nodes = ensureNodePresent(cfg.Repositories[i].Nodes, myID)
-		cfg.Repositories[i].Nodes = ensureExistingNodes(cfg.Repositories[i].Nodes, existingNodes)
-		cfg.Repositories[i].Nodes = ensureNoDuplicates(cfg.Repositories[i].Nodes)
-		sort.Sort(RepositoryNodeConfigurationList(cfg.Repositories[i].Nodes))
-	}
-
-	// An empty address list is equivalent to a single "dynamic" entry
-	for i := range cfg.Nodes {
-		n := &cfg.Nodes[i]
-		if len(n.Addresses) == 0 || len(n.Addresses) == 1 && n.Addresses[0] == "" {
-			n.Addresses = []string{"dynamic"}
-		}
-	}
-
-	return cfg, err
-}
-
-func convertV3V4(cfg *Configuration) {
-	// In previous versions, rescan interval was common for each repository.
-	// From now, it can be set independently. We have to make sure, that after upgrade
-	// the individual rescan interval will be defined for every existing repository.
-	for i := range cfg.Repositories {
-		cfg.Repositories[i].RescanIntervalS = cfg.Options.Deprecated_RescanIntervalS
-	}
-
-	cfg.Options.Deprecated_RescanIntervalS = 0
-
-	// In previous versions, repositories held full node configurations.
-	// Since that's the only place where node configs were in V1, we still have
-	// to define the deprecated fields to be able to upgrade from V1 to V4.
-	for i, repo := range cfg.Repositories {
-
-		for j := range repo.Nodes {
-			rncfg := cfg.Repositories[i].Nodes[j]
-			rncfg.Deprecated_Name = ""
-			rncfg.Deprecated_Addresses = nil
-		}
-	}
-
-	cfg.Version = 4
-}
-
-func convertV2V3(cfg *Configuration) {
-	// In previous versions, compression was always on. When upgrading, enable
-	// compression on all existing new. New nodes will get compression on by
-	// default by the GUI.
-	for i := range cfg.Nodes {
-		cfg.Nodes[i].Compression = true
-	}
-
-	// The global discovery format and port number changed in v0.9. Having the
-	// default announce server but old port number is guaranteed to be legacy.
-	if cfg.Options.GlobalAnnServer == "announce.syncthing.net:22025" {
-		cfg.Options.GlobalAnnServer = "announce.syncthing.net:22026"
-	}
-
-	cfg.Version = 3
-}
-
-func convertV1V2(cfg *Configuration) {
-	// Collect the list of nodes.
-	// Replace node configs inside repositories with only a reference to the nide ID.
-	// Set all repositories to read only if the global read only flag is set.
-	var nodes = map[string]RepositoryNodeConfiguration{}
-	for i, repo := range cfg.Repositories {
-		cfg.Repositories[i].ReadOnly = cfg.Options.Deprecated_ReadOnly
-		for j, node := range repo.Nodes {
-			id := node.NodeID.String()
-			if _, ok := nodes[id]; !ok {
-				nodes[id] = node
-			}
-			cfg.Repositories[i].Nodes[j] = RepositoryNodeConfiguration{NodeID: node.NodeID}
-		}
-	}
-	cfg.Options.Deprecated_ReadOnly = false
-
-	// Set and sort the list of nodes.
-	for _, node := range nodes {
-		cfg.Nodes = append(cfg.Nodes, NodeConfiguration{
-			NodeID:    node.NodeID,
-			Name:      node.Deprecated_Name,
-			Addresses: node.Deprecated_Addresses,
-		})
-	}
-	sort.Sort(NodeConfigurationList(cfg.Nodes))
-
-	// GUI
-	cfg.GUI.Address = cfg.Options.Deprecated_GUIAddress
-	cfg.GUI.Enabled = cfg.Options.Deprecated_GUIEnabled
-	cfg.Options.Deprecated_GUIEnabled = false
-	cfg.Options.Deprecated_GUIAddress = ""
-
-	cfg.Version = 2
-}
-
-type NodeConfigurationList []NodeConfiguration
-
-func (l NodeConfigurationList) Less(a, b int) bool {
-	return l[a].NodeID.Compare(l[b].NodeID) == -1
-}
-func (l NodeConfigurationList) Swap(a, b int) {
-	l[a], l[b] = l[b], l[a]
-}
-func (l NodeConfigurationList) Len() int {
-	return len(l)
-}
-
-type RepositoryNodeConfigurationList []RepositoryNodeConfiguration
-
-func (l RepositoryNodeConfigurationList) Less(a, b int) bool {
-	return l[a].NodeID.Compare(l[b].NodeID) == -1
-}
-func (l RepositoryNodeConfigurationList) Swap(a, b int) {
-	l[a], l[b] = l[b], l[a]
-}
-func (l RepositoryNodeConfigurationList) Len() int {
-	return len(l)
-}
-
-func ensureNodePresent(nodes []RepositoryNodeConfiguration, myID protocol.NodeID) []RepositoryNodeConfiguration {
-	for _, node := range nodes {
-		if node.NodeID.Equals(myID) {
-			return nodes
-		}
-	}
-
-	nodes = append(nodes, RepositoryNodeConfiguration{
-		NodeID: myID,
-	})
-
-	return nodes
-}
-
-func ensureExistingNodes(nodes []RepositoryNodeConfiguration, existingNodes map[protocol.NodeID]bool) []RepositoryNodeConfiguration {
-	count := len(nodes)
-	i := 0
-loop:
-	for i < count {
-		if _, ok := existingNodes[nodes[i].NodeID]; !ok {
-			nodes[i] = nodes[count-1]
-			count--
-			continue loop
-		}
-		i++
-	}
-	return nodes[0:count]
-}
-
-func ensureNoDuplicates(nodes []RepositoryNodeConfiguration) []RepositoryNodeConfiguration {
-	count := len(nodes)
-	i := 0
-	seenNodes := make(map[protocol.NodeID]bool)
-loop:
-	for i < count {
-		id := nodes[i].NodeID
-		if _, ok := seenNodes[id]; ok {
-			nodes[i] = nodes[count-1]
-			count--
-			continue loop
-		}
-		seenNodes[id] = true
-		i++
-	}
-	return nodes[0:count]
-}

config/config_test.go

@@ -1,378 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
-
-package config
-
-import (
-	"bytes"
-	"io"
-	"os"
-	"reflect"
-	"testing"
-
-	"github.com/syncthing/syncthing/protocol"
-)
-
-var node1, node2, node3, node4 protocol.NodeID
-
-func init() {
-	node1, _ = protocol.NodeIDFromString("AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ")
-	node2, _ = protocol.NodeIDFromString("GYRZZQB-IRNPV4Z-T7TC52W-EQYJ3TT-FDQW6MW-DFLMU42-SSSU6EM-FBK2VAY")
-	node3, _ = protocol.NodeIDFromString("LGFPDIT-7SKNNJL-VJZA4FC-7QNCRKA-CE753K7-2BW5QDK-2FOZ7FR-FEP57QJ")
-	node4, _ = protocol.NodeIDFromString("P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2")
-}
-
-func TestDefaultValues(t *testing.T) {
-	expected := OptionsConfiguration{
-		ListenAddress:      []string{"0.0.0.0:22000"},
-		GlobalAnnServer:    "announce.syncthing.net:22026",
-		GlobalAnnEnabled:   true,
-		LocalAnnEnabled:    true,
-		LocalAnnPort:       21025,
-		LocalAnnMCAddr:     "[ff32::5222]:21026",
-		ParallelRequests:   16,
-		MaxSendKbps:        0,
-		ReconnectIntervalS: 60,
-		StartBrowser:       true,
-		UPnPEnabled:        true,
-		UPnPLease:          0,
-		UPnPRenewal:        30,
-	}
-
-	cfg, err := Load(bytes.NewReader(nil), node1)
-	if err != io.EOF {
-		t.Error(err)
-	}
-
-	if !reflect.DeepEqual(cfg.Options, expected) {
-		t.Errorf("Default config differs;\n  E: %#v\n  A: %#v", expected, cfg.Options)
-	}
-}
-
-func TestNodeConfig(t *testing.T) {
-	v1data := []byte(`
-<configuration version="1">
-    <repository id="test" directory="~/Sync">
-        <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ" name="node one">
-            <address>a</address>
-        </node>
-        <node id="P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ" name="node two">
-            <address>b</address>
-        </node>
-        <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ" name="node one">
-            <address>a</address>
-        </node>
-        <node id="P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ" name="node two">
-            <address>b</address>
-        </node>
-    </repository>
-    <options>
-        <readOnly>true</readOnly>
-        <rescanIntervalS>600</rescanIntervalS>
-    </options>
-</configuration>
-`)
-
-	v2data := []byte(`
-<configuration version="2">
-    <repository id="test" directory="~/Sync" ro="true">
-        <node id="P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ"/>
-        <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ"/>
-        <node id="C4YBIESWDUAIGU62GOSRXCRAAJDWVE3TKCPMURZE2LH5QHAF576A"/>
-        <node id="P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ"/>
-        <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ"/>
-        <node id="C4YBIESWDUAIGU62GOSRXCRAAJDWVE3TKCPMURZE2LH5QHAF576A"/>
-    </repository>
-    <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ" name="node one">
-        <address>a</address>
-    </node>
-    <node id="P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ" name="node two">
-        <address>b</address>
-    </node>
-    <options>
-        <rescanIntervalS>600</rescanIntervalS>
-    </options>
-</configuration>
-`)
-
-	v3data := []byte(`
-<configuration version="3">
-    <repository id="test" directory="~/Sync" ro="true" ignorePerms="false">
-        <node id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" compression="false"></node>
-        <node id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" compression="false"></node>
-    </repository>
-    <node id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="true">
-        <address>a</address>
-    </node>
-    <node id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="true">
-        <address>b</address>
-    </node>
-    <options>
-        <rescanIntervalS>600</rescanIntervalS>
-    </options>
-</configuration>`)
-
-	v4data := []byte(`
-<configuration version="4">
-    <repository id="test" directory="~/Sync" ro="true" ignorePerms="false" rescanIntervalS="600">
-        <node id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></node>
-        <node id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></node>
-    </repository>
-    <node id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="true">
-        <address>a</address>
-    </node>
-    <node id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="true">
-        <address>b</address>
-    </node>
-</configuration>`)
-
-	for i, data := range [][]byte{v1data, v2data, v3data, v4data} {
-		cfg, err := Load(bytes.NewReader(data), node1)
-		if err != nil {
-			t.Error(err)
-		}
-
-		expectedRepos := []RepositoryConfiguration{
-			{
-				ID:              "test",
-				Directory:       "~/Sync",
-				Nodes:           []RepositoryNodeConfiguration{{NodeID: node1}, {NodeID: node4}},
-				ReadOnly:        true,
-				RescanIntervalS: 600,
-			},
-		}
-		expectedNodes := []NodeConfiguration{
-			{
-				NodeID:      node1,
-				Name:        "node one",
-				Addresses:   []string{"a"},
-				Compression: true,
-			},
-			{
-				NodeID:      node4,
-				Name:        "node two",
-				Addresses:   []string{"b"},
-				Compression: true,
-			},
-		}
-		expectedNodeIDs := []protocol.NodeID{node1, node4}
-
-		if cfg.Version != 4 {
-			t.Errorf("%d: Incorrect version %d != 3", i, cfg.Version)
-		}
-		if !reflect.DeepEqual(cfg.Repositories, expectedRepos) {
-			t.Errorf("%d: Incorrect Repositories\n  A: %#v\n  E: %#v", i, cfg.Repositories, expectedRepos)
-		}
-		if !reflect.DeepEqual(cfg.Nodes, expectedNodes) {
-			t.Errorf("%d: Incorrect Nodes\n  A: %#v\n  E: %#v", i, cfg.Nodes, expectedNodes)
-		}
-		if !reflect.DeepEqual(cfg.Repositories[0].NodeIDs(), expectedNodeIDs) {
-			t.Errorf("%d: Incorrect NodeIDs\n  A: %#v\n  E: %#v", i, cfg.Repositories[0].NodeIDs(), expectedNodeIDs)
-		}
-
-		if len(cfg.NodeMap()) != len(expectedNodes) {
-			t.Errorf("Unexpected number of NodeMap() entries")
-		}
-		if len(cfg.RepoMap()) != len(expectedRepos) {
-			t.Errorf("Unexpected number of RepoMap() entries")
-		}
-	}
-}
-
-func TestNoListenAddress(t *testing.T) {
-	data := []byte(`<configuration version="1">
-    <options>
-        <listenAddress></listenAddress>
-    </options>
-</configuration>
-`)
-
-	cfg, err := Load(bytes.NewReader(data), node1)
-	if err != nil {
-		t.Error(err)
-	}
-
-	expected := []string{""}
-	if !reflect.DeepEqual(cfg.Options.ListenAddress, expected) {
-		t.Errorf("Unexpected ListenAddress %#v", cfg.Options.ListenAddress)
-	}
-}
-
-func TestOverriddenValues(t *testing.T) {
-	data := []byte(`<configuration version="2">
-    <options>
-       <listenAddress>:23000</listenAddress>
-        <allowDelete>false</allowDelete>
-        <globalAnnounceServer>syncthing.nym.se:22026</globalAnnounceServer>
-        <globalAnnounceEnabled>false</globalAnnounceEnabled>
-        <localAnnounceEnabled>false</localAnnounceEnabled>
-        <localAnnouncePort>42123</localAnnouncePort>
-        <localAnnounceMCAddr>quux:3232</localAnnounceMCAddr>
-        <parallelRequests>32</parallelRequests>
-        <maxSendKbps>1234</maxSendKbps>
-        <reconnectionIntervalS>6000</reconnectionIntervalS>
-        <startBrowser>false</startBrowser>
-        <upnpEnabled>false</upnpEnabled>
-        <upnpLeaseMinutes>60</upnpLeaseMinutes>
-        <upnpRenewalMinutes>15</upnpRenewalMinutes>
-    </options>
-</configuration>
-`)
-
-	expected := OptionsConfiguration{
-		ListenAddress:      []string{":23000"},
-		GlobalAnnServer:    "syncthing.nym.se:22026",
-		GlobalAnnEnabled:   false,
-		LocalAnnEnabled:    false,
-		LocalAnnPort:       42123,
-		LocalAnnMCAddr:     "quux:3232",
-		ParallelRequests:   32,
-		MaxSendKbps:        1234,
-		ReconnectIntervalS: 6000,
-		StartBrowser:       false,
-		UPnPEnabled:        false,
-		UPnPLease:          60,
-		UPnPRenewal:        15,
-	}
-
-	cfg, err := Load(bytes.NewReader(data), node1)
-	if err != nil {
-		t.Error(err)
-	}
-
-	if !reflect.DeepEqual(cfg.Options, expected) {
-		t.Errorf("Overridden config differs;\n  E: %#v\n  A: %#v", expected, cfg.Options)
-	}
-}
-
-func TestNodeAddressesDynamic(t *testing.T) {
-	data := []byte(`
-<configuration version="2">
-    <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ">
-        <address></address>
-    </node>
-    <node id="GYRZZQBIRNPV4T7TC52WEQYJ3TFDQW6MWDFLMU4SSSU6EMFBK2VA">
-    </node>
-    <node id="LGFPDIT7SKNNJVJZA4FC7QNCRKCE753K72BW5QD2FOZ7FRFEP57Q">
-        <address>dynamic</address>
-    </node>
-</configuration>
-`)
-
-	name, _ := os.Hostname()
-	expected := []NodeConfiguration{
-		{
-			NodeID:      node1,
-			Addresses:   []string{"dynamic"},
-			Compression: true,
-		},
-		{
-			NodeID:      node2,
-			Addresses:   []string{"dynamic"},
-			Compression: true,
-		},
-		{
-			NodeID:      node3,
-			Addresses:   []string{"dynamic"},
-			Compression: true,
-		},
-		{
-			NodeID:    node4,
-			Name:      name, // Set when auto created
-			Addresses: []string{"dynamic"},
-		},
-	}
-
-	cfg, err := Load(bytes.NewReader(data), node4)
-	if err != nil {
-		t.Error(err)
-	}
-
-	if !reflect.DeepEqual(cfg.Nodes, expected) {
-		t.Errorf("Nodes differ;\n  E: %#v\n  A: %#v", expected, cfg.Nodes)
-	}
-}
-
-func TestNodeAddressesStatic(t *testing.T) {
-	data := []byte(`
-<configuration version="3">
-    <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ">
-        <address>192.0.2.1</address>
-        <address>192.0.2.2</address>
-    </node>
-    <node id="GYRZZQBIRNPV4T7TC52WEQYJ3TFDQW6MWDFLMU4SSSU6EMFBK2VA">
-        <address>192.0.2.3:6070</address>
-        <address>[2001:db8::42]:4242</address>
-    </node>
-    <node id="LGFPDIT7SKNNJVJZA4FC7QNCRKCE753K72BW5QD2FOZ7FRFEP57Q">
-        <address>[2001:db8::44]:4444</address>
-        <address>192.0.2.4:6090</address>
-    </node>
-</configuration>
-`)
-
-	name, _ := os.Hostname()
-	expected := []NodeConfiguration{
-		{
-			NodeID:    node1,
-			Addresses: []string{"192.0.2.1", "192.0.2.2"},
-		},
-		{
-			NodeID:    node2,
-			Addresses: []string{"192.0.2.3:6070", "[2001:db8::42]:4242"},
-		},
-		{
-			NodeID:    node3,
-			Addresses: []string{"[2001:db8::44]:4444", "192.0.2.4:6090"},
-		},
-		{
-			NodeID:    node4,
-			Name:      name, // Set when auto created
-			Addresses: []string{"dynamic"},
-		},
-	}
-
-	cfg, err := Load(bytes.NewReader(data), node4)
-	if err != nil {
-		t.Error(err)
-	}
-
-	if !reflect.DeepEqual(cfg.Nodes, expected) {
-		t.Errorf("Nodes differ;\n  E: %#v\n  A: %#v", expected, cfg.Nodes)
-	}
-}
-
-func TestVersioningConfig(t *testing.T) {
-	data := []byte(`
-		<configuration version="2">
-			<repository id="test" directory="~/Sync" ro="true">
-				<versioning type="simple">
-					<param key="foo" val="bar"/>
-					<param key="baz" val="quux"/>
-				</versioning>
-			</repository>
-		</configuration>
-		`)
-
-	cfg, err := Load(bytes.NewReader(data), node4)
-	if err != nil {
-		t.Error(err)
-	}
-
-	vc := cfg.Repositories[0].Versioning
-	if vc.Type != "simple" {
-		t.Errorf(`vc.Type %q != "simple"`, vc.Type)
-	}
-	if l := len(vc.Params); l != 2 {
-		t.Errorf("len(vc.Params) %d != 2", l)
-	}
-
-	expected := map[string]string{
-		"foo": "bar",
-		"baz": "quux",
-	}
-	if !reflect.DeepEqual(vc.Params, expected) {
-		t.Errorf("vc.Params differ;\n  E: %#v\n  A: %#v", expected, vc.Params)
-	}
-}

discover/debug.go

@@ -1,17 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
-
-package discover
-
-import (
-	"os"
-	"strings"
-
-	"github.com/syncthing/syncthing/logger"
-)
-
-var (
-	debug = strings.Contains(os.Getenv("STTRACE"), "discover") || os.Getenv("STTRACE") == "all"
-	l     = logger.DefaultLogger
-)

discover/discover_test.go

@@ -1,7 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
-
-package discover_test
-
-// Empty test file to generate 0% coverage rather than no coverage

discover/doc.go

@@ -1,6 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
-
-// Package discover implements the node discovery protocol.
-package discover

discover/packets.go

@@ -1,31 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
-
-package discover
-
-const (
-	AnnouncementMagic = 0x9D79BC39
-	QueryMagic        = 0x2CA856F5
-)
-
-type Query struct {
-	Magic  uint32
-	NodeID []byte // max:32
-}
-
-type Announce struct {
-	Magic uint32
-	This  Node
-	Extra []Node // max:16
-}
-
-type Node struct {
-	ID        []byte    // max:32
-	Addresses []Address // max:16
-}
-
-type Address struct {
-	IP   []byte // max:16
-	Port uint16
-}

events/debug.go

@@ -1,17 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
-
-package events
-
-import (
-	"os"
-	"strings"
-
-	"github.com/syncthing/syncthing/logger"
-)
-
-var (
-	debug = strings.Contains(os.Getenv("STTRACE"), "events") || os.Getenv("STTRACE") == "all"
-	dl    = logger.DefaultLogger
-)

files/debug.go

@@ -1,17 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
-
-package files
-
-import (
-	"os"
-	"strings"
-
-	"github.com/syncthing/syncthing/logger"
-)
-
-var (
-	debug = strings.Contains(os.Getenv("STTRACE"), "files") || os.Getenv("STTRACE") == "all"
-	l     = logger.DefaultLogger
-)

files/filenames_darwin.go

@@ -1,15 +0,0 @@
-// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
-// All rights reserved. Use of this source code is governed by an MIT-style
-// license that can be found in the LICENSE file.
-
-package files
-
-import "code.google.com/p/go.text/unicode/norm"
-
-func normalizedFilename(s string) string {
-	return norm.NFC.String(s)
-}
-
-func nativeFilename(s string) string {
-	return norm.NFD.String(s)
-}