Merge pull request #1861 from calmh/fix-1860

Be more lenient against errors when deleting (fixes #1860)

.gitattributes

@@ -0,0 +1,9 @@
+# Text files use LF line endings in this repository
+*   text=auto
+
+# Except the dependencies, which we leave alone
+Godeps/**   -text=auto
+
+# Diffs on these files are meaningless
+gui.files.go    -diff
+*.svg           -diff

.gitignore

@@ -3,8 +3,6 @@ syncthing.exe
 *.tar.gz
 *.zip
 *.asc
-*.sublime*
-.idea/
 .jshintrc
 coverage.out
 files/pidx
@@ -12,6 +10,7 @@ bin
 perfstats*.csv
 coverage.xml
 !gui/scripts/syncthing
-.DS_Store
 syncthing.md5
 syncthing.exe.md5
+RELEASE
+deb

AUTHORS

@@ -5,36 +5,49 @@ Alexander Graf <register-github@alex-graf.de>
 Andrew Dunham <andrew@du.nham.ca>
 Audrius Butkevicius <audrius.butkevicius@gmail.com>
 Arthur Axel fREW Schmidt <frew@afoolishmanifesto.com> <frioux@gmail.com>
+Ben Curthoys <ben@bencurthoys.com>
 Ben Schulz <ueomkail@gmail.com> <uok@users.noreply.github.com>
 Ben Sidhom <bsidhom@gmail.com>
 Brandon Philips <brandon@ifup.org>
 Brendan Long <self@brendanlong.com>
 Caleb Callaway <enlightened.despot@gmail.com>
+Carsten Hagemann <moter8@gmail.com>
 Cathryne Linenweaver <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
+Chris Howie <me@chrishowie.com>
 Chris Joel <chris@scriptolo.gy>
+Colin Kennedy <moshen.colin@gmail.com>
 Daniel Martí <mvdan@mvdan.cc>
 Dennis Wilson <dw@risu.io>
 Dominik Heidler <dominik@heidler.eu>
+Elias Jarlebring <jarlebring@gmail.com>
 Emil Hessman <emil@hessman.se>
 Federico Castagnini <federico.castagnini@gmail.com>
 Felix Ableitner <me@nutomic.com>
 Felix Unterpaintner <bigbear2nd@gmail.com>
+Francois-Xavier Gsell <fxgsell@gmail.com>
 Gilli Sigurdsson <gilli@vx.is>
 Jakob Borg <jakob@nym.se>
 James Patterson <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
+Jaroslav Malec <dzardacz@gmail.com>
 Jens Diemer <github.com@jensdiemer.de> <git@jensdiemer.de>
 Jochen Voss <voss@seehuhn.de>
+Johan Vromans <jvromans@squirrel.nl>
 Karol Różycki <rozycki.karol@gmail.com>
+Ken'ichi Kamada <kamada@nanohz.org>
 Lode Hoste <zillode@zillode.be>
+Lord Landon Agahnim <lordlandon@gmail.com>
 Marcin Dziadus <dziadus.marcin@gmail.com>
-Marc Laporte <marc@marclaporte.com>
+Marc Laporte <marc@marclaporte.com> <marc@laporte.name>
+Marc Pujol <kilburn@la3.org>
 Michael Jephcote <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
 Michael Tilli <pyfisch@gmail.com>
+Pascal Jungblut <github@pascalj.com> <mail@pascal-jungblut.com>
 Peter Hoeg <peter@speartail.com>
 Philippe Schommers <philippe@schommers.be>
 Phill Luby <phill.luby@newredo.com>
 Piotr Bejda <piotrb10@gmail.com>
 Ryan Sullivan <kayoticsully@gmail.com>
+Sergey Mishin <ralder@yandex.ru>
 Stefan Tatschner <stefan@sevenbyte.org>
 Tim Abell <tim@timwise.co.uk>
 Tobias Nygren <tnn@nygren.pp.se>

CONTRIBUTING.md

@@ -93,15 +93,15 @@ The Syncthing core team currently consists of the following members;
 
 ## Licensing
 
-All contributions are made under the same GPL license as the rest of the
-project, except documentation, user interface text and translation
+All contributions are made under the same MPLv2 license as the rest of
+the project, except documentation, user interface text and translation
 strings which are licensed under the Creative Commons Attribution 4.0
 International License. You retain the copyright to code you have
 written.
 
 When accepting your first contribution, the maintainer of the project
-will ensure that you are added to the AUTHORS file. You are welcome to
-add yourself as a separate commit in your first pull request.
+will ensure that you are added to the AUTHORS file, the NICKS file and
+the list of authors in the about box.
 
 ## Building
 
@@ -137,4 +137,4 @@ Yes please!
 
 ## License
 
-GPLv3
+MPLv2

Godeps/Godeps.json

@@ -11,35 +11,39 @@
 		},
 		{
 			"ImportPath": "github.com/calmh/logger",
-			"Rev": "f50d32b313bec2933a3e1049f7416a29f3413d29"
+			"Rev": "c96f6a1a8c7b6bf2f4860c667867d90174799eb2"
 		},
 		{
 			"ImportPath": "github.com/calmh/luhn",
 			"Rev": "0c8388ff95fa92d4094011e5a04fc99dea3d1632"
 		},
-		{
-			"ImportPath": "github.com/calmh/osext",
-			"Rev": "9bf61584e5f1f172e8766ddc9022d9c401faaa5e"
-		},
 		{
 			"ImportPath": "github.com/calmh/xdr",
-			"Rev": "ff948d7666c5e0fd18d398f6278881724d36a90b"
+			"Rev": "5f7208e86762911861c94f1849eddbfc0a60cbf0"
 		},
 		{
 			"ImportPath": "github.com/juju/ratelimit",
-			"Rev": "f9f36d11773655c0485207f0ad30dc2655f69d56"
+			"Rev": "c5abe513796336ee2869745bff0638508450e9c5"
+		},
+		{
+			"ImportPath": "github.com/kardianos/osext",
+			"Rev": "efacde03154693404c65e7aa7d461ac9014acd0c"
 		},
 		{
 			"ImportPath": "github.com/syncthing/protocol",
-			"Rev": "2e2d479103df8fb721d55d59b0198d6c24f4865b"
+			"Rev": "e7db2648034fb71b051902a02bc25d4468ed492e"
 		},
 		{
 			"ImportPath": "github.com/syndtr/goleveldb/leveldb",
-			"Rev": "63c9e642efad852f49e20a6f90194cae112fd2ac"
+			"Rev": "87e4e645d80ae9c537e8f2dee52b28036a5dd75e"
 		},
 		{
 			"ImportPath": "github.com/syndtr/gosnappy/snappy",
-			"Rev": "ce8acff4829e0c2458a67ead32390ac0a381c862"
+			"Rev": "156a073208e131d7d2e212cb749feae7c339e846"
+		},
+		{
+			"ImportPath": "github.com/thejerf/suture",
+			"Rev": "ff19fb384c3fe30f42717967eaa69da91e5f317c"
 		},
 		{
 			"ImportPath": "github.com/vitrun/qart/coding",
@@ -55,19 +59,19 @@
 		},
 		{
 			"ImportPath": "golang.org/x/crypto/bcrypt",
-			"Rev": "4ed45ec682102c643324fae5dff8dab085b6c300"
+			"Rev": "c57d4a71915a248dbad846d60825145062b4c18e"
 		},
 		{
 			"ImportPath": "golang.org/x/crypto/blowfish",
-			"Rev": "4ed45ec682102c643324fae5dff8dab085b6c300"
+			"Rev": "c57d4a71915a248dbad846d60825145062b4c18e"
 		},
 		{
 			"ImportPath": "golang.org/x/text/transform",
-			"Rev": "c980adc4a823548817b9c47d38c6ca6b7d7d8b6a"
+			"Rev": "2076e9cab4147459c82bc81169e46c139d358547"
 		},
 		{
 			"ImportPath": "golang.org/x/text/unicode/norm",
-			"Rev": "c980adc4a823548817b9c47d38c6ca6b7d7d8b6a"
+			"Rev": "2076e9cab4147459c82bc81169e46c139d358547"
 		}
 	]
 }

Godeps/_workspace/src/github.com/calmh/logger/logger.go

@@ -6,6 +6,7 @@ package logger
 
 import (
 	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
 	"strings"
@@ -16,6 +17,7 @@ type LogLevel int
 
 const (
 	LevelDebug LogLevel = iota
+	LevelVerbose
 	LevelInfo
 	LevelOK
 	LevelWarn
@@ -36,6 +38,13 @@ type Logger struct {
 var DefaultLogger = New()
 
 func New() *Logger {
+	if os.Getenv("LOGGER_DISCARD") != "" {
+		// Hack to completely disable logging, for example when running benchmarks.
+		return &Logger{
+			logger: log.New(ioutil.Discard, "", 0),
+		}
+	}
+
 	return &Logger{
 		logger: log.New(os.Stdout, "", log.Ltime),
 	}
@@ -83,6 +92,24 @@ func (l *Logger) Debugf(format string, vals ...interface{}) {
 	l.callHandlers(LevelDebug, s)
 }
 
+// Infoln logs a line with a VERBOSE prefix.
+func (l *Logger) Verboseln(vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintln(vals...)
+	l.logger.Output(2, "VERBOSE: "+s)
+	l.callHandlers(LevelVerbose, s)
+}
+
+// Infof logs a formatted line with a VERBOSE prefix.
+func (l *Logger) Verbosef(format string, vals ...interface{}) {
+	l.mut.Lock()
+	defer l.mut.Unlock()
+	s := fmt.Sprintf(format, vals...)
+	l.logger.Output(2, "VERBOSE: "+s)
+	l.callHandlers(LevelVerbose, s)
+}
+
 // Infoln logs a line with an INFO prefix.
 func (l *Logger) Infoln(vals ...interface{}) {
 	l.mut.Lock()

Godeps/_workspace/src/github.com/calmh/osext/LICENSE

@@ -1,20 +0,0 @@
-Copyright (c) 2012 Daniel Theophanes
-
-This software is provided 'as-is', without any express or implied
-warranty. In no event will the authors be held liable for any damages
-arising from the use of this software.
-
-Permission is granted to anyone to use this software for any purpose,
-including commercial applications, and to alter it and redistribute it
-freely, subject to the following restrictions:
-
-   1. The origin of this software must not be misrepresented; you must not
-   claim that you wrote the original software. If you use this software
-   in a product, an acknowledgment in the product documentation would be
-   appreciated but is not required.
-
-   2. Altered source versions must be plainly marked as such, and must not be
-   misrepresented as being the original software.
-
-   3. This notice may not be removed or altered from any source
-   distribution.

Godeps/_workspace/src/github.com/calmh/osext/osext_test.go

@@ -1,79 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build darwin linux freebsd netbsd windows
-
-package osext
-
-import (
-	"fmt"
-	"os"
-	oexec "os/exec"
-	"path/filepath"
-	"runtime"
-	"testing"
-)
-
-const execPath_EnvVar = "OSTEST_OUTPUT_EXECPATH"
-
-func TestExecPath(t *testing.T) {
-	ep, err := Executable()
-	if err != nil {
-		t.Fatalf("ExecPath failed: %v", err)
-	}
-	// we want fn to be of the form "dir/prog"
-	dir := filepath.Dir(filepath.Dir(ep))
-	fn, err := filepath.Rel(dir, ep)
-	if err != nil {
-		t.Fatalf("filepath.Rel: %v", err)
-	}
-	cmd := &oexec.Cmd{}
-	// make child start with a relative program path
-	cmd.Dir = dir
-	cmd.Path = fn
-	// forge argv[0] for child, so that we can verify we could correctly
-	// get real path of the executable without influenced by argv[0].
-	cmd.Args = []string{"-", "-test.run=XXXX"}
-	cmd.Env = []string{fmt.Sprintf("%s=1", execPath_EnvVar)}
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		t.Fatalf("exec(self) failed: %v", err)
-	}
-	outs := string(out)
-	if !filepath.IsAbs(outs) {
-		t.Fatalf("Child returned %q, want an absolute path", out)
-	}
-	if !sameFile(outs, ep) {
-		t.Fatalf("Child returned %q, not the same file as %q", out, ep)
-	}
-}
-
-func sameFile(fn1, fn2 string) bool {
-	fi1, err := os.Stat(fn1)
-	if err != nil {
-		return false
-	}
-	fi2, err := os.Stat(fn2)
-	if err != nil {
-		return false
-	}
-	return os.SameFile(fi1, fi2)
-}
-
-func init() {
-	if e := os.Getenv(execPath_EnvVar); e != "" {
-		// first chdir to another path
-		dir := "/"
-		if runtime.GOOS == "windows" {
-			dir = filepath.VolumeName(".")
-		}
-		os.Chdir(dir)
-		if ep, err := Executable(); err != nil {
-			fmt.Fprint(os.Stderr, "ERROR: ", err)
-		} else {
-			fmt.Fprint(os.Stderr, ep)
-		}
-		os.Exit(0)
-	}
-}

Godeps/_workspace/src/github.com/calmh/xdr/bench_test.go

@@ -67,7 +67,7 @@ func BenchmarkThisEncode(b *testing.B) {
 func BenchmarkThisEncoder(b *testing.B) {
 	w := xdr.NewWriter(ioutil.Discard)
 	for i := 0; i < b.N; i++ {
-		_, err := s.encodeXDR(w)
+		_, err := s.EncodeXDRInto(w)
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -108,7 +108,7 @@ func BenchmarkThisDecoder(b *testing.B) {
 	r := xdr.NewReader(rr)
 	var t XDRBenchStruct
 	for i := 0; i < b.N; i++ {
-		err := t.decodeXDR(r)
+		err := t.DecodeXDRFrom(r)
 		if err != nil {
 			b.Fatal(err)
 		}

Godeps/_workspace/src/github.com/calmh/xdr/bench_xdr_test.go

@@ -26,7 +26,9 @@ XDRBenchStruct Structure:
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |            0x0000             |              I3               |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             uint8                             |
+/                                                               /
+\                        uint8 Structure                        \
+/                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |                         Length of Bs0                         |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -69,7 +71,7 @@ struct XDRBenchStruct {
 
 func (o XDRBenchStruct) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o XDRBenchStruct) MarshalXDR() ([]byte, error) {
@@ -87,11 +89,11 @@ func (o XDRBenchStruct) MustMarshalXDR() []byte {
 func (o XDRBenchStruct) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o XDRBenchStruct) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o XDRBenchStruct) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	xw.WriteUint64(o.I1)
 	xw.WriteUint32(o.I2)
 	xw.WriteUint16(o.I3)
@@ -111,16 +113,16 @@ func (o XDRBenchStruct) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *XDRBenchStruct) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *XDRBenchStruct) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *XDRBenchStruct) decodeXDR(xr *xdr.Reader) error {
+func (o *XDRBenchStruct) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.I1 = xr.ReadUint64()
 	o.I2 = xr.ReadUint32()
 	o.I3 = xr.ReadUint16()
@@ -155,7 +157,7 @@ struct repeatReader {
 
 func (o repeatReader) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o repeatReader) MarshalXDR() ([]byte, error) {
@@ -173,27 +175,27 @@ func (o repeatReader) MustMarshalXDR() []byte {
 func (o repeatReader) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o repeatReader) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o repeatReader) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	xw.WriteBytes(o.data)
 	return xw.Tot(), xw.Error()
 }
 
 func (o *repeatReader) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *repeatReader) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *repeatReader) decodeXDR(xr *xdr.Reader) error {
+func (o *repeatReader) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.data = xr.ReadBytes()
 	return xr.Error()
 }

Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go

@@ -52,7 +52,7 @@ import (
 var encodeTpl = template.Must(template.New("encoder").Parse(`
 func (o {{.TypeName}}) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }//+n
 
 func (o {{.TypeName}}) MarshalXDR() ([]byte, error) {
@@ -70,11 +70,11 @@ func (o {{.TypeName}}) MustMarshalXDR() []byte {
 func (o {{.TypeName}}) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }//+n
 
-func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o {{.TypeName}}) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	{{range $fieldInfo := .Fields}}
 		{{if not $fieldInfo.IsSlice}}
 			{{if ne $fieldInfo.Convert ""}}
@@ -87,7 +87,7 @@ func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
 				{{end}}
 				xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}})
 			{{else}}
-				_, err := o.{{$fieldInfo.Name}}.encodeXDR(xw)
+				_, err := o.{{$fieldInfo.Name}}.EncodeXDRInto(xw)
 				if err != nil {
 					return xw.Tot(), err
 				}
@@ -105,7 +105,7 @@ func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
 			{{else if $fieldInfo.IsBasic}}
 				xw.Write{{$fieldInfo.Encoder}}(o.{{$fieldInfo.Name}}[i])
 			{{else}}
-				_, err := o.{{$fieldInfo.Name}}[i].encodeXDR(xw)
+				_, err := o.{{$fieldInfo.Name}}[i].EncodeXDRInto(xw)
 				if err != nil {
 					return xw.Tot(), err
 				}
@@ -118,16 +118,16 @@ func (o {{.TypeName}}) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *{{.TypeName}}) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }//+n
 
 func (o *{{.TypeName}}) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }//+n
 
-func (o *{{.TypeName}}) decodeXDR(xr *xdr.Reader) error {
+func (o *{{.TypeName}}) DecodeXDRFrom(xr *xdr.Reader) error {
 	{{range $fieldInfo := .Fields}}
 		{{if not $fieldInfo.IsSlice}}
 			{{if ne $fieldInfo.Convert ""}}
@@ -139,10 +139,13 @@ func (o *{{.TypeName}}) decodeXDR(xr *xdr.Reader) error {
 					o.{{$fieldInfo.Name}} = xr.Read{{$fieldInfo.Encoder}}()
 				{{end}}
 			{{else}}
-				(&o.{{$fieldInfo.Name}}).decodeXDR(xr)
+				(&o.{{$fieldInfo.Name}}).DecodeXDRFrom(xr)
 			{{end}}
 		{{else}}
 			_{{$fieldInfo.Name}}Size := int(xr.ReadUint32())
+			if _{{$fieldInfo.Name}}Size < 0 {
+				return xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", _{{$fieldInfo.Name}}Size, {{$fieldInfo.Max}})
+			}
 			{{if ge $fieldInfo.Max 1}}
 				if _{{$fieldInfo.Name}}Size > {{$fieldInfo.Max}} {
 					return xdr.ElementSizeExceeded("{{$fieldInfo.Name}}", _{{$fieldInfo.Name}}Size, {{$fieldInfo.Max}})
@@ -155,7 +158,7 @@ func (o *{{.TypeName}}) decodeXDR(xr *xdr.Reader) error {
 				{{else if $fieldInfo.IsBasic}}
 					o.{{$fieldInfo.Name}}[i] = xr.Read{{$fieldInfo.Encoder}}()
 				{{else}}
-					(&o.{{$fieldInfo.Name}}[i]).decodeXDR(xr)
+					(&o.{{$fieldInfo.Name}}[i]).DecodeXDRFrom(xr)
 				{{end}}
 			}
 		{{end}}
@@ -257,12 +260,18 @@ func handleStruct(t *ast.StructType) []fieldInfo {
 			} else {
 				f = fieldInfo{
 					Name:      fn,
-					IsBasic:   false,
 					IsSlice:   true,
 					FieldType: tn,
 					Max:       max,
 				}
 			}
+
+		case *ast.SelectorExpr:
+			f = fieldInfo{
+				Name:      fn,
+				FieldType: ft.Sel.Name,
+				Max:       max,
+			}
 		}
 
 		fs = append(fs, f)
@@ -310,10 +319,9 @@ func generateDiagram(output io.Writer, s structInfo) {
 
 	for _, f := range fs {
 		tn := f.FieldType
-		sl := f.IsSlice
 		name := uncamelize(f.Name)
 
-		if sl {
+		if f.IsSlice {
 			fmt.Fprintf(output, "| %s |\n", center("Number of "+name, 61))
 			fmt.Fprintln(output, line)
 		}
@@ -340,13 +348,16 @@ func generateDiagram(output io.Writer, s structInfo) {
 			fmt.Fprintf(output, "/ %61s /\n", "")
 			fmt.Fprintln(output, line)
 		default:
-			if sl {
+			if f.IsSlice {
 				tn = "Zero or more " + tn + " Structures"
 				fmt.Fprintf(output, "/ %s /\n", center("", 61))
 				fmt.Fprintf(output, "\\ %s \\\n", center(tn, 61))
 				fmt.Fprintf(output, "/ %s /\n", center("", 61))
 			} else {
-				fmt.Fprintf(output, "| %s |\n", center(tn, 61))
+				tn = tn + " Structure"
+				fmt.Fprintf(output, "/ %s /\n", center("", 61))
+				fmt.Fprintf(output, "\\ %s \\\n", center(tn, 61))
+				fmt.Fprintf(output, "/ %s /\n", center("", 61))
 			}
 			fmt.Fprintln(output, line)
 		}

Godeps/_workspace/src/github.com/calmh/xdr/encdec_test.go

@@ -32,11 +32,11 @@ type TestStruct struct {
 
 type Opaque [32]byte
 
-func (u *Opaque) encodeXDR(w *xdr.Writer) (int, error) {
+func (u *Opaque) EncodeXDRInto(w *xdr.Writer) (int, error) {
 	return w.WriteRaw(u[:])
 }
 
-func (u *Opaque) decodeXDR(r *xdr.Reader) (int, error) {
+func (u *Opaque) DecodeXDRFrom(r *xdr.Reader) (int, error) {
 	return r.ReadRaw(u[:])
 }
 

Godeps/_workspace/src/github.com/calmh/xdr/encdec_xdr_test.go

@@ -18,17 +18,23 @@ TestStruct Structure:
  0                   1                   2                   3
  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                              int                              |
+/                                                               /
+\                         int Structure                         \
+/                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             int8                              |
+/                                                               /
+\                        int8 Structure                         \
+/                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             uint8                             |
+/                                                               /
+\                        uint8 Structure                        \
+/                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             int16                             |
+|            0x0000             |              I16              |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |            0x0000             |             UI16              |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             int32                             |
+|                              I32                              |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |                             UI32                              |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -52,7 +58,9 @@ TestStruct Structure:
 \                      S (variable length)                      \
 /                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                            Opaque                             |
+/                                                               /
+\                       Opaque Structure                        \
+/                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |                         Number of SS                          |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
@@ -68,9 +76,9 @@ struct TestStruct {
 	int I;
 	int8 I8;
 	uint8 UI8;
-	int16 I16;
+	int I16;
 	unsigned int UI16;
-	int32 I32;
+	int I32;
 	unsigned int UI32;
 	hyper I64;
 	unsigned hyper UI64;
@@ -84,7 +92,7 @@ struct TestStruct {
 
 func (o TestStruct) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o TestStruct) MarshalXDR() ([]byte, error) {
@@ -102,11 +110,11 @@ func (o TestStruct) MustMarshalXDR() []byte {
 func (o TestStruct) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o TestStruct) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o TestStruct) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	xw.WriteUint64(uint64(o.I))
 	xw.WriteUint8(uint8(o.I8))
 	xw.WriteUint8(o.UI8)
@@ -124,7 +132,7 @@ func (o TestStruct) encodeXDR(xw *xdr.Writer) (int, error) {
 		return xw.Tot(), xdr.ElementSizeExceeded("S", l, 1024)
 	}
 	xw.WriteString(o.S)
-	_, err := o.C.encodeXDR(xw)
+	_, err := o.C.EncodeXDRInto(xw)
 	if err != nil {
 		return xw.Tot(), err
 	}
@@ -140,16 +148,16 @@ func (o TestStruct) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *TestStruct) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *TestStruct) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *TestStruct) decodeXDR(xr *xdr.Reader) error {
+func (o *TestStruct) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.I = int(xr.ReadUint64())
 	o.I8 = int8(xr.ReadUint8())
 	o.UI8 = xr.ReadUint8()
@@ -161,8 +169,11 @@ func (o *TestStruct) decodeXDR(xr *xdr.Reader) error {
 	o.UI64 = xr.ReadUint64()
 	o.BS = xr.ReadBytesMax(1024)
 	o.S = xr.ReadStringMax(1024)
-	(&o.C).decodeXDR(xr)
+	(&o.C).DecodeXDRFrom(xr)
 	_SSSize := int(xr.ReadUint32())
+	if _SSSize < 0 {
+		return xdr.ElementSizeExceeded("SS", _SSSize, 1024)
+	}
 	if _SSSize > 1024 {
 		return xdr.ElementSizeExceeded("SS", _SSSize, 1024)
 	}

Godeps/_workspace/src/github.com/calmh/xdr/reader.go

@@ -68,7 +68,8 @@ func (r *Reader) ReadBytesMaxInto(max int, dst []byte) []byte {
 	if r.err != nil {
 		return nil
 	}
-	if max > 0 && l > max {
+	if l < 0 || max > 0 && l > max {
+		// l may be negative on 32 bit builds
 		r.err = ElementSizeExceeded("bytes field", l, max)
 		return nil
 	}

Godeps/_workspace/src/github.com/juju/ratelimit/LICENSE

@@ -1,3 +1,6 @@
+This package contains an efficient token-bucket-based rate limiter.
+Copyright (C) 2015 Canonical Ltd.
+
 This software is licensed under the LGPLv3, included below.
 
 As a special exception to the GNU Lesser General Public License version 3

Godeps/_workspace/src/github.com/kardianos/osext/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2012 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Godeps/_workspace/src/github.com/kardianos/osext/README.md

@@ -0,0 +1,14 @@
+### Extensions to the "os" package.
+
+## Find the current Executable and ExecutableFolder.
+
+There is sometimes utility in finding the current executable file
+that is running. This can be used for upgrading the current executable
+or finding resources located relative to the executable file.
+
+Multi-platform and supports:
+ * Linux
+ * OS X
+ * Windows
+ * Plan 9
+ * BSDs.

Godeps/_workspace/src/github.com/kardianos/osext/osext.go

@@ -25,8 +25,3 @@ func ExecutableFolder() (string, error) {
 	folder, _ := filepath.Split(p)
 	return folder, nil
 }
-
-// Depricated. Same as Executable().
-func GetExePath() (exePath string, err error) {
-	return Executable()
-}

Godeps/_workspace/src/github.com/kardianos/osext/osext_plan9.go

@@ -5,16 +5,16 @@
 package osext
 
 import (
-    "syscall"
-    "os"
-    "strconv"
+	"os"
+	"strconv"
+	"syscall"
 )
 
 func executable() (string, error) {
-    f, err := os.Open("/proc/" + strconv.Itoa(os.Getpid()) + "/text")
-    if err != nil {
-        return "", err
-    }
-    defer f.Close()
-    return syscall.Fd2path(int(f.Fd()))
+	f, err := os.Open("/proc/" + strconv.Itoa(os.Getpid()) + "/text")
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	return syscall.Fd2path(int(f.Fd()))
 }

Godeps/_workspace/src/github.com/kardianos/osext/osext_procfs.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build linux netbsd openbsd solaris
+// +build linux netbsd openbsd solaris dragonfly
 
 package osext
 
@@ -11,15 +11,21 @@ import (
 	"fmt"
 	"os"
 	"runtime"
+	"strings"
 )
 
 func executable() (string, error) {
 	switch runtime.GOOS {
 	case "linux":
-		return os.Readlink("/proc/self/exe")
+		const deletedSuffix = " (deleted)"
+		execpath, err := os.Readlink("/proc/self/exe")
+		if err != nil {
+			return execpath, err
+		}
+		return strings.TrimSuffix(execpath, deletedSuffix), nil
 	case "netbsd":
 		return os.Readlink("/proc/curproc/exe")
-	case "openbsd":
+	case "openbsd", "dragonfly":
 		return os.Readlink("/proc/curproc/file")
 	case "solaris":
 		return os.Readlink(fmt.Sprintf("/proc/%d/path/a.out", os.Getpid()))

Godeps/_workspace/src/github.com/kardianos/osext/osext_test.go

@@ -0,0 +1,180 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin linux freebsd netbsd windows
+
+package osext
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"testing"
+)
+
+const (
+	executableEnvVar = "OSTEST_OUTPUT_EXECUTABLE"
+
+	executableEnvValueMatch  = "match"
+	executableEnvValueDelete = "delete"
+)
+
+func TestExecutableMatch(t *testing.T) {
+	ep, err := Executable()
+	if err != nil {
+		t.Fatalf("Executable failed: %v", err)
+	}
+
+	// fullpath to be of the form "dir/prog".
+	dir := filepath.Dir(filepath.Dir(ep))
+	fullpath, err := filepath.Rel(dir, ep)
+	if err != nil {
+		t.Fatalf("filepath.Rel: %v", err)
+	}
+	// Make child start with a relative program path.
+	// Alter argv[0] for child to verify getting real path without argv[0].
+	cmd := &exec.Cmd{
+		Dir:  dir,
+		Path: fullpath,
+		Env:  []string{fmt.Sprintf("%s=%s", executableEnvVar, executableEnvValueMatch)},
+	}
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("exec(self) failed: %v", err)
+	}
+	outs := string(out)
+	if !filepath.IsAbs(outs) {
+		t.Fatalf("Child returned %q, want an absolute path", out)
+	}
+	if !sameFile(outs, ep) {
+		t.Fatalf("Child returned %q, not the same file as %q", out, ep)
+	}
+}
+
+func TestExecutableDelete(t *testing.T) {
+	if runtime.GOOS != "linux" {
+		t.Skip()
+	}
+	fpath, err := Executable()
+	if err != nil {
+		t.Fatalf("Executable failed: %v", err)
+	}
+
+	r, w := io.Pipe()
+	stderrBuff := &bytes.Buffer{}
+	stdoutBuff := &bytes.Buffer{}
+	cmd := &exec.Cmd{
+		Path:   fpath,
+		Env:    []string{fmt.Sprintf("%s=%s", executableEnvVar, executableEnvValueDelete)},
+		Stdin:  r,
+		Stderr: stderrBuff,
+		Stdout: stdoutBuff,
+	}
+	err = cmd.Start()
+	if err != nil {
+		t.Fatalf("exec(self) start failed: %v", err)
+	}
+
+	tempPath := fpath + "_copy"
+	_ = os.Remove(tempPath)
+
+	err = copyFile(tempPath, fpath)
+	if err != nil {
+		t.Fatalf("copy file failed: %v", err)
+	}
+	err = os.Remove(fpath)
+	if err != nil {
+		t.Fatalf("remove running test file failed: %v", err)
+	}
+	err = os.Rename(tempPath, fpath)
+	if err != nil {
+		t.Fatalf("rename copy to previous name failed: %v", err)
+	}
+
+	w.Write([]byte{0})
+	w.Close()
+
+	err = cmd.Wait()
+	if err != nil {
+		t.Fatalf("exec wait failed: %v", err)
+	}
+
+	childPath := stderrBuff.String()
+	if !filepath.IsAbs(childPath) {
+		t.Fatalf("Child returned %q, want an absolute path", childPath)
+	}
+	if !sameFile(childPath, fpath) {
+		t.Fatalf("Child returned %q, not the same file as %q", childPath, fpath)
+	}
+}
+
+func sameFile(fn1, fn2 string) bool {
+	fi1, err := os.Stat(fn1)
+	if err != nil {
+		return false
+	}
+	fi2, err := os.Stat(fn2)
+	if err != nil {
+		return false
+	}
+	return os.SameFile(fi1, fi2)
+}
+func copyFile(dest, src string) error {
+	df, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+	defer df.Close()
+
+	sf, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer sf.Close()
+
+	_, err = io.Copy(df, sf)
+	return err
+}
+
+func TestMain(m *testing.M) {
+	env := os.Getenv(executableEnvVar)
+	switch env {
+	case "":
+		os.Exit(m.Run())
+	case executableEnvValueMatch:
+		// First chdir to another path.
+		dir := "/"
+		if runtime.GOOS == "windows" {
+			dir = filepath.VolumeName(".")
+		}
+		os.Chdir(dir)
+		if ep, err := Executable(); err != nil {
+			fmt.Fprint(os.Stderr, "ERROR: ", err)
+		} else {
+			fmt.Fprint(os.Stderr, ep)
+		}
+	case executableEnvValueDelete:
+		bb := make([]byte, 1)
+		var err error
+		n, err := os.Stdin.Read(bb)
+		if err != nil {
+			fmt.Fprint(os.Stderr, "ERROR: ", err)
+			os.Exit(2)
+		}
+		if n != 1 {
+			fmt.Fprint(os.Stderr, "ERROR: n != 1, n == ", n)
+			os.Exit(2)
+		}
+		if ep, err := Executable(); err != nil {
+			fmt.Fprint(os.Stderr, "ERROR: ", err)
+		} else {
+			fmt.Fprint(os.Stderr, ep)
+		}
+	}
+	os.Exit(0)
+}

Godeps/_workspace/src/github.com/syncthing/protocol/common_test.go

@@ -13,6 +13,9 @@ type TestModel struct {
 	name     string
 	offset   int64
 	size     int
+	hash     []byte
+	flags    uint32
+	options  []Option
 	closedCh chan bool
 }
 
@@ -22,17 +25,20 @@ func newTestModel() *TestModel {
 	}
 }
 
-func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
+func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
 }
 
-func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
+func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
 }
 
-func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int) ([]byte, error) {
+func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
 	t.folder = folder
 	t.name = name
 	t.offset = offset
 	t.size = size
+	t.hash = hash
+	t.flags = flags
+	t.options = options
 	return t.data, nil
 }
 

Godeps/_workspace/src/github.com/syncthing/protocol/compression.go

@@ -0,0 +1,53 @@
+// Copyright (C) 2015 The Protocol Authors.
+
+package protocol
+
+import "fmt"
+
+type Compression int
+
+const (
+	CompressMetadata Compression = iota // zero value is the default, default should be "metadata"
+	CompressNever
+	CompressAlways
+
+	compressionThreshold = 128 // don't bother compressing messages smaller than this many bytes
+)
+
+var compressionMarshal = map[Compression]string{
+	CompressNever:    "never",
+	CompressMetadata: "metadata",
+	CompressAlways:   "always",
+}
+
+var compressionUnmarshal = map[string]Compression{
+	// Legacy
+	"false": CompressNever,
+	"true":  CompressMetadata,
+
+	// Current
+	"never":    CompressNever,
+	"metadata": CompressMetadata,
+	"always":   CompressAlways,
+}
+
+func (c Compression) String() string {
+	s, ok := compressionMarshal[c]
+	if !ok {
+		return fmt.Sprintf("unknown:%d", c)
+	}
+	return s
+}
+
+func (c Compression) GoString() string {
+	return fmt.Sprintf("%q", c.String())
+}
+
+func (c Compression) MarshalText() ([]byte, error) {
+	return []byte(compressionMarshal[c]), nil
+}
+
+func (c *Compression) UnmarshalText(bs []byte) error {
+	*c = compressionUnmarshal[string(bs)]
+	return nil
+}

Godeps/_workspace/src/github.com/syncthing/protocol/compression_test.go

@@ -0,0 +1,49 @@
+// Copyright (C) 2015 The Protocol Authors.
+
+package protocol
+
+import "testing"
+
+func TestCompressionMarshal(t *testing.T) {
+	uTestcases := []struct {
+		s string
+		c Compression
+	}{
+		{"true", CompressMetadata},
+		{"false", CompressNever},
+		{"never", CompressNever},
+		{"metadata", CompressMetadata},
+		{"always", CompressAlways},
+		{"whatever", CompressMetadata},
+	}
+
+	mTestcases := []struct {
+		s string
+		c Compression
+	}{
+		{"never", CompressNever},
+		{"metadata", CompressMetadata},
+		{"always", CompressAlways},
+	}
+
+	var c Compression
+	for _, tc := range uTestcases {
+		err := c.UnmarshalText([]byte(tc.s))
+		if err != nil {
+			t.Error(err)
+		}
+		if c != tc.c {
+			t.Errorf("%s unmarshalled to %d, not %d", tc.s, c, tc.c)
+		}
+	}
+
+	for _, tc := range mTestcases {
+		bs, err := tc.c.MarshalText()
+		if err != nil {
+			t.Error(err)
+		}
+		if s := string(bs); s != tc.s {
+			t.Errorf("%d marshalled to %q, not %q", tc.c, s, tc.s)
+		}
+	}
+}

Godeps/_workspace/src/github.com/syncthing/protocol/deviceid.go

@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"crypto/sha256"
 	"encoding/base32"
+	"encoding/binary"
 	"errors"
 	"fmt"
 	"regexp"
@@ -67,6 +68,11 @@ func (n DeviceID) Equals(other DeviceID) bool {
 	return bytes.Compare(n[:], other[:]) == 0
 }
 
+// Short returns an integer representing bits 0-63 of the device ID.
+func (n DeviceID) Short() uint64 {
+	return binary.BigEndian.Uint64(n[:])
+}
+
 func (n *DeviceID) MarshalText() ([]byte, error) {
 	return []byte(n.String()), nil
 }

Godeps/_workspace/src/github.com/syncthing/protocol/errors.go

@@ -0,0 +1,51 @@
+// Copyright (C) 2014 The Protocol Authors.
+
+package protocol
+
+import (
+	"errors"
+)
+
+const (
+	ecNoError int32 = iota
+	ecGeneric
+	ecNoSuchFile
+	ecInvalid
+)
+
+var (
+	ErrNoError    error = nil
+	ErrGeneric          = errors.New("generic error")
+	ErrNoSuchFile       = errors.New("no such file")
+	ErrInvalid          = errors.New("file is invalid")
+)
+
+var lookupError = map[int32]error{
+	ecNoError:    ErrNoError,
+	ecGeneric:    ErrGeneric,
+	ecNoSuchFile: ErrNoSuchFile,
+	ecInvalid:    ErrInvalid,
+}
+
+var lookupCode = map[error]int32{
+	ErrNoError:    ecNoError,
+	ErrGeneric:    ecGeneric,
+	ErrNoSuchFile: ecNoSuchFile,
+	ErrInvalid:    ecInvalid,
+}
+
+func codeToError(errcode int32) error {
+	err, ok := lookupError[errcode]
+	if !ok {
+		return ErrGeneric
+	}
+	return err
+}
+
+func errorToCode(err error) int32 {
+	code, ok := lookupCode[err]
+	if !ok {
+		return ecGeneric
+	}
+	return code
+}

Godeps/_workspace/src/github.com/syncthing/protocol/message.go

@@ -1,5 +1,6 @@
 // Copyright (C) 2014 The Protocol Authors.
 
+//go:generate -command genxdr go run ../syncthing/Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go
 //go:generate genxdr -o message_xdr.go message.go
 
 package protocol
@@ -7,7 +8,7 @@ package protocol
 import "fmt"
 
 type IndexMessage struct {
-	Folder  string // max:64
+	Folder  string
 	Files   []FileInfo
 	Flags   uint32
 	Options []Option // max:64
@@ -17,13 +18,13 @@ type FileInfo struct {
 	Name         string // max:8192
 	Flags        uint32
 	Modified     int64
-	Version      int64
+	Version      Vector
 	LocalVersion int64
 	Blocks       []BlockInfo
 }
 
 func (f FileInfo) String() string {
-	return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, Blocks:%v}",
+	return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%v, Size:%d, Blocks:%v}",
 		f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.Blocks)
 }
 
@@ -78,14 +79,14 @@ type RequestMessage struct {
 }
 
 type ResponseMessage struct {
-	Data  []byte
-	Error int32
+	Data []byte
+	Code int32
 }
 
 type ClusterConfigMessage struct {
-	ClientName    string   // max:64
-	ClientVersion string   // max:64
-	Folders       []Folder // max:64
+	ClientName    string // max:64
+	ClientVersion string // max:64
+	Folders       []Folder
 	Options       []Option // max:64
 }
 
@@ -101,12 +102,15 @@ func (o *ClusterConfigMessage) GetOption(key string) string {
 type Folder struct {
 	ID      string // max:64
 	Devices []Device
+	Flags   uint32
+	Options []Option // max:64
 }
 
 type Device struct {
 	ID              []byte // max:32
-	Flags           uint32
 	MaxLocalVersion int64
+	Flags           uint32
+	Options         []Option // max:64
 }
 
 type Option struct {

Godeps/_workspace/src/github.com/syncthing/protocol/message_xdr.go

@@ -41,7 +41,7 @@ IndexMessage Structure:
 
 
 struct IndexMessage {
-	string Folder<64>;
+	string Folder<>;
 	FileInfo Files<>;
 	unsigned int Flags;
 	Option Options<64>;
@@ -51,7 +51,7 @@ struct IndexMessage {
 
 func (o IndexMessage) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o IndexMessage) MarshalXDR() ([]byte, error) {
@@ -69,18 +69,15 @@ func (o IndexMessage) MustMarshalXDR() []byte {
 func (o IndexMessage) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) {
-	if l := len(o.Folder); l > 64 {
-		return xw.Tot(), xdr.ElementSizeExceeded("Folder", l, 64)
-	}
+func (o IndexMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	xw.WriteString(o.Folder)
 	xw.WriteUint32(uint32(len(o.Files)))
 	for i := range o.Files {
-		_, err := o.Files[i].encodeXDR(xw)
+		_, err := o.Files[i].EncodeXDRInto(xw)
 		if err != nil {
 			return xw.Tot(), err
 		}
@@ -91,7 +88,7 @@ func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) {
 	}
 	xw.WriteUint32(uint32(len(o.Options)))
 	for i := range o.Options {
-		_, err := o.Options[i].encodeXDR(xw)
+		_, err := o.Options[i].EncodeXDRInto(xw)
 		if err != nil {
 			return xw.Tot(), err
 		}
@@ -101,30 +98,36 @@ func (o IndexMessage) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *IndexMessage) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *IndexMessage) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *IndexMessage) decodeXDR(xr *xdr.Reader) error {
-	o.Folder = xr.ReadStringMax(64)
+func (o *IndexMessage) DecodeXDRFrom(xr *xdr.Reader) error {
+	o.Folder = xr.ReadString()
 	_FilesSize := int(xr.ReadUint32())
+	if _FilesSize < 0 {
+		return xdr.ElementSizeExceeded("Files", _FilesSize, 0)
+	}
 	o.Files = make([]FileInfo, _FilesSize)
 	for i := range o.Files {
-		(&o.Files[i]).decodeXDR(xr)
+		(&o.Files[i]).DecodeXDRFrom(xr)
 	}
 	o.Flags = xr.ReadUint32()
 	_OptionsSize := int(xr.ReadUint32())
+	if _OptionsSize < 0 {
+		return xdr.ElementSizeExceeded("Options", _OptionsSize, 64)
+	}
 	if _OptionsSize > 64 {
 		return xdr.ElementSizeExceeded("Options", _OptionsSize, 64)
 	}
 	o.Options = make([]Option, _OptionsSize)
 	for i := range o.Options {
-		(&o.Options[i]).decodeXDR(xr)
+		(&o.Options[i]).DecodeXDRFrom(xr)
 	}
 	return xr.Error()
 }
@@ -148,9 +151,9 @@ FileInfo Structure:
 +                      Modified (64 bits)                       +
 |                                                               |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                                                               |
-+                       Version (64 bits)                       +
-|                                                               |
+/                                                               /
+\                       Vector Structure                        \
+/                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |                                                               |
 +                    Local Version (64 bits)                    +
@@ -168,7 +171,7 @@ struct FileInfo {
 	string Name<8192>;
 	unsigned int Flags;
 	hyper Modified;
-	hyper Version;
+	Vector Version;
 	hyper LocalVersion;
 	BlockInfo Blocks<>;
 }
@@ -177,7 +180,7 @@ struct FileInfo {
 
 func (o FileInfo) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o FileInfo) MarshalXDR() ([]byte, error) {
@@ -195,22 +198,25 @@ func (o FileInfo) MustMarshalXDR() []byte {
 func (o FileInfo) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o FileInfo) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o FileInfo) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	if l := len(o.Name); l > 8192 {
 		return xw.Tot(), xdr.ElementSizeExceeded("Name", l, 8192)
 	}
 	xw.WriteString(o.Name)
 	xw.WriteUint32(o.Flags)
 	xw.WriteUint64(uint64(o.Modified))
-	xw.WriteUint64(uint64(o.Version))
+	_, err := o.Version.EncodeXDRInto(xw)
+	if err != nil {
+		return xw.Tot(), err
+	}
 	xw.WriteUint64(uint64(o.LocalVersion))
 	xw.WriteUint32(uint32(len(o.Blocks)))
 	for i := range o.Blocks {
-		_, err := o.Blocks[i].encodeXDR(xw)
+		_, err := o.Blocks[i].EncodeXDRInto(xw)
 		if err != nil {
 			return xw.Tot(), err
 		}
@@ -220,25 +226,28 @@ func (o FileInfo) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *FileInfo) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *FileInfo) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *FileInfo) decodeXDR(xr *xdr.Reader) error {
+func (o *FileInfo) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.Name = xr.ReadStringMax(8192)
 	o.Flags = xr.ReadUint32()
 	o.Modified = int64(xr.ReadUint64())
-	o.Version = int64(xr.ReadUint64())
+	(&o.Version).DecodeXDRFrom(xr)
 	o.LocalVersion = int64(xr.ReadUint64())
 	_BlocksSize := int(xr.ReadUint32())
+	if _BlocksSize < 0 {
+		return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 0)
+	}
 	o.Blocks = make([]BlockInfo, _BlocksSize)
 	for i := range o.Blocks {
-		(&o.Blocks[i]).decodeXDR(xr)
+		(&o.Blocks[i]).DecodeXDRFrom(xr)
 	}
 	return xr.Error()
 }
@@ -269,7 +278,7 @@ struct BlockInfo {
 
 func (o BlockInfo) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o BlockInfo) MarshalXDR() ([]byte, error) {
@@ -287,11 +296,11 @@ func (o BlockInfo) MustMarshalXDR() []byte {
 func (o BlockInfo) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o BlockInfo) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o BlockInfo) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	xw.WriteUint32(uint32(o.Size))
 	if l := len(o.Hash); l > 64 {
 		return xw.Tot(), xdr.ElementSizeExceeded("Hash", l, 64)
@@ -302,16 +311,16 @@ func (o BlockInfo) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *BlockInfo) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *BlockInfo) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *BlockInfo) decodeXDR(xr *xdr.Reader) error {
+func (o *BlockInfo) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.Size = int32(xr.ReadUint32())
 	o.Hash = xr.ReadBytesMax(64)
 	return xr.Error()
@@ -372,7 +381,7 @@ struct RequestMessage {
 
 func (o RequestMessage) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o RequestMessage) MarshalXDR() ([]byte, error) {
@@ -390,11 +399,11 @@ func (o RequestMessage) MustMarshalXDR() []byte {
 func (o RequestMessage) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o RequestMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	if l := len(o.Folder); l > 64 {
 		return xw.Tot(), xdr.ElementSizeExceeded("Folder", l, 64)
 	}
@@ -415,7 +424,7 @@ func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) {
 	}
 	xw.WriteUint32(uint32(len(o.Options)))
 	for i := range o.Options {
-		_, err := o.Options[i].encodeXDR(xw)
+		_, err := o.Options[i].EncodeXDRInto(xw)
 		if err != nil {
 			return xw.Tot(), err
 		}
@@ -425,16 +434,16 @@ func (o RequestMessage) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *RequestMessage) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *RequestMessage) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *RequestMessage) decodeXDR(xr *xdr.Reader) error {
+func (o *RequestMessage) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.Folder = xr.ReadStringMax(64)
 	o.Name = xr.ReadStringMax(8192)
 	o.Offset = int64(xr.ReadUint64())
@@ -442,12 +451,15 @@ func (o *RequestMessage) decodeXDR(xr *xdr.Reader) error {
 	o.Hash = xr.ReadBytesMax(64)
 	o.Flags = xr.ReadUint32()
 	_OptionsSize := int(xr.ReadUint32())
+	if _OptionsSize < 0 {
+		return xdr.ElementSizeExceeded("Options", _OptionsSize, 64)
+	}
 	if _OptionsSize > 64 {
 		return xdr.ElementSizeExceeded("Options", _OptionsSize, 64)
 	}
 	o.Options = make([]Option, _OptionsSize)
 	for i := range o.Options {
-		(&o.Options[i]).decodeXDR(xr)
+		(&o.Options[i]).DecodeXDRFrom(xr)
 	}
 	return xr.Error()
 }
@@ -465,20 +477,20 @@ ResponseMessage Structure:
 \                    Data (variable length)                     \
 /                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             Error                             |
+|                             Code                              |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 
 struct ResponseMessage {
 	opaque Data<>;
-	int Error;
+	int Code;
 }
 
 */
 
 func (o ResponseMessage) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o ResponseMessage) MarshalXDR() ([]byte, error) {
@@ -496,30 +508,30 @@ func (o ResponseMessage) MustMarshalXDR() []byte {
 func (o ResponseMessage) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o ResponseMessage) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o ResponseMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	xw.WriteBytes(o.Data)
-	xw.WriteUint32(uint32(o.Error))
+	xw.WriteUint32(uint32(o.Code))
 	return xw.Tot(), xw.Error()
 }
 
 func (o *ResponseMessage) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *ResponseMessage) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *ResponseMessage) decodeXDR(xr *xdr.Reader) error {
+func (o *ResponseMessage) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.Data = xr.ReadBytes()
-	o.Error = int32(xr.ReadUint32())
+	o.Code = int32(xr.ReadUint32())
 	return xr.Error()
 }
 
@@ -559,7 +571,7 @@ ClusterConfigMessage Structure:
 struct ClusterConfigMessage {
 	string ClientName<64>;
 	string ClientVersion<64>;
-	Folder Folders<64>;
+	Folder Folders<>;
 	Option Options<64>;
 }
 
@@ -567,7 +579,7 @@ struct ClusterConfigMessage {
 
 func (o ClusterConfigMessage) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o ClusterConfigMessage) MarshalXDR() ([]byte, error) {
@@ -585,11 +597,11 @@ func (o ClusterConfigMessage) MustMarshalXDR() []byte {
 func (o ClusterConfigMessage) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o ClusterConfigMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	if l := len(o.ClientName); l > 64 {
 		return xw.Tot(), xdr.ElementSizeExceeded("ClientName", l, 64)
 	}
@@ -598,12 +610,9 @@ func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) {
 		return xw.Tot(), xdr.ElementSizeExceeded("ClientVersion", l, 64)
 	}
 	xw.WriteString(o.ClientVersion)
-	if l := len(o.Folders); l > 64 {
-		return xw.Tot(), xdr.ElementSizeExceeded("Folders", l, 64)
-	}
 	xw.WriteUint32(uint32(len(o.Folders)))
 	for i := range o.Folders {
-		_, err := o.Folders[i].encodeXDR(xw)
+		_, err := o.Folders[i].EncodeXDRInto(xw)
 		if err != nil {
 			return xw.Tot(), err
 		}
@@ -613,7 +622,7 @@ func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) {
 	}
 	xw.WriteUint32(uint32(len(o.Options)))
 	for i := range o.Options {
-		_, err := o.Options[i].encodeXDR(xw)
+		_, err := o.Options[i].EncodeXDRInto(xw)
 		if err != nil {
 			return xw.Tot(), err
 		}
@@ -623,33 +632,36 @@ func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *ClusterConfigMessage) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *ClusterConfigMessage) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error {
+func (o *ClusterConfigMessage) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.ClientName = xr.ReadStringMax(64)
 	o.ClientVersion = xr.ReadStringMax(64)
 	_FoldersSize := int(xr.ReadUint32())
-	if _FoldersSize > 64 {
-		return xdr.ElementSizeExceeded("Folders", _FoldersSize, 64)
+	if _FoldersSize < 0 {
+		return xdr.ElementSizeExceeded("Folders", _FoldersSize, 0)
 	}
 	o.Folders = make([]Folder, _FoldersSize)
 	for i := range o.Folders {
-		(&o.Folders[i]).decodeXDR(xr)
+		(&o.Folders[i]).DecodeXDRFrom(xr)
 	}
 	_OptionsSize := int(xr.ReadUint32())
+	if _OptionsSize < 0 {
+		return xdr.ElementSizeExceeded("Options", _OptionsSize, 64)
+	}
 	if _OptionsSize > 64 {
 		return xdr.ElementSizeExceeded("Options", _OptionsSize, 64)
 	}
 	o.Options = make([]Option, _OptionsSize)
 	for i := range o.Options {
-		(&o.Options[i]).decodeXDR(xr)
+		(&o.Options[i]).DecodeXDRFrom(xr)
 	}
 	return xr.Error()
 }
@@ -673,18 +685,28 @@ Folder Structure:
 \                Zero or more Device Structures                 \
 /                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             Flags                             |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                       Number of Options                       |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                Zero or more Option Structures                 \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 
 struct Folder {
 	string ID<64>;
 	Device Devices<>;
+	unsigned int Flags;
+	Option Options<64>;
 }
 
 */
 
 func (o Folder) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o Folder) MarshalXDR() ([]byte, error) {
@@ -702,18 +724,29 @@ func (o Folder) MustMarshalXDR() []byte {
 func (o Folder) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o Folder) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o Folder) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	if l := len(o.ID); l > 64 {
 		return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 64)
 	}
 	xw.WriteString(o.ID)
 	xw.WriteUint32(uint32(len(o.Devices)))
 	for i := range o.Devices {
-		_, err := o.Devices[i].encodeXDR(xw)
+		_, err := o.Devices[i].EncodeXDRInto(xw)
+		if err != nil {
+			return xw.Tot(), err
+		}
+	}
+	xw.WriteUint32(o.Flags)
+	if l := len(o.Options); l > 64 {
+		return xw.Tot(), xdr.ElementSizeExceeded("Options", l, 64)
+	}
+	xw.WriteUint32(uint32(len(o.Options)))
+	for i := range o.Options {
+		_, err := o.Options[i].EncodeXDRInto(xw)
 		if err != nil {
 			return xw.Tot(), err
 		}
@@ -723,21 +756,36 @@ func (o Folder) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *Folder) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *Folder) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *Folder) decodeXDR(xr *xdr.Reader) error {
+func (o *Folder) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.ID = xr.ReadStringMax(64)
 	_DevicesSize := int(xr.ReadUint32())
+	if _DevicesSize < 0 {
+		return xdr.ElementSizeExceeded("Devices", _DevicesSize, 0)
+	}
 	o.Devices = make([]Device, _DevicesSize)
 	for i := range o.Devices {
-		(&o.Devices[i]).decodeXDR(xr)
+		(&o.Devices[i]).DecodeXDRFrom(xr)
+	}
+	o.Flags = xr.ReadUint32()
+	_OptionsSize := int(xr.ReadUint32())
+	if _OptionsSize < 0 {
+		return xdr.ElementSizeExceeded("Options", _OptionsSize, 64)
+	}
+	if _OptionsSize > 64 {
+		return xdr.ElementSizeExceeded("Options", _OptionsSize, 64)
+	}
+	o.Options = make([]Option, _OptionsSize)
+	for i := range o.Options {
+		(&o.Options[i]).DecodeXDRFrom(xr)
 	}
 	return xr.Error()
 }
@@ -755,25 +803,32 @@ Device Structure:
 \                     ID (variable length)                      \
 /                                                               /
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             Flags                             |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 |                                                               |
 +                  Max Local Version (64 bits)                  +
 |                                                               |
 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                             Flags                             |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                       Number of Options                       |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+/                                                               /
+\                Zero or more Option Structures                 \
+/                                                               /
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 
 struct Device {
 	opaque ID<32>;
-	unsigned int Flags;
 	hyper MaxLocalVersion;
+	unsigned int Flags;
+	Option Options<64>;
 }
 
 */
 
 func (o Device) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o Device) MarshalXDR() ([]byte, error) {
@@ -791,35 +846,56 @@ func (o Device) MustMarshalXDR() []byte {
 func (o Device) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o Device) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o Device) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	if l := len(o.ID); l > 32 {
 		return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 32)
 	}
 	xw.WriteBytes(o.ID)
-	xw.WriteUint32(o.Flags)
 	xw.WriteUint64(uint64(o.MaxLocalVersion))
+	xw.WriteUint32(o.Flags)
+	if l := len(o.Options); l > 64 {
+		return xw.Tot(), xdr.ElementSizeExceeded("Options", l, 64)
+	}
+	xw.WriteUint32(uint32(len(o.Options)))
+	for i := range o.Options {
+		_, err := o.Options[i].EncodeXDRInto(xw)
+		if err != nil {
+			return xw.Tot(), err
+		}
+	}
 	return xw.Tot(), xw.Error()
 }
 
 func (o *Device) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *Device) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *Device) decodeXDR(xr *xdr.Reader) error {
+func (o *Device) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.ID = xr.ReadBytesMax(32)
-	o.Flags = xr.ReadUint32()
 	o.MaxLocalVersion = int64(xr.ReadUint64())
+	o.Flags = xr.ReadUint32()
+	_OptionsSize := int(xr.ReadUint32())
+	if _OptionsSize < 0 {
+		return xdr.ElementSizeExceeded("Options", _OptionsSize, 64)
+	}
+	if _OptionsSize > 64 {
+		return xdr.ElementSizeExceeded("Options", _OptionsSize, 64)
+	}
+	o.Options = make([]Option, _OptionsSize)
+	for i := range o.Options {
+		(&o.Options[i]).DecodeXDRFrom(xr)
+	}
 	return xr.Error()
 }
 
@@ -853,7 +929,7 @@ struct Option {
 
 func (o Option) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o Option) MarshalXDR() ([]byte, error) {
@@ -871,11 +947,11 @@ func (o Option) MustMarshalXDR() []byte {
 func (o Option) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o Option) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o Option) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	if l := len(o.Key); l > 64 {
 		return xw.Tot(), xdr.ElementSizeExceeded("Key", l, 64)
 	}
@@ -889,16 +965,16 @@ func (o Option) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *Option) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *Option) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *Option) decodeXDR(xr *xdr.Reader) error {
+func (o *Option) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.Key = xr.ReadStringMax(64)
 	o.Value = xr.ReadStringMax(1024)
 	return xr.Error()
@@ -930,7 +1006,7 @@ struct CloseMessage {
 
 func (o CloseMessage) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o CloseMessage) MarshalXDR() ([]byte, error) {
@@ -948,11 +1024,11 @@ func (o CloseMessage) MustMarshalXDR() []byte {
 func (o CloseMessage) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o CloseMessage) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o CloseMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	if l := len(o.Reason); l > 1024 {
 		return xw.Tot(), xdr.ElementSizeExceeded("Reason", l, 1024)
 	}
@@ -963,16 +1039,16 @@ func (o CloseMessage) encodeXDR(xw *xdr.Writer) (int, error) {
 
 func (o *CloseMessage) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *CloseMessage) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *CloseMessage) decodeXDR(xr *xdr.Reader) error {
+func (o *CloseMessage) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.Reason = xr.ReadStringMax(1024)
 	o.Code = int32(xr.ReadUint32())
 	return xr.Error()
@@ -994,7 +1070,7 @@ struct EmptyMessage {
 
 func (o EmptyMessage) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
-	return o.encodeXDR(xw)
+	return o.EncodeXDRInto(xw)
 }
 
 func (o EmptyMessage) MarshalXDR() ([]byte, error) {
@@ -1012,25 +1088,25 @@ func (o EmptyMessage) MustMarshalXDR() []byte {
 func (o EmptyMessage) AppendXDR(bs []byte) ([]byte, error) {
 	var aw = xdr.AppendWriter(bs)
 	var xw = xdr.NewWriter(&aw)
-	_, err := o.encodeXDR(xw)
+	_, err := o.EncodeXDRInto(xw)
 	return []byte(aw), err
 }
 
-func (o EmptyMessage) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o EmptyMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	return xw.Tot(), xw.Error()
 }
 
 func (o *EmptyMessage) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
 func (o *EmptyMessage) UnmarshalXDR(bs []byte) error {
 	var br = bytes.NewReader(bs)
 	var xr = xdr.NewReader(br)
-	return o.decodeXDR(xr)
+	return o.DecodeXDRFrom(xr)
 }
 
-func (o *EmptyMessage) decodeXDR(xr *xdr.Reader) error {
+func (o *EmptyMessage) DecodeXDRFrom(xr *xdr.Reader) error {
 	return xr.Error()
 }

Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_darwin.go

@@ -12,23 +12,23 @@ type nativeModel struct {
 	next Model
 }
 
-func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
+func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
 	for i := range files {
 		files[i].Name = norm.NFD.String(files[i].Name)
 	}
-	m.next.Index(deviceID, folder, files)
+	m.next.Index(deviceID, folder, files, flags, options)
 }
 
-func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
+func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
 	for i := range files {
 		files[i].Name = norm.NFD.String(files[i].Name)
 	}
-	m.next.IndexUpdate(deviceID, folder, files)
+	m.next.IndexUpdate(deviceID, folder, files, flags, options)
 }
 
-func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) {
+func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
 	name = norm.NFD.String(name)
-	return m.next.Request(deviceID, folder, name, offset, size)
+	return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options)
 }
 
 func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {

Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_unix.go

@@ -10,16 +10,16 @@ type nativeModel struct {
 	next Model
 }
 
-func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
-	m.next.Index(deviceID, folder, files)
+func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+	m.next.Index(deviceID, folder, files, flags, options)
 }
 
-func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
-	m.next.IndexUpdate(deviceID, folder, files)
+func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+	m.next.IndexUpdate(deviceID, folder, files, flags, options)
 }
 
-func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) {
-	return m.next.Request(deviceID, folder, name, offset, size)
+func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
+	return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options)
 }
 
 func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {

Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_windows.go

@@ -24,23 +24,30 @@ type nativeModel struct {
 	next Model
 }
 
-func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
-	for i, f := range files {
-		if strings.ContainsAny(f.Name, disallowedCharacters) {
-			if f.IsDeleted() {
-				// Don't complain if the file is marked as deleted, since it
-				// can't possibly exist here anyway.
-				continue
-			}
-			files[i].Flags |= FlagInvalid
-			l.Warnf("File name %q contains invalid characters; marked as invalid.", f.Name)
-		}
-		files[i].Name = filepath.FromSlash(f.Name)
-	}
-	m.next.Index(deviceID, folder, files)
+func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+	fixupFiles(files)
+	m.next.Index(deviceID, folder, files, flags, options)
 }
 
-func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
+func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+	fixupFiles(files)
+	m.next.IndexUpdate(deviceID, folder, files, flags, options)
+}
+
+func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
+	name = filepath.FromSlash(name)
+	return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options)
+}
+
+func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
+	m.next.ClusterConfig(deviceID, config)
+}
+
+func (m nativeModel) Close(deviceID DeviceID, err error) {
+	m.next.Close(deviceID, err)
+}
+
+func fixupFiles(files []FileInfo) {
 	for i, f := range files {
 		if strings.ContainsAny(f.Name, disallowedCharacters) {
 			if f.IsDeleted() {
@@ -53,18 +60,4 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI
 		}
 		files[i].Name = filepath.FromSlash(files[i].Name)
 	}
-	m.next.IndexUpdate(deviceID, folder, files)
-}
-
-func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error) {
-	name = filepath.FromSlash(name)
-	return m.next.Request(deviceID, folder, name, offset, size)
-}
-
-func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
-	m.next.ClusterConfig(deviceID, config)
-}
-
-func (m nativeModel) Close(deviceID DeviceID, err error) {
-	m.next.Close(deviceID, err)
 }

Godeps/_workspace/src/github.com/syncthing/protocol/protocol.go

@@ -35,6 +35,7 @@ const (
 	stateIdxRcvd
 )
 
+// FileInfo flags
 const (
 	FlagDeleted              uint32 = 1 << 12
 	FlagInvalid                     = 1 << 13
@@ -48,6 +49,17 @@ const (
 	SymlinkTypeMask = FlagDirectory | FlagSymlinkMissingTarget
 )
 
+// IndexMessage message flags (for IndexUpdate)
+const (
+	FlagIndexTemporary uint32 = 1 << iota
+)
+
+// Request message flags
+const (
+	FlagRequestTemporary uint32 = 1 << iota
+)
+
+// ClusterConfigMessage.Folders.Devices flags
 const (
 	FlagShareTrusted  uint32 = 1 << 0
 	FlagShareReadOnly        = 1 << 1
@@ -66,11 +78,11 @@ type pongMessage struct{ EmptyMessage }
 
 type Model interface {
 	// An index was received from the peer device
-	Index(deviceID DeviceID, folder string, files []FileInfo)
+	Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option)
 	// An index update was received from the peer device
-	IndexUpdate(deviceID DeviceID, folder string, files []FileInfo)
+	IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option)
 	// A request was made by the peer device
-	Request(deviceID DeviceID, folder string, name string, offset int64, size int) ([]byte, error)
+	Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error)
 	// A cluster configuration message was received
 	ClusterConfig(deviceID DeviceID, config ClusterConfigMessage)
 	// The peer device closed the connection
@@ -80,9 +92,9 @@ type Model interface {
 type Connection interface {
 	ID() DeviceID
 	Name() string
-	Index(folder string, files []FileInfo) error
-	IndexUpdate(folder string, files []FileInfo) error
-	Request(folder string, name string, offset int64, size int) ([]byte, error)
+	Index(folder string, files []FileInfo, flags uint32, options []Option) error
+	IndexUpdate(folder string, files []FileInfo, flags uint32, options []Option) error
+	Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error)
 	ClusterConfig(config ClusterConfigMessage)
 	Statistics() Statistics
 }
@@ -106,7 +118,7 @@ type rawConnection struct {
 	closed chan struct{}
 	once   sync.Once
 
-	compressionThreshold int // compress messages larger than this many bytes
+	compression Compression
 
 	rdbuf0 []byte // used & reused by readMessage
 	rdbuf1 []byte // used & reused by readMessage
@@ -135,25 +147,21 @@ const (
 	pingIdleTime = 60 * time.Second
 )
 
-func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiver Model, name string, compress bool) Connection {
+func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiver Model, name string, compress Compression) Connection {
 	cr := &countingReader{Reader: reader}
 	cw := &countingWriter{Writer: writer}
 
-	compThres := 1<<31 - 1 // compression disabled
-	if compress {
-		compThres = 128 // compress messages that are 128 bytes long or larger
-	}
 	c := rawConnection{
-		id:                   deviceID,
-		name:                 name,
-		receiver:             nativeModel{receiver},
-		state:                stateInitial,
-		cr:                   cr,
-		cw:                   cw,
-		outbox:               make(chan hdrMsg),
-		nextID:               make(chan int),
-		closed:               make(chan struct{}),
-		compressionThreshold: compThres,
+		id:          deviceID,
+		name:        name,
+		receiver:    nativeModel{receiver},
+		state:       stateInitial,
+		cr:          cr,
+		cw:          cw,
+		outbox:      make(chan hdrMsg),
+		nextID:      make(chan int),
+		closed:      make(chan struct{}),
+		compression: compress,
 	}
 
 	go c.readerLoop()
@@ -173,7 +181,7 @@ func (c *rawConnection) Name() string {
 }
 
 // Index writes the list of file information to the connected peer device
-func (c *rawConnection) Index(folder string, idx []FileInfo) error {
+func (c *rawConnection) Index(folder string, idx []FileInfo, flags uint32, options []Option) error {
 	select {
 	case <-c.closed:
 		return ErrClosed
@@ -181,15 +189,17 @@ func (c *rawConnection) Index(folder string, idx []FileInfo) error {
 	}
 	c.idxMut.Lock()
 	c.send(-1, messageTypeIndex, IndexMessage{
-		Folder: folder,
-		Files:  idx,
+		Folder:  folder,
+		Files:   idx,
+		Flags:   flags,
+		Options: options,
 	})
 	c.idxMut.Unlock()
 	return nil
 }
 
 // IndexUpdate writes the list of file information to the connected peer device as an update
-func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo) error {
+func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo, flags uint32, options []Option) error {
 	select {
 	case <-c.closed:
 		return ErrClosed
@@ -197,15 +207,17 @@ func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo) error {
 	}
 	c.idxMut.Lock()
 	c.send(-1, messageTypeIndexUpdate, IndexMessage{
-		Folder: folder,
-		Files:  idx,
+		Folder:  folder,
+		Files:   idx,
+		Flags:   flags,
+		Options: options,
 	})
 	c.idxMut.Unlock()
 	return nil
 }
 
 // Request returns the bytes for the specified block after fetching them from the connected peer.
-func (c *rawConnection) Request(folder string, name string, offset int64, size int) ([]byte, error) {
+func (c *rawConnection) Request(folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
 	var id int
 	select {
 	case id = <-c.nextID:
@@ -222,10 +234,13 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i
 	c.awaitingMut.Unlock()
 
 	ok := c.send(id, messageTypeRequest, RequestMessage{
-		Folder: folder,
-		Name:   name,
-		Offset: offset,
-		Size:   int32(size),
+		Folder:  folder,
+		Name:    name,
+		Offset:  offset,
+		Size:    int32(size),
+		Hash:    hash,
+		Flags:   flags,
+		Options: options,
 	})
 	if !ok {
 		return nil, ErrClosed
@@ -284,11 +299,6 @@ func (c *rawConnection) readerLoop() (err error) {
 
 		switch msg := msg.(type) {
 		case IndexMessage:
-			if msg.Flags != 0 {
-				// We don't currently support or expect any flags.
-				return fmt.Errorf("protocol error: unknown flags 0x%x in Index(Update) message", msg.Flags)
-			}
-
 			switch hdr.msgType {
 			case messageTypeIndex:
 				if c.state < stateCCRcvd {
@@ -305,10 +315,6 @@ func (c *rawConnection) readerLoop() (err error) {
 			}
 
 		case RequestMessage:
-			if msg.Flags != 0 {
-				// We don't currently support or expect any flags.
-				return fmt.Errorf("protocol error: unknown flags 0x%x in Request message", msg.Flags)
-			}
 			if c.state < stateIdxRcvd {
 				return fmt.Errorf("protocol error: request message in state %d", c.state)
 			}
@@ -464,16 +470,16 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) {
 
 func (c *rawConnection) handleIndex(im IndexMessage) {
 	if debug {
-		l.Debugf("Index(%v, %v, %d files)", c.id, im.Folder, len(im.Files))
+		l.Debugf("Index(%v, %v, %d file, flags %x, opts: %s)", c.id, im.Folder, len(im.Files), im.Flags, im.Options)
 	}
-	c.receiver.Index(c.id, im.Folder, filterIndexMessageFiles(im.Files))
+	c.receiver.Index(c.id, im.Folder, filterIndexMessageFiles(im.Files), im.Flags, im.Options)
 }
 
 func (c *rawConnection) handleIndexUpdate(im IndexMessage) {
 	if debug {
-		l.Debugf("queueing IndexUpdate(%v, %v, %d files)", c.id, im.Folder, len(im.Files))
+		l.Debugf("queueing IndexUpdate(%v, %v, %d files, flags %x, opts: %s)", c.id, im.Folder, len(im.Files), im.Flags, im.Options)
 	}
-	c.receiver.IndexUpdate(c.id, im.Folder, filterIndexMessageFiles(im.Files))
+	c.receiver.IndexUpdate(c.id, im.Folder, filterIndexMessageFiles(im.Files), im.Flags, im.Options)
 }
 
 func filterIndexMessageFiles(fs []FileInfo) []FileInfo {
@@ -503,10 +509,11 @@ func filterIndexMessageFiles(fs []FileInfo) []FileInfo {
 }
 
 func (c *rawConnection) handleRequest(msgID int, req RequestMessage) {
-	data, _ := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size))
+	data, err := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size), req.Hash, req.Flags, req.Options)
 
 	c.send(msgID, messageTypeResponse, ResponseMessage{
 		Data: data,
+		Code: errorToCode(err),
 	})
 }
 
@@ -514,7 +521,7 @@ func (c *rawConnection) handleResponse(msgID int, resp ResponseMessage) {
 	c.awaitingMut.Lock()
 	if rc := c.awaiting[msgID]; rc != nil {
 		c.awaiting[msgID] = nil
-		rc <- asyncResult{resp.Data, nil}
+		rc <- asyncResult{resp.Data, codeToError(resp.Code)}
 		close(rc)
 	}
 	c.awaitingMut.Unlock()
@@ -571,7 +578,15 @@ func (c *rawConnection) writerLoop() {
 					return
 				}
 
-				if len(uncBuf) >= c.compressionThreshold {
+				compress := false
+				switch c.compression {
+				case CompressAlways:
+					compress = true
+				case CompressMetadata:
+					compress = hm.hdr.msgType != messageTypeResponse
+				}
+
+				if compress && len(uncBuf) >= compressionThreshold {
 					// Use compression for large messages
 					hm.hdr.compression = true
 

Godeps/_workspace/src/github.com/syncthing/protocol/protocol_test.go

@@ -67,8 +67,8 @@ func TestPing(t *testing.T) {
 	ar, aw := io.Pipe()
 	br, bw := io.Pipe()
 
-	c0 := NewConnection(c0ID, ar, bw, nil, "name", true).(wireFormatConnection).next.(*rawConnection)
-	c1 := NewConnection(c1ID, br, aw, nil, "name", true).(wireFormatConnection).next.(*rawConnection)
+	c0 := NewConnection(c0ID, ar, bw, nil, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
+	c1 := NewConnection(c1ID, br, aw, nil, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
 
 	if ok := c0.ping(); !ok {
 		t.Error("c0 ping failed")
@@ -91,8 +91,8 @@ func TestPingErr(t *testing.T) {
 			eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e}
 			ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e}
 
-			c0 := NewConnection(c0ID, ar, ebw, m0, "name", true).(wireFormatConnection).next.(*rawConnection)
-			NewConnection(c1ID, br, eaw, m1, "name", true)
+			c0 := NewConnection(c0ID, ar, ebw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
+			NewConnection(c1ID, br, eaw, m1, "name", CompressAlways)
 
 			res := c0.ping()
 			if (i < 8 || j < 8) && res {
@@ -167,8 +167,8 @@ func TestVersionErr(t *testing.T) {
 	ar, aw := io.Pipe()
 	br, bw := io.Pipe()
 
-	c0 := NewConnection(c0ID, ar, bw, m0, "name", true).(wireFormatConnection).next.(*rawConnection)
-	NewConnection(c1ID, br, aw, m1, "name", true)
+	c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
+	NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
 
 	w := xdr.NewWriter(c0.cw)
 	w.WriteUint32(encodeHeader(header{
@@ -190,8 +190,8 @@ func TestTypeErr(t *testing.T) {
 	ar, aw := io.Pipe()
 	br, bw := io.Pipe()
 
-	c0 := NewConnection(c0ID, ar, bw, m0, "name", true).(wireFormatConnection).next.(*rawConnection)
-	NewConnection(c1ID, br, aw, m1, "name", true)
+	c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
+	NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
 
 	w := xdr.NewWriter(c0.cw)
 	w.WriteUint32(encodeHeader(header{
@@ -213,8 +213,8 @@ func TestClose(t *testing.T) {
 	ar, aw := io.Pipe()
 	br, bw := io.Pipe()
 
-	c0 := NewConnection(c0ID, ar, bw, m0, "name", true).(wireFormatConnection).next.(*rawConnection)
-	NewConnection(c1ID, br, aw, m1, "name", true)
+	c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
+	NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
 
 	c0.close(nil)
 
@@ -229,10 +229,10 @@ func TestClose(t *testing.T) {
 		t.Error("Ping should not return true")
 	}
 
-	c0.Index("default", nil)
-	c0.Index("default", nil)
+	c0.Index("default", nil, 0, nil)
+	c0.Index("default", nil, 0, nil)
 
-	if _, err := c0.Request("default", "foo", 0, 0); err == nil {
+	if _, err := c0.Request("default", "foo", 0, 0, nil, 0, nil); err == nil {
 		t.Error("Request should return an error")
 	}
 }

Godeps/_workspace/src/github.com/syncthing/protocol/vector.go

@@ -0,0 +1,115 @@
+// Copyright (C) 2015 The Protocol Authors.
+
+package protocol
+
+// The Vector type represents a version vector. The zero value is a usable
+// version vector. The vector has slice semantics and some operations on it
+// are "append-like" in that they may return the same vector modified, or a
+// new allocated Vector with the modified contents.
+type Vector []Counter
+
+// Counter represents a single counter in the version vector.
+type Counter struct {
+	ID    uint64
+	Value uint64
+}
+
+// Update returns a Vector with the index for the specific ID incremented by
+// one. If it is possible, the vector v is updated and returned. If it is not,
+// a copy will be created, updated and returned.
+func (v Vector) Update(ID uint64) Vector {
+	for i := range v {
+		if v[i].ID == ID {
+			// Update an existing index
+			v[i].Value++
+			return v
+		} else if v[i].ID > ID {
+			// Insert a new index
+			nv := make(Vector, len(v)+1)
+			copy(nv, v[:i])
+			nv[i].ID = ID
+			nv[i].Value = 1
+			copy(nv[i+1:], v[i:])
+			return nv
+		}
+	}
+	// Append a new new index
+	return append(v, Counter{ID, 1})
+}
+
+// Merge returns the vector containing the maximum indexes from a and b. If it
+// is possible, the vector a is updated and returned. If it is not, a copy
+// will be created, updated and returned.
+func (a Vector) Merge(b Vector) Vector {
+	var ai, bi int
+	for bi < len(b) {
+		if ai == len(a) {
+			// We've reach the end of a, all that remains are appends
+			return append(a, b[bi:]...)
+		}
+
+		if a[ai].ID > b[bi].ID {
+			// The index from b should be inserted here
+			n := make(Vector, len(a)+1)
+			copy(n, a[:ai])
+			n[ai] = b[bi]
+			copy(n[ai+1:], a[ai:])
+			a = n
+		}
+
+		if a[ai].ID == b[bi].ID {
+			if v := b[bi].Value; v > a[ai].Value {
+				a[ai].Value = v
+			}
+		}
+
+		if bi < len(b) && a[ai].ID == b[bi].ID {
+			bi++
+		}
+		ai++
+	}
+
+	return a
+}
+
+// Copy returns an identical vector that is not shared with v.
+func (v Vector) Copy() Vector {
+	nv := make(Vector, len(v))
+	copy(nv, v)
+	return nv
+}
+
+// Equal returns true when the two vectors are equivalent.
+func (a Vector) Equal(b Vector) bool {
+	return a.Compare(b) == Equal
+}
+
+// LesserEqual returns true when the two vectors are equivalent or a is Lesser
+// than b.
+func (a Vector) LesserEqual(b Vector) bool {
+	comp := a.Compare(b)
+	return comp == Lesser || comp == Equal
+}
+
+// LesserEqual returns true when the two vectors are equivalent or a is Greater
+// than b.
+func (a Vector) GreaterEqual(b Vector) bool {
+	comp := a.Compare(b)
+	return comp == Greater || comp == Equal
+}
+
+// Concurrent returns true when the two vectors are concrurrent.
+func (a Vector) Concurrent(b Vector) bool {
+	comp := a.Compare(b)
+	return comp == ConcurrentGreater || comp == ConcurrentLesser
+}
+
+// Counter returns the current value of the given counter ID.
+func (v Vector) Counter(id uint64) uint64 {
+	for _, c := range v {
+		if c.ID == id {
+			return c.Value
+		}
+	}
+	return 0
+}

Godeps/_workspace/src/github.com/syncthing/protocol/vector_compare.go

@@ -0,0 +1,89 @@
+// Copyright (C) 2015 The Protocol Authors.
+
+package protocol
+
+// Ordering represents the relationship between two Vectors.
+type Ordering int
+
+const (
+	Equal Ordering = iota
+	Greater
+	Lesser
+	ConcurrentLesser
+	ConcurrentGreater
+)
+
+// There's really no such thing as "concurrent lesser" and "concurrent
+// greater" in version vectors, just "concurrent". But it's useful to be able
+// to get a strict ordering between versions for stable sorts and so on, so we
+// return both variants. The convenience method Concurrent() can be used to
+// check for either case.
+
+// Compare returns the Ordering that describes a's relation to b.
+func (a Vector) Compare(b Vector) Ordering {
+	var ai, bi int     // index into a and b
+	var av, bv Counter // value at current index
+
+	result := Equal
+
+	for ai < len(a) || bi < len(b) {
+		var aMissing, bMissing bool
+
+		if ai < len(a) {
+			av = a[ai]
+		} else {
+			av = Counter{}
+			aMissing = true
+		}
+
+		if bi < len(b) {
+			bv = b[bi]
+		} else {
+			bv = Counter{}
+			bMissing = true
+		}
+
+		switch {
+		case av.ID == bv.ID:
+			// We have a counter value for each side
+			if av.Value > bv.Value {
+				if result == Lesser {
+					return ConcurrentLesser
+				}
+				result = Greater
+			} else if av.Value < bv.Value {
+				if result == Greater {
+					return ConcurrentGreater
+				}
+				result = Lesser
+			}
+
+		case !aMissing && av.ID < bv.ID || bMissing:
+			// Value is missing on the b side
+			if av.Value > 0 {
+				if result == Lesser {
+					return ConcurrentLesser
+				}
+				result = Greater
+			}
+
+		case !bMissing && bv.ID < av.ID || aMissing:
+			// Value is missing on the a side
+			if bv.Value > 0 {
+				if result == Greater {
+					return ConcurrentGreater
+				}
+				result = Lesser
+			}
+		}
+
+		if ai < len(a) && (av.ID <= bv.ID || bMissing) {
+			ai++
+		}
+		if bi < len(b) && (bv.ID <= av.ID || aMissing) {
+			bi++
+		}
+	}
+
+	return result
+}

Godeps/_workspace/src/github.com/syncthing/protocol/vector_compare_test.go

@@ -0,0 +1,249 @@
+// Copyright (C) 2015 The Protocol Authors.
+
+package protocol
+
+import (
+	"math"
+	"testing"
+)
+
+func TestCompare(t *testing.T) {
+	testcases := []struct {
+		a, b Vector
+		r    Ordering
+	}{
+		// Empty vectors are identical
+		{Vector{}, Vector{}, Equal},
+		{Vector{}, nil, Equal},
+		{nil, Vector{}, Equal},
+		{nil, Vector{Counter{42, 0}}, Equal},
+		{Vector{}, Vector{Counter{42, 0}}, Equal},
+		{Vector{Counter{42, 0}}, nil, Equal},
+		{Vector{Counter{42, 0}}, Vector{}, Equal},
+
+		// Zero is the implied value for a missing Counter
+		{
+			Vector{Counter{42, 0}},
+			Vector{Counter{77, 0}},
+			Equal,
+		},
+
+		// Equal vectors are equal
+		{
+			Vector{Counter{42, 33}},
+			Vector{Counter{42, 33}},
+			Equal,
+		},
+		{
+			Vector{Counter{42, 33}, Counter{77, 24}},
+			Vector{Counter{42, 33}, Counter{77, 24}},
+			Equal,
+		},
+
+		// These a-vectors are all greater than the b-vector
+		{
+			Vector{Counter{42, 1}},
+			nil,
+			Greater,
+		},
+		{
+			Vector{Counter{42, 1}},
+			Vector{},
+			Greater,
+		},
+		{
+			Vector{Counter{0, 1}},
+			Vector{Counter{0, 0}},
+			Greater,
+		},
+		{
+			Vector{Counter{42, 1}},
+			Vector{Counter{42, 0}},
+			Greater,
+		},
+		{
+			Vector{Counter{math.MaxUint64, 1}},
+			Vector{Counter{math.MaxUint64, 0}},
+			Greater,
+		},
+		{
+			Vector{Counter{0, math.MaxUint64}},
+			Vector{Counter{0, 0}},
+			Greater,
+		},
+		{
+			Vector{Counter{42, math.MaxUint64}},
+			Vector{Counter{42, 0}},
+			Greater,
+		},
+		{
+			Vector{Counter{math.MaxUint64, math.MaxUint64}},
+			Vector{Counter{math.MaxUint64, 0}},
+			Greater,
+		},
+		{
+			Vector{Counter{0, math.MaxUint64}},
+			Vector{Counter{0, math.MaxUint64 - 1}},
+			Greater,
+		},
+		{
+			Vector{Counter{42, math.MaxUint64}},
+			Vector{Counter{42, math.MaxUint64 - 1}},
+			Greater,
+		},
+		{
+			Vector{Counter{math.MaxUint64, math.MaxUint64}},
+			Vector{Counter{math.MaxUint64, math.MaxUint64 - 1}},
+			Greater,
+		},
+		{
+			Vector{Counter{42, 2}},
+			Vector{Counter{42, 1}},
+			Greater,
+		},
+		{
+			Vector{Counter{22, 22}, Counter{42, 2}},
+			Vector{Counter{22, 22}, Counter{42, 1}},
+			Greater,
+		},
+		{
+			Vector{Counter{42, 2}, Counter{77, 3}},
+			Vector{Counter{42, 1}, Counter{77, 3}},
+			Greater,
+		},
+		{
+			Vector{Counter{22, 22}, Counter{42, 2}, Counter{77, 3}},
+			Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
+			Greater,
+		},
+		{
+			Vector{Counter{22, 23}, Counter{42, 2}, Counter{77, 4}},
+			Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
+			Greater,
+		},
+
+		// These a-vectors are all lesser than the b-vector
+		{nil, Vector{Counter{42, 1}}, Lesser},
+		{Vector{}, Vector{Counter{42, 1}}, Lesser},
+		{
+			Vector{Counter{42, 0}},
+			Vector{Counter{42, 1}},
+			Lesser,
+		},
+		{
+			Vector{Counter{42, 1}},
+			Vector{Counter{42, 2}},
+			Lesser,
+		},
+		{
+			Vector{Counter{22, 22}, Counter{42, 1}},
+			Vector{Counter{22, 22}, Counter{42, 2}},
+			Lesser,
+		},
+		{
+			Vector{Counter{42, 1}, Counter{77, 3}},
+			Vector{Counter{42, 2}, Counter{77, 3}},
+			Lesser,
+		},
+		{
+			Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
+			Vector{Counter{22, 22}, Counter{42, 2}, Counter{77, 3}},
+			Lesser,
+		},
+		{
+			Vector{Counter{22, 22}, Counter{42, 1}, Counter{77, 3}},
+			Vector{Counter{22, 23}, Counter{42, 2}, Counter{77, 4}},
+			Lesser,
+		},
+
+		// These are all in conflict
+		{
+			Vector{Counter{42, 2}},
+			Vector{Counter{43, 1}},
+			ConcurrentGreater,
+		},
+		{
+			Vector{Counter{43, 1}},
+			Vector{Counter{42, 2}},
+			ConcurrentLesser,
+		},
+		{
+			Vector{Counter{22, 23}, Counter{42, 1}},
+			Vector{Counter{22, 22}, Counter{42, 2}},
+			ConcurrentGreater,
+		},
+		{
+			Vector{Counter{22, 21}, Counter{42, 2}},
+			Vector{Counter{22, 22}, Counter{42, 1}},
+			ConcurrentLesser,
+		},
+		{
+			Vector{Counter{22, 21}, Counter{42, 2}, Counter{43, 1}},
+			Vector{Counter{20, 1}, Counter{22, 22}, Counter{42, 1}},
+			ConcurrentLesser,
+		},
+	}
+
+	for i, tc := range testcases {
+		// Test real Compare
+		if r := tc.a.Compare(tc.b); r != tc.r {
+			t.Errorf("%d: %+v.Compare(%+v) == %v (expected %v)", i, tc.a, tc.b, r, tc.r)
+		}
+
+		// Test convenience functions
+		switch tc.r {
+		case Greater:
+			if tc.a.Equal(tc.b) {
+				t.Errorf("%+v == %+v", tc.a, tc.b)
+			}
+			if tc.a.Concurrent(tc.b) {
+				t.Errorf("%+v concurrent %+v", tc.a, tc.b)
+			}
+			if !tc.a.GreaterEqual(tc.b) {
+				t.Errorf("%+v not >= %+v", tc.a, tc.b)
+			}
+			if tc.a.LesserEqual(tc.b) {
+				t.Errorf("%+v <= %+v", tc.a, tc.b)
+			}
+		case Lesser:
+			if tc.a.Concurrent(tc.b) {
+				t.Errorf("%+v concurrent %+v", tc.a, tc.b)
+			}
+			if tc.a.Equal(tc.b) {
+				t.Errorf("%+v == %+v", tc.a, tc.b)
+			}
+			if tc.a.GreaterEqual(tc.b) {
+				t.Errorf("%+v >= %+v", tc.a, tc.b)
+			}
+			if !tc.a.LesserEqual(tc.b) {
+				t.Errorf("%+v not <= %+v", tc.a, tc.b)
+			}
+		case Equal:
+			if tc.a.Concurrent(tc.b) {
+				t.Errorf("%+v concurrent %+v", tc.a, tc.b)
+			}
+			if !tc.a.Equal(tc.b) {
+				t.Errorf("%+v not == %+v", tc.a, tc.b)
+			}
+			if !tc.a.GreaterEqual(tc.b) {
+				t.Errorf("%+v not <= %+v", tc.a, tc.b)
+			}
+			if !tc.a.LesserEqual(tc.b) {
+				t.Errorf("%+v not <= %+v", tc.a, tc.b)
+			}
+		case ConcurrentLesser, ConcurrentGreater:
+			if !tc.a.Concurrent(tc.b) {
+				t.Errorf("%+v not concurrent %+v", tc.a, tc.b)
+			}
+			if tc.a.Equal(tc.b) {
+				t.Errorf("%+v == %+v", tc.a, tc.b)
+			}
+			if tc.a.GreaterEqual(tc.b) {
+				t.Errorf("%+v >= %+v", tc.a, tc.b)
+			}
+			if tc.a.LesserEqual(tc.b) {
+				t.Errorf("%+v <= %+v", tc.a, tc.b)
+			}
+		}
+	}
+}

Godeps/_workspace/src/github.com/syncthing/protocol/vector_test.go

@@ -0,0 +1,134 @@
+// Copyright (C) 2015 The Protocol Authors.
+
+package protocol
+
+import "testing"
+
+func TestUpdate(t *testing.T) {
+	var v Vector
+
+	// Append
+
+	v = v.Update(42)
+	expected := Vector{Counter{42, 1}}
+
+	if v.Compare(expected) != Equal {
+		t.Errorf("Update error, %+v != %+v", v, expected)
+	}
+
+	// Insert at front
+
+	v = v.Update(36)
+	expected = Vector{Counter{36, 1}, Counter{42, 1}}
+
+	if v.Compare(expected) != Equal {
+		t.Errorf("Update error, %+v != %+v", v, expected)
+	}
+
+	// Insert in moddle
+
+	v = v.Update(37)
+	expected = Vector{Counter{36, 1}, Counter{37, 1}, Counter{42, 1}}
+
+	if v.Compare(expected) != Equal {
+		t.Errorf("Update error, %+v != %+v", v, expected)
+	}
+
+	// Update existing
+
+	v = v.Update(37)
+	expected = Vector{Counter{36, 1}, Counter{37, 2}, Counter{42, 1}}
+
+	if v.Compare(expected) != Equal {
+		t.Errorf("Update error, %+v != %+v", v, expected)
+	}
+}
+
+func TestCopy(t *testing.T) {
+	v0 := Vector{Counter{42, 1}}
+	v1 := v0.Copy()
+	v1.Update(42)
+	if v0.Compare(v1) != Lesser {
+		t.Errorf("Copy error, %+v should be ancestor of %+v", v0, v1)
+	}
+}
+
+func TestMerge(t *testing.T) {
+	testcases := []struct {
+		a, b, m Vector
+	}{
+		// No-ops
+		{
+			Vector{},
+			Vector{},
+			Vector{},
+		},
+		{
+			Vector{Counter{22, 1}, Counter{42, 1}},
+			Vector{Counter{22, 1}, Counter{42, 1}},
+			Vector{Counter{22, 1}, Counter{42, 1}},
+		},
+
+		// Appends
+		{
+			Vector{},
+			Vector{Counter{22, 1}, Counter{42, 1}},
+			Vector{Counter{22, 1}, Counter{42, 1}},
+		},
+		{
+			Vector{Counter{22, 1}},
+			Vector{Counter{42, 1}},
+			Vector{Counter{22, 1}, Counter{42, 1}},
+		},
+		{
+			Vector{Counter{22, 1}},
+			Vector{Counter{22, 1}, Counter{42, 1}},
+			Vector{Counter{22, 1}, Counter{42, 1}},
+		},
+
+		// Insert
+		{
+			Vector{Counter{22, 1}, Counter{42, 1}},
+			Vector{Counter{22, 1}, Counter{23, 2}, Counter{42, 1}},
+			Vector{Counter{22, 1}, Counter{23, 2}, Counter{42, 1}},
+		},
+		{
+			Vector{Counter{42, 1}},
+			Vector{Counter{22, 1}},
+			Vector{Counter{22, 1}, Counter{42, 1}},
+		},
+
+		// Update
+		{
+			Vector{Counter{22, 1}, Counter{42, 2}},
+			Vector{Counter{22, 2}, Counter{42, 1}},
+			Vector{Counter{22, 2}, Counter{42, 2}},
+		},
+
+		// All of the above
+		{
+			Vector{Counter{10, 1}, Counter{20, 2}, Counter{30, 1}},
+			Vector{Counter{5, 1}, Counter{10, 2}, Counter{15, 1}, Counter{20, 1}, Counter{25, 1}, Counter{35, 1}},
+			Vector{Counter{5, 1}, Counter{10, 2}, Counter{15, 1}, Counter{20, 2}, Counter{25, 1}, Counter{30, 1}, Counter{35, 1}},
+		},
+	}
+
+	for i, tc := range testcases {
+		if m := tc.a.Merge(tc.b); m.Compare(tc.m) != Equal {
+			t.Errorf("%d: %+v.Merge(%+v) == %+v (expected %+v)", i, tc.a, tc.b, m, tc.m)
+		}
+	}
+}
+
+func TestCounterValue(t *testing.T) {
+	v0 := Vector{Counter{42, 1}, Counter{64, 5}}
+	if v0.Counter(42) != 1 {
+		t.Error("Counter error, %d != %d", v0.Counter(42), 1)
+	}
+	if v0.Counter(64) != 5 {
+		t.Error("Counter error, %d != %d", v0.Counter(64), 5)
+	}
+	if v0.Counter(72) != 0 {
+		t.Error("Counter error, %d != %d", v0.Counter(72), 0)
+	}
+}

Godeps/_workspace/src/github.com/syncthing/protocol/vector_xdr.go

@@ -0,0 +1,38 @@
+// Copyright (C) 2015 The Protocol Authors.
+
+package protocol
+
+// This stuff is hacked up manually because genxdr doesn't support 'type
+// Vector []Counter' declarations and it was tricky when I tried to add it...
+
+type xdrWriter interface {
+	WriteUint32(uint32) (int, error)
+	WriteUint64(uint64) (int, error)
+}
+type xdrReader interface {
+	ReadUint32() uint32
+	ReadUint64() uint64
+}
+
+// EncodeXDRInto encodes the vector as an XDR object into the given XDR
+// encoder.
+func (v Vector) EncodeXDRInto(w xdrWriter) (int, error) {
+	w.WriteUint32(uint32(len(v)))
+	for i := range v {
+		w.WriteUint64(v[i].ID)
+		w.WriteUint64(v[i].Value)
+	}
+	return 4 + 16*len(v), nil
+}
+
+// DecodeXDRFrom decodes the XDR objects from the given reader into itself.
+func (v *Vector) DecodeXDRFrom(r xdrReader) error {
+	l := int(r.ReadUint32())
+	n := make(Vector, l)
+	for i := range n {
+		n[i].ID = r.ReadUint64()
+		n[i].Value = r.ReadUint64()
+	}
+	*v = n
+	return nil
+}

Godeps/_workspace/src/github.com/syncthing/protocol/wireformat.go

@@ -20,7 +20,7 @@ func (c wireFormatConnection) Name() string {
 	return c.next.Name()
 }
 
-func (c wireFormatConnection) Index(folder string, fs []FileInfo) error {
+func (c wireFormatConnection) Index(folder string, fs []FileInfo, flags uint32, options []Option) error {
 	var myFs = make([]FileInfo, len(fs))
 	copy(myFs, fs)
 
@@ -28,10 +28,10 @@ func (c wireFormatConnection) Index(folder string, fs []FileInfo) error {
 		myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name))
 	}
 
-	return c.next.Index(folder, myFs)
+	return c.next.Index(folder, myFs, flags, options)
 }
 
-func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo) error {
+func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo, flags uint32, options []Option) error {
 	var myFs = make([]FileInfo, len(fs))
 	copy(myFs, fs)
 
@@ -39,12 +39,12 @@ func (c wireFormatConnection) IndexUpdate(folder string, fs []FileInfo) error {
 		myFs[i].Name = norm.NFC.String(filepath.ToSlash(myFs[i].Name))
 	}
 
-	return c.next.IndexUpdate(folder, myFs)
+	return c.next.IndexUpdate(folder, myFs, flags, options)
 }
 
-func (c wireFormatConnection) Request(folder, name string, offset int64, size int) ([]byte, error) {
+func (c wireFormatConnection) Request(folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
 	name = norm.NFC.String(filepath.ToSlash(name))
-	return c.next.Request(folder, name, offset, size)
+	return c.next.Request(folder, name, offset, size, hash, flags, options)
 }
 
 func (c wireFormatConnection) ClusterConfig(config ClusterConfigMessage) {

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench2_test.go

@@ -4,7 +4,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-// +build go1.3
+// +build !go1.2
 
 package leveldb
 

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/bench2_test.go

@@ -0,0 +1,30 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// +build !go1.2
+
+package cache
+
+import (
+	"math/rand"
+	"testing"
+)
+
+func BenchmarkLRUCache(b *testing.B) {
+	c := NewCache(NewLRU(10000))
+
+	b.SetParallelism(10)
+	b.RunParallel(func(pb *testing.PB) {
+		r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+		for pb.Next() {
+			key := uint64(r.Intn(1000000))
+			c.Get(0, key, func() (int, Value) {
+				return 1, key
+			}).Release()
+		}
+	})
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go

@@ -552,19 +552,3 @@ func TestLRUCache_Close(t *testing.T) {
 		t.Errorf("delFunc isn't called 1 times: got=%d", delFuncCalled)
 	}
 }
-
-func BenchmarkLRUCache(b *testing.B) {
-	c := NewCache(NewLRU(10000))
-
-	b.SetParallelism(10)
-	b.RunParallel(func(pb *testing.PB) {
-		r := rand.New(rand.NewSource(time.Now().UnixNano()))
-
-		for pb.Next() {
-			key := uint64(r.Intn(1000000))
-			c.Get(0, key, func() (int, Value) {
-				return 1, key
-			}).Release()
-		}
-	})
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go

@@ -347,12 +347,14 @@ func recoverTable(s *session, o *opt.Options) error {
 			return err
 		}
 		iter := tr.NewIterator(nil, nil)
-		iter.(iterator.ErrorCallbackSetter).SetErrorCallback(func(err error) {
-			if errors.IsCorrupted(err) {
-				s.logf("table@recovery block corruption @%d %q", file.Num(), err)
-				tcorruptedBlock++
-			}
-		})
+		if itererr, ok := iter.(iterator.ErrorCallbackSetter); ok {
+			itererr.SetErrorCallback(func(err error) {
+				if errors.IsCorrupted(err) {
+					s.logf("table@recovery block corruption @%d %q", file.Num(), err)
+					tcorruptedBlock++
+				}
+			})
+		}
 
 		// Scan the table.
 		for iter.Next() {

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go

@@ -153,7 +153,7 @@ type Options struct {
 	BlockCacher Cacher
 
 	// BlockCacheCapacity defines the capacity of the 'sorted table' block caching.
-	// Use -1 for zero, this has same effect with specifying NoCacher to BlockCacher.
+	// Use -1 for zero, this has same effect as specifying NoCacher to BlockCacher.
 	//
 	// The default value is 8MiB.
 	BlockCacheCapacity int
@@ -308,7 +308,7 @@ type Options struct {
 	OpenFilesCacher Cacher
 
 	// OpenFilesCacheCapacity defines the capacity of the open files caching.
-	// Use -1 for zero, this has same effect with specifying NoCacher to OpenFilesCacher.
+	// Use -1 for zero, this has same effect as specifying NoCacher to OpenFilesCacher.
 	//
 	// The default value is 500.
 	OpenFilesCacheCapacity int
@@ -355,9 +355,9 @@ func (o *Options) GetBlockCacher() Cacher {
 }
 
 func (o *Options) GetBlockCacheCapacity() int {
-	if o == nil || o.BlockCacheCapacity <= 0 {
+	if o == nil || o.BlockCacheCapacity == 0 {
 		return DefaultBlockCacheCapacity
-	} else if o.BlockCacheCapacity == -1 {
+	} else if o.BlockCacheCapacity < 0 {
 		return 0
 	}
 	return o.BlockCacheCapacity
@@ -497,7 +497,7 @@ func (o *Options) GetMaxMemCompationLevel() int {
 	if o != nil {
 		if o.MaxMemCompationLevel > 0 {
 			level = o.MaxMemCompationLevel
-		} else if o.MaxMemCompationLevel == -1 {
+		} else if o.MaxMemCompationLevel < 0 {
 			level = 0
 		}
 	}
@@ -525,9 +525,9 @@ func (o *Options) GetOpenFilesCacher() Cacher {
 }
 
 func (o *Options) GetOpenFilesCacheCapacity() int {
-	if o == nil || o.OpenFilesCacheCapacity <= 0 {
+	if o == nil || o.OpenFilesCacheCapacity == 0 {
 		return DefaultOpenFilesCacheCapacity
-	} else if o.OpenFilesCacheCapacity == -1 {
+	} else if o.OpenFilesCacheCapacity < 0 {
 		return 0
 	}
 	return o.OpenFilesCacheCapacity

Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/decode.go

@@ -7,10 +7,15 @@ package snappy
 import (
 	"encoding/binary"
 	"errors"
+	"io"
 )
 
-// ErrCorrupt reports that the input is invalid.
-var ErrCorrupt = errors.New("snappy: corrupt input")
+var (
+	// ErrCorrupt reports that the input is invalid.
+	ErrCorrupt = errors.New("snappy: corrupt input")
+	// ErrUnsupported reports that the input isn't supported.
+	ErrUnsupported = errors.New("snappy: unsupported input")
+)
 
 // DecodedLen returns the length of the decoded block.
 func DecodedLen(src []byte) (int, error) {
@@ -122,3 +127,166 @@ func Decode(dst, src []byte) ([]byte, error) {
 	}
 	return dst[:d], nil
 }
+
+// NewReader returns a new Reader that decompresses from r, using the framing
+// format described at
+// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
+func NewReader(r io.Reader) *Reader {
+	return &Reader{
+		r:       r,
+		decoded: make([]byte, maxUncompressedChunkLen),
+		buf:     make([]byte, MaxEncodedLen(maxUncompressedChunkLen)+checksumSize),
+	}
+}
+
+// Reader is an io.Reader than can read Snappy-compressed bytes.
+type Reader struct {
+	r       io.Reader
+	err     error
+	decoded []byte
+	buf     []byte
+	// decoded[i:j] contains decoded bytes that have not yet been passed on.
+	i, j       int
+	readHeader bool
+}
+
+// Reset discards any buffered data, resets all state, and switches the Snappy
+// reader to read from r. This permits reusing a Reader rather than allocating
+// a new one.
+func (r *Reader) Reset(reader io.Reader) {
+	r.r = reader
+	r.err = nil
+	r.i = 0
+	r.j = 0
+	r.readHeader = false
+}
+
+func (r *Reader) readFull(p []byte) (ok bool) {
+	if _, r.err = io.ReadFull(r.r, p); r.err != nil {
+		if r.err == io.ErrUnexpectedEOF {
+			r.err = ErrCorrupt
+		}
+		return false
+	}
+	return true
+}
+
+// Read satisfies the io.Reader interface.
+func (r *Reader) Read(p []byte) (int, error) {
+	if r.err != nil {
+		return 0, r.err
+	}
+	for {
+		if r.i < r.j {
+			n := copy(p, r.decoded[r.i:r.j])
+			r.i += n
+			return n, nil
+		}
+		if !r.readFull(r.buf[:4]) {
+			return 0, r.err
+		}
+		chunkType := r.buf[0]
+		if !r.readHeader {
+			if chunkType != chunkTypeStreamIdentifier {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			r.readHeader = true
+		}
+		chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16
+		if chunkLen > len(r.buf) {
+			r.err = ErrUnsupported
+			return 0, r.err
+		}
+
+		// The chunk types are specified at
+		// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
+		switch chunkType {
+		case chunkTypeCompressedData:
+			// Section 4.2. Compressed data (chunk type 0x00).
+			if chunkLen < checksumSize {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			buf := r.buf[:chunkLen]
+			if !r.readFull(buf) {
+				return 0, r.err
+			}
+			checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
+			buf = buf[checksumSize:]
+
+			n, err := DecodedLen(buf)
+			if err != nil {
+				r.err = err
+				return 0, r.err
+			}
+			if n > len(r.decoded) {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			if _, err := Decode(r.decoded, buf); err != nil {
+				r.err = err
+				return 0, r.err
+			}
+			if crc(r.decoded[:n]) != checksum {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			r.i, r.j = 0, n
+			continue
+
+		case chunkTypeUncompressedData:
+			// Section 4.3. Uncompressed data (chunk type 0x01).
+			if chunkLen < checksumSize {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			buf := r.buf[:checksumSize]
+			if !r.readFull(buf) {
+				return 0, r.err
+			}
+			checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
+			// Read directly into r.decoded instead of via r.buf.
+			n := chunkLen - checksumSize
+			if !r.readFull(r.decoded[:n]) {
+				return 0, r.err
+			}
+			if crc(r.decoded[:n]) != checksum {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			r.i, r.j = 0, n
+			continue
+
+		case chunkTypeStreamIdentifier:
+			// Section 4.1. Stream identifier (chunk type 0xff).
+			if chunkLen != len(magicBody) {
+				r.err = ErrCorrupt
+				return 0, r.err
+			}
+			if !r.readFull(r.buf[:len(magicBody)]) {
+				return 0, r.err
+			}
+			for i := 0; i < len(magicBody); i++ {
+				if r.buf[i] != magicBody[i] {
+					r.err = ErrCorrupt
+					return 0, r.err
+				}
+			}
+			continue
+		}
+
+		if chunkType <= 0x7f {
+			// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
+			r.err = ErrUnsupported
+			return 0, r.err
+
+		} else {
+			// Section 4.4 Padding (chunk type 0xfe).
+			// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
+			if !r.readFull(r.buf[:chunkLen]) {
+				return 0, r.err
+			}
+		}
+	}
+}

Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/encode.go

@@ -6,6 +6,7 @@ package snappy
 
 import (
 	"encoding/binary"
+	"io"
 )
 
 // We limit how far copy back-references can go, the same as the C++ code.
@@ -172,3 +173,86 @@ func MaxEncodedLen(srcLen int) int {
 	// This last factor dominates the blowup, so the final estimate is:
 	return 32 + srcLen + srcLen/6
 }
+
+// NewWriter returns a new Writer that compresses to w, using the framing
+// format described at
+// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
+func NewWriter(w io.Writer) *Writer {
+	return &Writer{
+		w:   w,
+		enc: make([]byte, MaxEncodedLen(maxUncompressedChunkLen)),
+	}
+}
+
+// Writer is an io.Writer than can write Snappy-compressed bytes.
+type Writer struct {
+	w           io.Writer
+	err         error
+	enc         []byte
+	buf         [checksumSize + chunkHeaderSize]byte
+	wroteHeader bool
+}
+
+// Reset discards the writer's state and switches the Snappy writer to write to
+// w. This permits reusing a Writer rather than allocating a new one.
+func (w *Writer) Reset(writer io.Writer) {
+	w.w = writer
+	w.err = nil
+	w.wroteHeader = false
+}
+
+// Write satisfies the io.Writer interface.
+func (w *Writer) Write(p []byte) (n int, errRet error) {
+	if w.err != nil {
+		return 0, w.err
+	}
+	if !w.wroteHeader {
+		copy(w.enc, magicChunk)
+		if _, err := w.w.Write(w.enc[:len(magicChunk)]); err != nil {
+			w.err = err
+			return n, err
+		}
+		w.wroteHeader = true
+	}
+	for len(p) > 0 {
+		var uncompressed []byte
+		if len(p) > maxUncompressedChunkLen {
+			uncompressed, p = p[:maxUncompressedChunkLen], p[maxUncompressedChunkLen:]
+		} else {
+			uncompressed, p = p, nil
+		}
+		checksum := crc(uncompressed)
+
+		// Compress the buffer, discarding the result if the improvement
+		// isn't at least 12.5%.
+		chunkType := uint8(chunkTypeCompressedData)
+		chunkBody, err := Encode(w.enc, uncompressed)
+		if err != nil {
+			w.err = err
+			return n, err
+		}
+		if len(chunkBody) >= len(uncompressed)-len(uncompressed)/8 {
+			chunkType, chunkBody = chunkTypeUncompressedData, uncompressed
+		}
+
+		chunkLen := 4 + len(chunkBody)
+		w.buf[0] = chunkType
+		w.buf[1] = uint8(chunkLen >> 0)
+		w.buf[2] = uint8(chunkLen >> 8)
+		w.buf[3] = uint8(chunkLen >> 16)
+		w.buf[4] = uint8(checksum >> 0)
+		w.buf[5] = uint8(checksum >> 8)
+		w.buf[6] = uint8(checksum >> 16)
+		w.buf[7] = uint8(checksum >> 24)
+		if _, err = w.w.Write(w.buf[:]); err != nil {
+			w.err = err
+			return n, err
+		}
+		if _, err = w.w.Write(chunkBody); err != nil {
+			w.err = err
+			return n, err
+		}
+		n += len(uncompressed)
+	}
+	return n, nil
+}

Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy.go

@@ -8,6 +8,10 @@
 // The C++ snappy implementation is at http://code.google.com/p/snappy/
 package snappy
 
+import (
+	"hash/crc32"
+)
+
 /*
 Each encoded block begins with the varint-encoded length of the decoded data,
 followed by a sequence of chunks. Chunks begin and end on byte boundaries. The
@@ -36,3 +40,29 @@ const (
 	tagCopy2   = 0x02
 	tagCopy4   = 0x03
 )
+
+const (
+	checksumSize    = 4
+	chunkHeaderSize = 4
+	magicChunk      = "\xff\x06\x00\x00" + magicBody
+	magicBody       = "sNaPpY"
+	// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt says
+	// that "the uncompressed data in a chunk must be no longer than 65536 bytes".
+	maxUncompressedChunkLen = 65536
+)
+
+const (
+	chunkTypeCompressedData   = 0x00
+	chunkTypeUncompressedData = 0x01
+	chunkTypePadding          = 0xfe
+	chunkTypeStreamIdentifier = 0xff
+)
+
+var crcTable = crc32.MakeTable(crc32.Castagnoli)
+
+// crc implements the checksum specified in section 3 of
+// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
+func crc(b []byte) uint32 {
+	c := crc32.Update(0, crcTable, b)
+	return uint32(c>>15|c<<17) + 0xa282ead8
+}

Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy/snappy_test.go

@@ -18,7 +18,10 @@ import (
 	"testing"
 )
 
-var download = flag.Bool("download", false, "If true, download any missing files before running benchmarks")
+var (
+	download = flag.Bool("download", false, "If true, download any missing files before running benchmarks")
+	testdata = flag.String("testdata", "testdata", "Directory containing the test data")
+)
 
 func roundtrip(b, ebuf, dbuf []byte) error {
 	e, err := Encode(ebuf, b)
@@ -55,11 +58,11 @@ func TestSmallCopy(t *testing.T) {
 }
 
 func TestSmallRand(t *testing.T) {
-	rand.Seed(27354294)
+	rng := rand.New(rand.NewSource(27354294))
 	for n := 1; n < 20000; n += 23 {
 		b := make([]byte, n)
-		for i, _ := range b {
-			b[i] = uint8(rand.Uint32())
+		for i := range b {
+			b[i] = uint8(rng.Uint32())
 		}
 		if err := roundtrip(b, nil, nil); err != nil {
 			t.Fatal(err)
@@ -70,7 +73,7 @@ func TestSmallRand(t *testing.T) {
 func TestSmallRegular(t *testing.T) {
 	for n := 1; n < 20000; n += 23 {
 		b := make([]byte, n)
-		for i, _ := range b {
+		for i := range b {
 			b[i] = uint8(i%10 + 'a')
 		}
 		if err := roundtrip(b, nil, nil); err != nil {
@@ -79,6 +82,120 @@ func TestSmallRegular(t *testing.T) {
 	}
 }
 
+func cmp(a, b []byte) error {
+	if len(a) != len(b) {
+		return fmt.Errorf("got %d bytes, want %d", len(a), len(b))
+	}
+	for i := range a {
+		if a[i] != b[i] {
+			return fmt.Errorf("byte #%d: got 0x%02x, want 0x%02x", i, a[i], b[i])
+		}
+	}
+	return nil
+}
+
+func TestFramingFormat(t *testing.T) {
+	// src is comprised of alternating 1e5-sized sequences of random
+	// (incompressible) bytes and repeated (compressible) bytes. 1e5 was chosen
+	// because it is larger than maxUncompressedChunkLen (64k).
+	src := make([]byte, 1e6)
+	rng := rand.New(rand.NewSource(1))
+	for i := 0; i < 10; i++ {
+		if i%2 == 0 {
+			for j := 0; j < 1e5; j++ {
+				src[1e5*i+j] = uint8(rng.Intn(256))
+			}
+		} else {
+			for j := 0; j < 1e5; j++ {
+				src[1e5*i+j] = uint8(i)
+			}
+		}
+	}
+
+	buf := new(bytes.Buffer)
+	if _, err := NewWriter(buf).Write(src); err != nil {
+		t.Fatalf("Write: encoding: %v", err)
+	}
+	dst, err := ioutil.ReadAll(NewReader(buf))
+	if err != nil {
+		t.Fatalf("ReadAll: decoding: %v", err)
+	}
+	if err := cmp(dst, src); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestReaderReset(t *testing.T) {
+	gold := bytes.Repeat([]byte("All that is gold does not glitter,\n"), 10000)
+	buf := new(bytes.Buffer)
+	if _, err := NewWriter(buf).Write(gold); err != nil {
+		t.Fatalf("Write: %v", err)
+	}
+	encoded, invalid, partial := buf.String(), "invalid", "partial"
+	r := NewReader(nil)
+	for i, s := range []string{encoded, invalid, partial, encoded, partial, invalid, encoded, encoded} {
+		if s == partial {
+			r.Reset(strings.NewReader(encoded))
+			if _, err := r.Read(make([]byte, 101)); err != nil {
+				t.Errorf("#%d: %v", i, err)
+				continue
+			}
+			continue
+		}
+		r.Reset(strings.NewReader(s))
+		got, err := ioutil.ReadAll(r)
+		switch s {
+		case encoded:
+			if err != nil {
+				t.Errorf("#%d: %v", i, err)
+				continue
+			}
+			if err := cmp(got, gold); err != nil {
+				t.Errorf("#%d: %v", i, err)
+				continue
+			}
+		case invalid:
+			if err == nil {
+				t.Errorf("#%d: got nil error, want non-nil", i)
+				continue
+			}
+		}
+	}
+}
+
+func TestWriterReset(t *testing.T) {
+	gold := bytes.Repeat([]byte("Not all those who wander are lost;\n"), 10000)
+	var gots, wants [][]byte
+	const n = 20
+	w, failed := NewWriter(nil), false
+	for i := 0; i <= n; i++ {
+		buf := new(bytes.Buffer)
+		w.Reset(buf)
+		want := gold[:len(gold)*i/n]
+		if _, err := w.Write(want); err != nil {
+			t.Errorf("#%d: Write: %v", i, err)
+			failed = true
+			continue
+		}
+		got, err := ioutil.ReadAll(NewReader(buf))
+		if err != nil {
+			t.Errorf("#%d: ReadAll: %v", i, err)
+			failed = true
+			continue
+		}
+		gots = append(gots, got)
+		wants = append(wants, want)
+	}
+	if failed {
+		return
+	}
+	for i := range gots {
+		if err := cmp(gots[i], wants[i]); err != nil {
+			t.Errorf("#%d: %v", i, err)
+		}
+	}
+}
+
 func benchDecode(b *testing.B, src []byte) {
 	encoded, err := Encode(nil, src)
 	if err != nil {
@@ -102,7 +219,7 @@ func benchEncode(b *testing.B, src []byte) {
 	}
 }
 
-func readFile(b *testing.B, filename string) []byte {
+func readFile(b testing.TB, filename string) []byte {
 	src, err := ioutil.ReadFile(filename)
 	if err != nil {
 		b.Fatalf("failed reading %s: %s", filename, err)
@@ -144,7 +261,7 @@ func BenchmarkWordsEncode1e5(b *testing.B) { benchWords(b, 1e5, false) }
 func BenchmarkWordsEncode1e6(b *testing.B) { benchWords(b, 1e6, false) }
 
 // testFiles' values are copied directly from
-// https://code.google.com/p/snappy/source/browse/trunk/snappy_unittest.cc.
+// https://raw.githubusercontent.com/google/snappy/master/snappy_unittest.cc
 // The label field is unused in snappy-go.
 var testFiles = []struct {
 	label    string
@@ -152,29 +269,36 @@ var testFiles = []struct {
 }{
 	{"html", "html"},
 	{"urls", "urls.10K"},
-	{"jpg", "house.jpg"},
-	{"pdf", "mapreduce-osdi-1.pdf"},
+	{"jpg", "fireworks.jpeg"},
+	{"jpg_200", "fireworks.jpeg"},
+	{"pdf", "paper-100k.pdf"},
 	{"html4", "html_x_4"},
-	{"cp", "cp.html"},
-	{"c", "fields.c"},
-	{"lsp", "grammar.lsp"},
-	{"xls", "kennedy.xls"},
 	{"txt1", "alice29.txt"},
 	{"txt2", "asyoulik.txt"},
 	{"txt3", "lcet10.txt"},
 	{"txt4", "plrabn12.txt"},
-	{"bin", "ptt5"},
-	{"sum", "sum"},
-	{"man", "xargs.1"},
 	{"pb", "geo.protodata"},
 	{"gaviota", "kppkn.gtb"},
 }
 
 // The test data files are present at this canonical URL.
-const baseURL = "https://snappy.googlecode.com/svn/trunk/testdata/"
+const baseURL = "https://raw.githubusercontent.com/google/snappy/master/testdata/"
 
 func downloadTestdata(basename string) (errRet error) {
-	filename := filepath.Join("testdata", basename)
+	filename := filepath.Join(*testdata, basename)
+	if stat, err := os.Stat(filename); err == nil && stat.Size() != 0 {
+		return nil
+	}
+
+	if !*download {
+		return fmt.Errorf("test data not found; skipping benchmark without the -download flag")
+	}
+	// Download the official snappy C++ implementation reference test data
+	// files for benchmarking.
+	if err := os.Mkdir(*testdata, 0777); err != nil && !os.IsExist(err) {
+		return fmt.Errorf("failed to create testdata: %s", err)
+	}
+
 	f, err := os.Create(filename)
 	if err != nil {
 		return fmt.Errorf("failed to create %s: %s", filename, err)
@@ -185,36 +309,27 @@ func downloadTestdata(basename string) (errRet error) {
 			os.Remove(filename)
 		}
 	}()
-	resp, err := http.Get(baseURL + basename)
+	url := baseURL + basename
+	resp, err := http.Get(url)
 	if err != nil {
-		return fmt.Errorf("failed to download %s: %s", baseURL+basename, err)
+		return fmt.Errorf("failed to download %s: %s", url, err)
 	}
 	defer resp.Body.Close()
+	if s := resp.StatusCode; s != http.StatusOK {
+		return fmt.Errorf("downloading %s: HTTP status code %d (%s)", url, s, http.StatusText(s))
+	}
 	_, err = io.Copy(f, resp.Body)
 	if err != nil {
-		return fmt.Errorf("failed to write %s: %s", filename, err)
+		return fmt.Errorf("failed to download %s to %s: %s", url, filename, err)
 	}
 	return nil
 }
 
 func benchFile(b *testing.B, n int, decode bool) {
-	filename := filepath.Join("testdata", testFiles[n].filename)
-	if stat, err := os.Stat(filename); err != nil || stat.Size() == 0 {
-		if !*download {
-			b.Fatal("test data not found; skipping benchmark without the -download flag")
-		}
-		// Download the official snappy C++ implementation reference test data
-		// files for benchmarking.
-		if err := os.Mkdir("testdata", 0777); err != nil && !os.IsExist(err) {
-			b.Fatalf("failed to create testdata: %s", err)
-		}
-		for _, tf := range testFiles {
-			if err := downloadTestdata(tf.filename); err != nil {
-				b.Fatalf("failed to download testdata: %s", err)
-			}
-		}
+	if err := downloadTestdata(testFiles[n].filename); err != nil {
+		b.Fatalf("failed to download testdata: %s", err)
 	}
-	data := readFile(b, filename)
+	data := readFile(b, filepath.Join(*testdata, testFiles[n].filename))
 	if decode {
 		benchDecode(b, data)
 	} else {
@@ -235,12 +350,6 @@ func Benchmark_UFlat8(b *testing.B)  { benchFile(b, 8, true) }
 func Benchmark_UFlat9(b *testing.B)  { benchFile(b, 9, true) }
 func Benchmark_UFlat10(b *testing.B) { benchFile(b, 10, true) }
 func Benchmark_UFlat11(b *testing.B) { benchFile(b, 11, true) }
-func Benchmark_UFlat12(b *testing.B) { benchFile(b, 12, true) }
-func Benchmark_UFlat13(b *testing.B) { benchFile(b, 13, true) }
-func Benchmark_UFlat14(b *testing.B) { benchFile(b, 14, true) }
-func Benchmark_UFlat15(b *testing.B) { benchFile(b, 15, true) }
-func Benchmark_UFlat16(b *testing.B) { benchFile(b, 16, true) }
-func Benchmark_UFlat17(b *testing.B) { benchFile(b, 17, true) }
 func Benchmark_ZFlat0(b *testing.B)  { benchFile(b, 0, false) }
 func Benchmark_ZFlat1(b *testing.B)  { benchFile(b, 1, false) }
 func Benchmark_ZFlat2(b *testing.B)  { benchFile(b, 2, false) }
@@ -253,9 +362,3 @@ func Benchmark_ZFlat8(b *testing.B)  { benchFile(b, 8, false) }
 func Benchmark_ZFlat9(b *testing.B)  { benchFile(b, 9, false) }
 func Benchmark_ZFlat10(b *testing.B) { benchFile(b, 10, false) }
 func Benchmark_ZFlat11(b *testing.B) { benchFile(b, 11, false) }
-func Benchmark_ZFlat12(b *testing.B) { benchFile(b, 12, false) }
-func Benchmark_ZFlat13(b *testing.B) { benchFile(b, 13, false) }
-func Benchmark_ZFlat14(b *testing.B) { benchFile(b, 14, false) }
-func Benchmark_ZFlat15(b *testing.B) { benchFile(b, 15, false) }
-func Benchmark_ZFlat16(b *testing.B) { benchFile(b, 16, false) }
-func Benchmark_ZFlat17(b *testing.B) { benchFile(b, 17, false) }

Godeps/_workspace/src/github.com/thejerf/suture/.travis.yml

@@ -0,0 +1,7 @@
+language: go
+go:
+  - 1.1
+  - 1.2
+  - 1.3
+  - 1.4
+  - tip

Godeps/_workspace/src/github.com/thejerf/suture/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2014 Barracuda Networks, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

Godeps/_workspace/src/github.com/thejerf/suture/README.md

@@ -0,0 +1,45 @@
+Suture
+======
+
+[![Build Status](https://travis-ci.org/thejerf/suture.png?branch=master)](https://travis-ci.org/thejerf/suture)
+
+Suture provides Erlang-ish supervisor trees for Go. "Supervisor trees" ->
+"sutree" -> "suture" -> holds your code together when it's trying to die.
+
+This is intended to be a production-quality library going into code that I
+will be very early on the phone tree to support when it goes down. However,
+it has not been deployed into something quite that serious yet. (I will
+update this statement when that changes.)
+
+It is intended to deal gracefully with the real failure cases that can
+occur with supervision trees (such as burning all your CPU time endlessly
+restarting dead services), while also making no unnecessary demands on the
+"service" code, and providing hooks to perform adequate logging with in a
+production environment.
+
+[A blog post describing the design decisions](http://www.jerf.org/iri/post/2930)
+is available.
+
+This module is fully covered with [godoc](http://godoc.org/github.com/thejerf/suture),
+including an example, usage, and everything else you might expect from a
+README.md on GitHub. (DRY.)
+
+This is not currently tagged with particular git tags for Go as this is
+currently considered to be alpha code. As I move this into production and
+feel more confident about it, I'll give it relevant tags.
+
+Code Signing
+------------
+
+Starting with the commit after ac7cf8591b, I will be signing this repository
+with the ["jerf" keybase account](https://keybase.io/jerf).
+
+Aspiration
+----------
+
+One of the big wins the Erlang community has with their pervasive OTP
+support is that it makes it easy for them to distribute libraries that
+easily fit into the OTP paradigm. It ought to someday be considered a good
+idea to distribute libraries that provide some sort of supervisor tree
+functionality out of the box. It is possible to provide this functionality
+without explicitly depending on the Suture library.

Godeps/_workspace/src/github.com/thejerf/suture/pre-commit

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+GOLINTOUT=$(golint *go)
+
+if [ ! -z "$GOLINTOUT" -o "$?" != 0 ]; then
+    echo golint failed:
+    echo $GOLINTOUT
+    exit 1
+fi
+
+go test
+

Godeps/_workspace/src/github.com/thejerf/suture/suture.go

@@ -0,0 +1,650 @@
+/*
+
+Package suture provides Erlang-like supervisor trees.
+
+This implements Erlang-esque supervisor trees, as adapted for Go. This is
+intended to be an industrial-strength implementation, but it has not yet
+been deployed in a hostile environment. (It's headed there, though.)
+
+Supervisor Tree -> SuTree -> suture -> holds your code together when it's
+trying to fall apart.
+
+Why use Suture?
+
+ * You want to write bullet-resistant services that will remain available
+   despite unforeseen failure.
+ * You need the code to be smart enough not to consume 100% of the CPU
+   restarting things.
+ * You want to easily compose multiple such services in one program.
+ * You want the Erlang programmers to stop lording their supervision
+   trees over you.
+
+Suture has 100% test coverage, and is golint clean. This doesn't prove it
+free of bugs, but it shows I care.
+
+A blog post describing the design decisions is available at
+http://www.jerf.org/iri/post/2930 .
+
+Using Suture
+
+To idiomatically use Suture, create a Supervisor which is your top level
+"application" supervisor. This will often occur in your program's "main"
+function.
+
+Create "Service"s, which implement the Service interface. .Add() them
+to your Supervisor. Supervisors are also services, so you can create a
+tree structure here, depending on the exact combination of restarts
+you want to create.
+
+Finally, as what is probably the last line of your main() function, call
+.Serve() on your top level supervisor. This will start all the services
+you've defined.
+
+See the Example for an example, using a simple service that serves out
+incrementing integers.
+
+*/
+package suture
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"math"
+	"runtime"
+	"sync/atomic"
+	"time"
+)
+
+const (
+	notRunning = iota
+	normal
+	paused
+)
+
+type supervisorID uint32
+type serviceID uint32
+
+var currentSupervisorID uint32
+
+// ErrWrongSupervisor is returned by the (*Supervisor).Remove method
+// if you pass a ServiceToken from the wrong Supervisor.
+var ErrWrongSupervisor = errors.New("wrong supervisor for this service token, no service removed")
+
+// ServiceToken is an opaque identifier that can be used to terminate a service that
+// has been Add()ed to a Supervisor.
+type ServiceToken struct {
+	id uint64
+}
+
+/*
+Supervisor is the core type of the module that represents a Supervisor.
+
+Supervisors should be constructed either by New or NewSimple.
+
+Once constructed, a Supervisor should be started in one of three ways:
+
+ 1. Calling .Serve().
+ 2. Calling .ServeBackground().
+ 3. Adding it to an existing Supervisor.
+
+Calling Serve will cause the supervisor to run until it is shut down by
+an external user calling Stop() on it. If that never happens, it simply
+runs forever. I suggest creating your services in Supervisors, then making
+a Serve() call on your top-level Supervisor be the last line of your main
+func.
+
+Calling ServeBackground will CORRECTLY start the supervisor running in a
+new goroutine. You do not want to just:
+
+  go supervisor.Serve()
+
+because that will briefly create a race condition as it starts up, if you
+try to .Add() services immediately afterward.
+
+*/
+type Supervisor struct {
+	Name string
+	id   supervisorID
+
+	failureDecay     float64
+	failureThreshold float64
+	failureBackoff   time.Duration
+	timeout          time.Duration
+	log              func(string)
+	services         map[serviceID]Service
+	lastFail         time.Time
+	failures         float64
+	restartQueue     []serviceID
+	state            uint8
+	serviceCounter   serviceID
+	control          chan supervisorMessage
+	resumeTimer      <-chan time.Time
+
+	// The testing uses the ability to grab these individual logging functions
+	// and get inside of suture's handling at a deep level.
+	// If you ever come up with some need to get into these, submit a pull
+	// request to make them public and some smidge of justification, and
+	// I'll happily do it.
+	logBadStop func(Service)
+	logFailure func(service Service, currentFailures float64, failureThreshold float64, restarting bool, error interface{}, stacktrace []byte)
+	logBackoff func(*Supervisor, bool)
+
+	// avoid a dependency on github.com/thejerf/abtime by just implementing
+	// a minimal chunk.
+	getNow    func() time.Time
+	getResume func(time.Duration) <-chan time.Time
+}
+
+// Spec is used to pass arguments to the New function to create a
+// supervisor. See the New function for full documentation.
+type Spec struct {
+	Log              func(string)
+	FailureDecay     float64
+	FailureThreshold float64
+	FailureBackoff   time.Duration
+	Timeout          time.Duration
+}
+
+/*
+
+New is the full constructor function for a supervisor.
+
+The name is a friendly human name for the supervisor, used in logging. Suture
+does not care if this is unique, but it is good for your sanity if it is.
+
+If not set, the following values are used:
+
+ * Log:               A function is created that uses log.Print.
+ * FailureDecay:      30 seconds
+ * FailureThreshold:  5 failures
+ * FailureBackoff:    15 seconds
+ * Timeout:           10 seconds
+
+The Log function will be called when errors occur. Suture will log the
+following:
+
+ * When a service has failed, with a descriptive message about the
+   current backoff status, and whether it was immediately restarted
+ * When the supervisor has gone into its backoff mode, and when it
+   exits it
+ * When a service fails to stop
+
+The failureRate, failureThreshold, and failureBackoff controls how failures
+are handled, in order to avoid the supervisor failure case where the
+program does nothing but restarting failed services. If you do not
+care how failures behave, the default values should be fine for the
+vast majority of services, but if you want the details:
+
+The supervisor tracks the number of failures that have occurred, with an
+exponential decay on the count. Every FailureDecay seconds, the number of
+failures that have occurred is cut in half. (This is done smoothly with an
+exponential function.) When a failure occurs, the number of failures
+is incremented by one. When the number of failures passes the
+FailureThreshold, the entire service waits for FailureBackoff seconds
+before attempting any further restarts, at which point it resets its
+failure count to zero.
+
+Timeout is how long Suture will wait for a service to properly terminate.
+
+*/
+func New(name string, spec Spec) (s *Supervisor) {
+	s = new(Supervisor)
+
+	s.Name = name
+	s.id = supervisorID(atomic.AddUint32(&currentSupervisorID, 1))
+
+	if spec.Log == nil {
+		s.log = func(msg string) {
+			log.Print(fmt.Sprintf("Supervisor %s: %s", s.Name, msg))
+		}
+	} else {
+		s.log = spec.Log
+	}
+
+	if spec.FailureDecay == 0 {
+		s.failureDecay = 30
+	} else {
+		s.failureDecay = spec.FailureDecay
+	}
+	if spec.FailureThreshold == 0 {
+		s.failureThreshold = 5
+	} else {
+		s.failureThreshold = spec.FailureThreshold
+	}
+	if spec.FailureBackoff == 0 {
+		s.failureBackoff = time.Second * 15
+	} else {
+		s.failureBackoff = spec.FailureBackoff
+	}
+	if spec.Timeout == 0 {
+		s.timeout = time.Second * 10
+	} else {
+		s.timeout = spec.Timeout
+	}
+
+	// overriding these allows for testing the threshold behavior
+	s.getNow = time.Now
+	s.getResume = time.After
+
+	s.control = make(chan supervisorMessage)
+	s.services = make(map[serviceID]Service)
+	s.restartQueue = make([]serviceID, 0, 1)
+	s.resumeTimer = make(chan time.Time)
+
+	// set up the default logging handlers
+	s.logBadStop = func(service Service) {
+		s.log(fmt.Sprintf("Service %s failed to terminate in a timely manner", serviceName(service)))
+	}
+	s.logFailure = func(service Service, failures float64, threshold float64, restarting bool, err interface{}, st []byte) {
+		var errString string
+
+		e, canError := err.(error)
+		if canError {
+			errString = e.Error()
+		} else {
+			errString = fmt.Sprintf("%#v", err)
+		}
+
+		s.log(fmt.Sprintf("Failed service '%s' (%f failures of %f), restarting: %#v, error: %s, stacktrace: %s", serviceName(service), failures, threshold, restarting, errString, string(st)))
+	}
+	s.logBackoff = func(s *Supervisor, entering bool) {
+		if entering {
+			s.log("Entering the backoff state.")
+		} else {
+			s.log("Exiting backoff state.")
+		}
+	}
+
+	return
+}
+
+func serviceName(service Service) (serviceName string) {
+	stringer, canStringer := service.(fmt.Stringer)
+	if canStringer {
+		serviceName = stringer.String()
+	} else {
+		serviceName = fmt.Sprintf("%#v", service)
+	}
+	return
+}
+
+// NewSimple is a convenience function to create a service with just a name
+// and the sensible defaults.
+func NewSimple(name string) *Supervisor {
+	return New(name, Spec{})
+}
+
+/*
+Service is the interface that describes a service to a Supervisor.
+
+Serve Method
+
+The Serve method is called by a Supervisor to start the service.
+The service should execute within the goroutine that this is
+called in. If this function either returns or panics, the Supervisor
+will call it again.
+
+A Serve method SHOULD do as much cleanup of the state as possible,
+to prevent any corruption in the previous state from crashing the
+service again.
+
+Stop Method
+
+This method is used by the supervisor to stop the service. Calling this
+directly on a Service given to a Supervisor will simply result in the
+Service being restarted; use the Supervisor's .Remove(ServiceToken) method
+to stop a service. A supervisor will call .Stop() only once. Thus, it may
+be as destructive as it likes to get the service to stop.
+
+Once Stop has been called on a Service, the Service SHOULD NOT be
+reused in any other supervisor! Because of the impossibility of
+guaranteeing that the service has actually stopped in Go, you can't
+prove that you won't be starting two goroutines using the exact
+same memory to store state, causing completely unpredictable behavior.
+
+Stop should not return until the service has actually stopped.
+"Stopped" here is defined as "the service will stop servicing any
+further requests in the future". For instance, a common implementation
+is to receive a message on a dedicated "stop" channel and immediately
+returning. Once the stop command has been processed, the service is
+stopped.
+
+Another common Stop implementation is to forcibly close an open socket
+or other resource, which will cause detectable errors to manifest in the
+service code. Bear in mind that to perfectly correctly use this
+approach requires a bit more work to handle the chance of a Stop
+command coming in before the resource has been created.
+
+If a service does not Stop within the supervisor's timeout duration, a log
+entry will be made with a descriptive string to that effect. This does
+not guarantee that the service is hung; it may still get around to being
+properly stopped in the future. Until the service is fully stopped,
+both the service and the spawned goroutine trying to stop it will be
+"leaked".
+
+Stringer Interface
+
+It is not mandatory to implement the fmt.Stringer interface on your
+service, but if your Service does happen to implement that, the log
+messages that describe your service will use that when naming the
+service. Otherwise, you'll see the GoString of your service object,
+obtained via fmt.Sprintf("%#v", service).
+
+*/
+type Service interface {
+	Serve()
+	Stop()
+}
+
+/*
+Add adds a service to this supervisor.
+
+If the supervisor is currently running, the service will be started
+immediately. If the supervisor is not currently running, the service
+will be started when the supervisor is.
+
+The returned ServiceID may be passed to the Remove method of the Supervisor
+to terminate the service.
+*/
+func (s *Supervisor) Add(service Service) ServiceToken {
+	if s == nil {
+		panic("can't add service to nil *suture.Supervisor")
+	}
+
+	if s.state == notRunning {
+		id := s.serviceCounter
+		s.serviceCounter++
+
+		s.services[id] = service
+		s.restartQueue = append(s.restartQueue, id)
+
+		return ServiceToken{uint64(s.id)<<32 | uint64(id)}
+	}
+
+	response := make(chan serviceID)
+	s.control <- addService{service, response}
+	return ServiceToken{uint64(s.id)<<32 | uint64(<-response)}
+}
+
+// ServeBackground starts running a supervisor in its own goroutine. This
+// method does not return until it is safe to use .Add() on the Supervisor.
+func (s *Supervisor) ServeBackground() {
+	go s.Serve()
+	s.sync()
+}
+
+/*
+Serve starts the supervisor. You should call this on the top-level supervisor,
+but nothing else.
+*/
+func (s *Supervisor) Serve() {
+	if s == nil {
+		panic("Can't serve with a nil *suture.Supervisor")
+	}
+	if s.id == 0 {
+		panic("Can't call Serve on an incorrectly-constructed *suture.Supervisor")
+	}
+
+	defer func() {
+		s.state = notRunning
+	}()
+
+	if s.state != notRunning {
+		// FIXME: Don't explain why I don't need a semaphore, just use one
+		// This doesn't use a semaphore because it's just a sanity check.
+		panic("Running a supervisor while it is already running?")
+	}
+
+	s.state = normal
+
+	// for all the services I currently know about, start them
+	for _, id := range s.restartQueue {
+		service, present := s.services[id]
+		if present {
+			s.runService(service, id)
+		}
+	}
+	s.restartQueue = make([]serviceID, 0, 1)
+
+	for {
+		select {
+		case m := <-s.control:
+			switch msg := m.(type) {
+			case serviceFailed:
+				s.handleFailedService(msg.id, msg.err, msg.stacktrace)
+			case serviceEnded:
+				service, monitored := s.services[msg.id]
+				if monitored {
+					s.handleFailedService(msg.id, fmt.Sprintf("%s returned unexpectedly", service), []byte("[unknown stack trace]"))
+				}
+			case addService:
+				id := s.serviceCounter
+				s.serviceCounter++
+
+				s.services[id] = msg.service
+				s.runService(msg.service, id)
+
+				msg.response <- id
+			case removeService:
+				s.removeService(msg.id)
+			case stopSupervisor:
+				for id := range s.services {
+					s.removeService(id)
+				}
+				return
+			case listServices:
+				services := []Service{}
+				for _, service := range s.services {
+					services = append(services, service)
+				}
+				msg.c <- services
+			case syncSupervisor:
+				// this does nothing on purpose; its sole purpose is to
+				// introduce a sync point via the channel receive
+			case panicSupervisor:
+				// used only by tests
+				panic("Panicking as requested!")
+			}
+		case _ = <-s.resumeTimer:
+			// We're resuming normal operation after a pause due to
+			// excessive thrashing
+			// FIXME: Ought to permit some spacing of these functions, rather
+			// than simply hammering through them
+			s.state = normal
+			s.failures = 0
+			s.logBackoff(s, false)
+			for _, id := range s.restartQueue {
+				service, present := s.services[id]
+				if present {
+					s.runService(service, id)
+				}
+			}
+			s.restartQueue = make([]serviceID, 0, 1)
+		}
+	}
+}
+
+func (s *Supervisor) handleFailedService(id serviceID, err interface{}, stacktrace []byte) {
+	now := s.getNow()
+
+	if s.lastFail.IsZero() {
+		s.lastFail = now
+		s.failures = 1.0
+	} else {
+		sinceLastFail := now.Sub(s.lastFail).Seconds()
+		intervals := sinceLastFail / s.failureDecay
+		s.failures = s.failures*math.Pow(.5, intervals) + 1
+	}
+
+	if s.failures > s.failureThreshold {
+		s.state = paused
+		s.logBackoff(s, true)
+		s.resumeTimer = s.getResume(s.failureBackoff)
+	}
+
+	s.lastFail = now
+
+	failedService, monitored := s.services[id]
+
+	// It is possible for a service to be no longer monitored
+	// by the time we get here. In that case, just ignore it.
+	if monitored {
+		if s.state == normal {
+			s.runService(failedService, id)
+			s.logFailure(failedService, s.failures, s.failureThreshold, true, err, stacktrace)
+		} else {
+			// FIXME: When restarting, check that the service still
+			// exists (it may have been stopped in the meantime)
+			s.restartQueue = append(s.restartQueue, id)
+			s.logFailure(failedService, s.failures, s.failureThreshold, false, err, stacktrace)
+		}
+	}
+}
+
+func (s *Supervisor) runService(service Service, id serviceID) {
+	go func() {
+		defer func() {
+			if r := recover(); r != nil {
+				buf := make([]byte, 65535, 65535)
+				written := runtime.Stack(buf, false)
+				buf = buf[:written]
+				s.fail(id, r, buf)
+			}
+		}()
+
+		service.Serve()
+
+		s.serviceEnded(id)
+	}()
+}
+
+func (s *Supervisor) removeService(id serviceID) {
+	service, present := s.services[id]
+	if present {
+		delete(s.services, id)
+		go func() {
+			successChan := make(chan bool)
+			go func() {
+				service.Stop()
+				successChan <- true
+			}()
+
+			failChan := s.getResume(s.timeout)
+
+			select {
+			case <-successChan:
+				// Life is good!
+			case <-failChan:
+				s.logBadStop(service)
+			}
+		}()
+	}
+}
+
+// String implements the fmt.Stringer interface.
+func (s *Supervisor) String() string {
+	return s.Name
+}
+
+// sum type pattern for type-safe message passing; see
+// http://www.jerf.org/iri/post/2917
+
+type supervisorMessage interface {
+	isSupervisorMessage()
+}
+
+/*
+Remove will remove the given service from the Supervisor, and attempt to Stop() it.
+The ServiceID token comes from the Add() call.
+*/
+func (s *Supervisor) Remove(id ServiceToken) error {
+	sID := supervisorID(id.id >> 32)
+	if sID != s.id {
+		return ErrWrongSupervisor
+	}
+	s.control <- removeService{serviceID(id.id & 0xffffffff)}
+	return nil
+}
+
+/*
+
+Services returns a []Service containing a snapshot of the services this
+Supervisor is managing.
+
+*/
+func (s *Supervisor) Services() []Service {
+	ls := listServices{make(chan []Service)}
+	s.control <- ls
+	return <-ls.c
+}
+
+type listServices struct {
+	c chan []Service
+}
+
+func (ls listServices) isSupervisorMessage() {}
+
+type removeService struct {
+	id serviceID
+}
+
+func (rs removeService) isSupervisorMessage() {}
+
+func (s *Supervisor) sync() {
+	s.control <- syncSupervisor{}
+}
+
+type syncSupervisor struct {
+}
+
+func (ss syncSupervisor) isSupervisorMessage() {}
+
+func (s *Supervisor) fail(id serviceID, err interface{}, stacktrace []byte) {
+	s.control <- serviceFailed{id, err, stacktrace}
+}
+
+type serviceFailed struct {
+	id         serviceID
+	err        interface{}
+	stacktrace []byte
+}
+
+func (sf serviceFailed) isSupervisorMessage() {}
+
+func (s *Supervisor) serviceEnded(id serviceID) {
+	s.control <- serviceEnded{id}
+}
+
+type serviceEnded struct {
+	id serviceID
+}
+
+func (s serviceEnded) isSupervisorMessage() {}
+
+// added by the Add() method
+type addService struct {
+	service  Service
+	response chan serviceID
+}
+
+func (as addService) isSupervisorMessage() {}
+
+// Stop stops the Supervisor.
+func (s *Supervisor) Stop() {
+	s.control <- stopSupervisor{}
+}
+
+type stopSupervisor struct {
+}
+
+func (ss stopSupervisor) isSupervisorMessage() {}
+
+func (s *Supervisor) panic() {
+	s.control <- panicSupervisor{}
+}
+
+type panicSupervisor struct {
+}
+
+func (ps panicSupervisor) isSupervisorMessage() {}

Godeps/_workspace/src/github.com/thejerf/suture/suture_simple_test.go

@@ -0,0 +1,49 @@
+package suture
+
+import "fmt"
+
+type Incrementor struct {
+	current int
+	next    chan int
+	stop    chan bool
+}
+
+func (i *Incrementor) Stop() {
+	fmt.Println("Stopping the service")
+	i.stop <- true
+}
+
+func (i *Incrementor) Serve() {
+	for {
+		select {
+		case i.next <- i.current:
+			i.current += 1
+		case <-i.stop:
+			// We sync here just to guarantee the output of "Stopping the service",
+			// so this passes the test reliably.
+			// Most services would simply "return" here.
+			i.stop <- true
+			return
+		}
+	}
+}
+
+func ExampleNew_simple() {
+	supervisor := NewSimple("Supervisor")
+	service := &Incrementor{0, make(chan int), make(chan bool)}
+	supervisor.Add(service)
+
+	go supervisor.ServeBackground()
+
+	fmt.Println("Got:", <-service.next)
+	fmt.Println("Got:", <-service.next)
+	supervisor.Stop()
+
+	// We sync here just to guarantee the output of "Stopping the service"
+	<-service.stop
+
+	// Output:
+	// Got: 0
+	// Got: 1
+	// Stopping the service
+}

Godeps/_workspace/src/github.com/thejerf/suture/suture_test.go

@@ -0,0 +1,592 @@
+package suture
+
+import (
+	"errors"
+	"fmt"
+	"reflect"
+	"sync"
+	"testing"
+	"time"
+)
+
+const (
+	Happy = iota
+	Fail
+	Panic
+	Hang
+	UseStopChan
+)
+
+var everMultistarted = false
+
+// Test that supervisors work perfectly when everything is hunky dory.
+func TestTheHappyCase(t *testing.T) {
+	t.Parallel()
+
+	s := NewSimple("A")
+	if s.String() != "A" {
+		t.Fatal("Can't get name from a supervisor")
+	}
+	service := NewService("B")
+
+	s.Add(service)
+
+	go s.Serve()
+
+	<-service.started
+
+	// If we stop the service, it just gets restarted
+	service.Stop()
+	<-service.started
+
+	// And it is shut down when we stop the supervisor
+	service.take <- UseStopChan
+	s.Stop()
+	<-service.stop
+}
+
+// Test that adding to a running supervisor does indeed start the service.
+func TestAddingToRunningSupervisor(t *testing.T) {
+	t.Parallel()
+
+	s := NewSimple("A1")
+
+	s.ServeBackground()
+	defer s.Stop()
+
+	service := NewService("B1")
+	s.Add(service)
+
+	<-service.started
+
+	services := s.Services()
+	if !reflect.DeepEqual([]Service{service}, services) {
+		t.Fatal("Can't get list of services as expected.")
+	}
+}
+
+// Test what happens when services fail.
+func TestFailures(t *testing.T) {
+	t.Parallel()
+
+	s := NewSimple("A2")
+	s.failureThreshold = 3.5
+
+	go s.Serve()
+	defer func() {
+		// to avoid deadlocks during shutdown, we have to not try to send
+		// things out on channels while we're shutting down (this undoes the
+		// logFailure overide about 25 lines down)
+		s.logFailure = func(Service, float64, float64, bool, interface{}, []byte) {}
+		s.Stop()
+	}()
+	s.sync()
+
+	service1 := NewService("B2")
+	service2 := NewService("C2")
+
+	s.Add(service1)
+	<-service1.started
+	s.Add(service2)
+	<-service2.started
+
+	nowFeeder := NewNowFeeder()
+	pastVal := time.Unix(1000000, 0)
+	nowFeeder.appendTimes(pastVal)
+	s.getNow = nowFeeder.getter
+
+	resumeChan := make(chan time.Time)
+	s.getResume = func(d time.Duration) <-chan time.Time {
+		return resumeChan
+	}
+
+	failNotify := make(chan bool)
+	// use this to synchronize on here
+	s.logFailure = func(s Service, cf float64, ft float64, r bool, error interface{}, stacktrace []byte) {
+		failNotify <- r
+	}
+
+	// All that setup was for this: Service1, please return now.
+	service1.take <- Fail
+	restarted := <-failNotify
+	<-service1.started
+
+	if !restarted || s.failures != 1 || s.lastFail != pastVal {
+		t.Fatal("Did not fail in the expected manner")
+	}
+	// Getting past this means the service was restarted.
+	service1.take <- Happy
+
+	// Service2, your turn.
+	service2.take <- Fail
+	nowFeeder.appendTimes(pastVal)
+	restarted = <-failNotify
+	<-service2.started
+	if !restarted || s.failures != 2 || s.lastFail != pastVal {
+		t.Fatal("Did not fail in the expected manner")
+	}
+	// And you're back. (That is, the correct service was restarted.)
+	service2.take <- Happy
+
+	// Now, one failureDecay later, is everything working correctly?
+	oneDecayLater := time.Unix(1000030, 0)
+	nowFeeder.appendTimes(oneDecayLater)
+	service2.take <- Fail
+	restarted = <-failNotify
+	<-service2.started
+	// playing a bit fast and loose here with floating point, but...
+	// we get 2 by taking the current failure value of 2, decaying it
+	// by one interval, which cuts it in half to 1, then adding 1 again,
+	// all of which "should" be precise
+	if !restarted || s.failures != 2 || s.lastFail != oneDecayLater {
+		t.Fatal("Did not decay properly", s.lastFail, oneDecayLater)
+	}
+
+	// For a change of pace, service1 would you be so kind as to panic?
+	nowFeeder.appendTimes(oneDecayLater)
+	service1.take <- Panic
+	restarted = <-failNotify
+	<-service1.started
+	if !restarted || s.failures != 3 || s.lastFail != oneDecayLater {
+		t.Fatal("Did not correctly recover from a panic")
+	}
+
+	nowFeeder.appendTimes(oneDecayLater)
+	backingoff := make(chan bool)
+	s.logBackoff = func(s *Supervisor, backingOff bool) {
+		backingoff <- backingOff
+	}
+
+	// And with this failure, we trigger the backoff code.
+	service1.take <- Fail
+	backoff := <-backingoff
+	restarted = <-failNotify
+
+	if !backoff || restarted || s.failures != 4 {
+		t.Fatal("Broke past the threshold but did not log correctly", s.failures)
+	}
+	if service1.existing != 0 {
+		t.Fatal("service1 still exists according to itself?")
+	}
+
+	// service2 is still running, because we don't shut anything down in a
+	// backoff, we just stop restarting.
+	service2.take <- Happy
+
+	var correct bool
+	timer := time.NewTimer(time.Millisecond * 10)
+	// verify the service has not been restarted
+	// hard to get around race conditions here without simply using a timer...
+	select {
+	case service1.take <- Happy:
+		correct = false
+	case <-timer.C:
+		correct = true
+	}
+	if !correct {
+		t.Fatal("Restarted the service during the backoff interval")
+	}
+
+	// tell the supervisor the restart interval has passed
+	resumeChan <- time.Time{}
+	backoff = <-backingoff
+	<-service1.started
+	s.sync()
+	if s.failures != 0 {
+		t.Fatal("Did not reset failure count after coming back from timeout.")
+	}
+
+	nowFeeder.appendTimes(oneDecayLater)
+	service1.take <- Fail
+	restarted = <-failNotify
+	<-service1.started
+	if !restarted || backoff {
+		t.Fatal("For some reason, got that we were backing off again.", restarted, backoff)
+	}
+}
+
+func TestRunningAlreadyRunning(t *testing.T) {
+	t.Parallel()
+
+	s := NewSimple("A3")
+	go s.Serve()
+	defer s.Stop()
+
+	// ensure the supervisor has made it to its main loop
+	s.sync()
+	var errored bool
+	func() {
+		defer func() {
+			if r := recover(); r != nil {
+				errored = true
+			}
+		}()
+
+		s.Serve()
+	}()
+	if !errored {
+		t.Fatal("Supervisor failed to prevent itself from double-running.")
+	}
+}
+
+func TestFullConstruction(t *testing.T) {
+	t.Parallel()
+
+	s := New("Moo", Spec{
+		Log:              func(string) {},
+		FailureDecay:     1,
+		FailureThreshold: 2,
+		FailureBackoff:   3,
+		Timeout:          time.Second * 29,
+	})
+	if s.String() != "Moo" || s.failureDecay != 1 || s.failureThreshold != 2 || s.failureBackoff != 3 || s.timeout != time.Second*29 {
+		t.Fatal("Full construction failed somehow")
+	}
+}
+
+// This is mostly for coverage testing.
+func TestDefaultLogging(t *testing.T) {
+	t.Parallel()
+
+	s := NewSimple("A4")
+
+	service := NewService("B4")
+	s.Add(service)
+
+	s.failureThreshold = .5
+	s.failureBackoff = time.Millisecond * 25
+	go s.Serve()
+	s.sync()
+
+	<-service.started
+
+	resumeChan := make(chan time.Time)
+	s.getResume = func(d time.Duration) <-chan time.Time {
+		return resumeChan
+	}
+
+	service.take <- UseStopChan
+	service.take <- Fail
+	<-service.stop
+	resumeChan <- time.Time{}
+
+	<-service.started
+
+	service.take <- Happy
+
+	serviceName(&BarelyService{})
+
+	s.logBadStop(service)
+	s.logFailure(service, 1, 1, true, errors.New("test error"), []byte{})
+
+	s.Stop()
+}
+
+func TestNestedSupervisors(t *testing.T) {
+	t.Parallel()
+
+	super1 := NewSimple("Top5")
+	super2 := NewSimple("Nested5")
+	service := NewService("Service5")
+
+	super1.Add(super2)
+	super2.Add(service)
+
+	go super1.Serve()
+	super1.sync()
+
+	<-service.started
+	service.take <- Happy
+
+	super1.Stop()
+}
+
+func TestStoppingSupervisorStopsServices(t *testing.T) {
+	t.Parallel()
+
+	s := NewSimple("Top6")
+	service := NewService("Service 6")
+
+	s.Add(service)
+
+	go s.Serve()
+	s.sync()
+
+	<-service.started
+
+	service.take <- UseStopChan
+
+	s.Stop()
+	<-service.stop
+}
+
+func TestStoppingStillWorksWithHungServices(t *testing.T) {
+	t.Parallel()
+
+	s := NewSimple("Top7")
+	service := NewService("Service WillHang7")
+
+	s.Add(service)
+
+	go s.Serve()
+
+	<-service.started
+
+	service.take <- UseStopChan
+	service.take <- Hang
+
+	resumeChan := make(chan time.Time)
+	s.getResume = func(d time.Duration) <-chan time.Time {
+		return resumeChan
+	}
+	failNotify := make(chan struct{})
+	s.logBadStop = func(s Service) {
+		failNotify <- struct{}{}
+	}
+
+	s.Stop()
+
+	resumeChan <- time.Time{}
+	<-failNotify
+	service.release <- true
+	<-service.stop
+}
+
+func TestRemoveService(t *testing.T) {
+	t.Parallel()
+
+	s := NewSimple("Top")
+	service := NewService("ServiceToRemove8")
+
+	id := s.Add(service)
+
+	go s.Serve()
+
+	<-service.started
+	service.take <- UseStopChan
+
+	err := s.Remove(id)
+	if err != nil {
+		t.Fatal("Removing service somehow failed")
+	}
+	<-service.stop
+
+	err = s.Remove(ServiceToken{1<<36 + 1})
+	if err != ErrWrongSupervisor {
+		t.Fatal("Did not detect that the ServiceToken was wrong")
+	}
+}
+
+func TestFailureToConstruct(t *testing.T) {
+	t.Parallel()
+
+	var s *Supervisor
+
+	panics(func() {
+		s.Serve()
+	})
+
+	s = new(Supervisor)
+	panics(func() {
+		s.Serve()
+	})
+}
+
+func TestFailingSupervisors(t *testing.T) {
+	t.Parallel()
+
+	// This is a bit of a complicated test, so let me explain what
+	// all this is doing:
+	// 1. Set up a top-level supervisor with a hair-trigger backoff.
+	// 2. Add a supervisor to that.
+	// 3. To that supervisor, add a service.
+	// 4. Panic the supervisor in the middle, sending the top-level into
+	//    backoff.
+	// 5. Kill the lower level service too.
+	// 6. Verify that when the top-level service comes out of backoff,
+	//    the service ends up restarted as expected.
+
+	// Ultimately, we can't have more than a best-effort recovery here.
+	// A panic'ed supervisor can't really be trusted to have consistent state,
+	// and without *that*, we can't trust it to do anything sensible with
+	// the children it may have been running. So unlike Erlang, we can't
+	// can't really expect to be able to safely restart them or anything.
+	// Really, the "correct" answer is that the Supervisor must never panic,
+	// but in the event that it does, this verifies that it at least tries
+	// to get on with life.
+
+	// This also tests that if a Supervisor itself panics, and one of its
+	// monitored services goes down in the meantime, that the monitored
+	// service also gets correctly restarted when the supervisor does.
+
+	s1 := NewSimple("Top9")
+	s2 := NewSimple("Nested9")
+	service := NewService("Service9")
+
+	s1.Add(s2)
+	s2.Add(service)
+
+	go s1.Serve()
+	<-service.started
+
+	s1.failureThreshold = .5
+
+	// let us control precisely when s1 comes back
+	resumeChan := make(chan time.Time)
+	s1.getResume = func(d time.Duration) <-chan time.Time {
+		return resumeChan
+	}
+	failNotify := make(chan string)
+	// use this to synchronize on here
+	s1.logFailure = func(s Service, cf float64, ft float64, r bool, error interface{}, stacktrace []byte) {
+		failNotify <- fmt.Sprintf("%s", s)
+	}
+
+	s2.panic()
+
+	failing := <-failNotify
+	// that's enough sync to guarantee this:
+	if failing != "Nested9" || s1.state != paused {
+		t.Fatal("Top-level supervisor did not go into backoff as expected")
+	}
+
+	service.take <- Fail
+
+	resumeChan <- time.Time{}
+	<-service.started
+}
+
+func TestNilSupervisorAdd(t *testing.T) {
+	t.Parallel()
+
+	var s *Supervisor
+
+	defer func() {
+		if r := recover(); r == nil {
+			t.Fatal("did not panic as expected on nil add")
+		}
+	}()
+
+	s.Add(s)
+}
+
+// http://golangtutorials.blogspot.com/2011/10/gotest-unit-testing-and-benchmarking-go.html
+// claims test function are run in the same order as the source file...
+// I'm not sure if this is part of the contract, though. Especially in the
+// face of "t.Parallel()"...
+func TestEverMultistarted(t *testing.T) {
+	if everMultistarted {
+		t.Fatal("Seem to have multistarted a service at some point, bummer.")
+	}
+}
+
+// A test service that can be induced to fail, panic, or hang on demand.
+func NewService(name string) *FailableService {
+	return &FailableService{name, make(chan bool), make(chan int),
+		make(chan bool, 1), make(chan bool), make(chan bool), 0}
+}
+
+type FailableService struct {
+	name     string
+	started  chan bool
+	take     chan int
+	shutdown chan bool
+	release  chan bool
+	stop     chan bool
+	existing int
+}
+
+func (s *FailableService) Serve() {
+	if s.existing != 0 {
+		everMultistarted = true
+		panic("Multi-started the same service! " + s.name)
+	}
+	s.existing += 1
+
+	s.started <- true
+
+	useStopChan := false
+
+	for {
+		select {
+		case val := <-s.take:
+			switch val {
+			case Happy:
+				// Do nothing on purpose. Life is good!
+			case Fail:
+				s.existing -= 1
+				if useStopChan {
+					s.stop <- true
+				}
+				return
+			case Panic:
+				s.existing -= 1
+				panic("Panic!")
+			case Hang:
+				// or more specifically, "hang until I release you"
+				<-s.release
+			case UseStopChan:
+				useStopChan = true
+			}
+		case <-s.shutdown:
+			s.existing -= 1
+			if useStopChan {
+				s.stop <- true
+			}
+			return
+		}
+	}
+}
+
+func (s *FailableService) String() string {
+	return s.name
+}
+
+func (s *FailableService) Stop() {
+	s.shutdown <- true
+}
+
+type NowFeeder struct {
+	values []time.Time
+	getter func() time.Time
+	m      sync.Mutex
+}
+
+// This is used to test serviceName; it's a service without a Stringer.
+type BarelyService struct{}
+
+func (bs *BarelyService) Serve() {}
+func (bs *BarelyService) Stop()  {}
+
+func NewNowFeeder() (nf *NowFeeder) {
+	nf = new(NowFeeder)
+	nf.getter = func() time.Time {
+		nf.m.Lock()
+		defer nf.m.Unlock()
+		if len(nf.values) > 0 {
+			ret := nf.values[0]
+			nf.values = nf.values[1:]
+			return ret
+		}
+		panic("Ran out of values for NowFeeder")
+	}
+	return
+}
+
+func (nf *NowFeeder) appendTimes(t ...time.Time) {
+	nf.m.Lock()
+	defer nf.m.Unlock()
+	nf.values = append(nf.values, t...)
+}
+
+func panics(doesItPanic func()) (panics bool) {
+	defer func() {
+		if r := recover(); r != nil {
+			panics = true
+		}
+	}()
+
+	doesItPanic()
+
+	return
+}

Godeps/_workspace/src/golang.org/x/text/unicode/norm/Makefile

@@ -1,23 +0,0 @@
-# Copyright 2011 The Go Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-maketables: maketables.go triegen.go
-	go build $^
-
-normregtest: normregtest.go
-	go build $^
-
-tables:	maketables
-	./maketables > tables.go
-	gofmt -w tables.go
-
-# Downloads from www.unicode.org, so not part
-# of standard test scripts.
-test: testtables regtest
-
-testtables: maketables
-	./maketables -test > data_test.go && go test -tags=test
-
-regtest: normregtest
-	./normregtest

Godeps/_workspace/src/golang.org/x/text/unicode/norm/maketables.go

@@ -16,20 +16,17 @@ import (
 	"fmt"
 	"io"
 	"log"
-	"net/http"
-	"os"
-	"regexp"
 	"sort"
 	"strconv"
 	"strings"
-	"unicode"
 
+	"golang.org/x/text/internal/gen"
 	"golang.org/x/text/internal/triegen"
 	"golang.org/x/text/internal/ucd"
 )
 
 func main() {
-	flag.Parse()
+	gen.Init()
 	loadUnicodeData()
 	compactCCC()
 	loadCompositionExclusions()
@@ -46,24 +43,18 @@ func main() {
 	}
 }
 
-var url = flag.String("url",
-	"http://www.unicode.org/Public/"+unicode.Version+"/ucd/",
-	"URL of Unicode database directory")
-var tablelist = flag.String("tables",
-	"all",
-	"comma-separated list of which tables to generate; "+
-		"can be 'decomp', 'recomp', 'info' and 'all'")
-var test = flag.Bool("test",
-	false,
-	"test existing tables against DerivedNormalizationProps and generate test data for regression testing")
-var verbose = flag.Bool("verbose",
-	false,
-	"write data to stdout as it is parsed")
-var localFiles = flag.Bool("local",
-	false,
-	"data files have been copied to the current directory; for debugging only")
-
-var logger = log.New(os.Stderr, "", log.Lshortfile)
+var (
+	tablelist = flag.String("tables",
+		"all",
+		"comma-separated list of which tables to generate; "+
+			"can be 'decomp', 'recomp', 'info' and 'all'")
+	test = flag.Bool("test",
+		false,
+		"test existing tables against DerivedNormalizationProps and generate test data for regression testing")
+	verbose = flag.Bool("verbose",
+		false,
+		"write data to stdout as it is parsed")
+)
 
 const MaxChar = 0x10FFFF // anything above this shouldn't exist
 
@@ -189,27 +180,6 @@ func (f FormInfo) String() string {
 
 type Decomposition []rune
 
-func openReader(file string) (input io.ReadCloser) {
-	if *localFiles {
-		f, err := os.Open(file)
-		if err != nil {
-			logger.Fatal(err)
-		}
-		input = f
-	} else {
-		path := *url + file
-		resp, err := http.Get(path)
-		if err != nil {
-			logger.Fatal(err)
-		}
-		if resp.StatusCode != 200 {
-			logger.Fatal("bad GET status for "+file, resp.Status)
-		}
-		input = resp.Body
-	}
-	return
-}
-
 func parseDecomposition(s string, skipfirst bool) (a []rune, err error) {
 	decomp := strings.Split(s, " ")
 	if len(decomp) > 0 && skipfirst {
@@ -226,7 +196,7 @@ func parseDecomposition(s string, skipfirst bool) (a []rune, err error) {
 }
 
 func loadUnicodeData() {
-	f := openReader("UnicodeData.txt")
+	f := gen.OpenUCDFile("UnicodeData.txt")
 	defer f.Close()
 	p := ucd.New(f)
 	for p.Next() {
@@ -242,7 +212,7 @@ func loadUnicodeData() {
 			if len(decmap) > 0 {
 				exp, err = parseDecomposition(decmap, true)
 				if err != nil {
-					logger.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err)
+					log.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err)
 				}
 				isCompat = true
 			}
@@ -261,7 +231,7 @@ func loadUnicodeData() {
 		}
 	}
 	if err := p.Err(); err != nil {
-		logger.Fatal(err)
+		log.Fatal(err)
 	}
 }
 
@@ -296,18 +266,18 @@ func compactCCC() {
 // 0958    # ...
 // See http://unicode.org/reports/tr44/ for full explanation
 func loadCompositionExclusions() {
-	f := openReader("CompositionExclusions.txt")
+	f := gen.OpenUCDFile("CompositionExclusions.txt")
 	defer f.Close()
 	p := ucd.New(f)
 	for p.Next() {
 		c := &chars[p.Rune(0)]
 		if c.excludeInComp {
-			logger.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint)
+			log.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint)
 		}
 		c.excludeInComp = true
 	}
 	if e := p.Err(); e != nil {
-		logger.Fatal(e)
+		log.Fatal(e)
 	}
 }
 
@@ -542,19 +512,19 @@ func computeNonStarterCounts() {
 	}
 }
 
-func printBytes(b []byte, name string) {
-	fmt.Printf("// %s: %d bytes\n", name, len(b))
-	fmt.Printf("var %s = [...]byte {", name)
+func printBytes(w io.Writer, b []byte, name string) {
+	fmt.Fprintf(w, "// %s: %d bytes\n", name, len(b))
+	fmt.Fprintf(w, "var %s = [...]byte {", name)
 	for i, c := range b {
 		switch {
 		case i%64 == 0:
-			fmt.Printf("\n// Bytes %x - %x\n", i, i+63)
+			fmt.Fprintf(w, "\n// Bytes %x - %x\n", i, i+63)
 		case i%8 == 0:
-			fmt.Printf("\n")
+			fmt.Fprintf(w, "\n")
 		}
-		fmt.Printf("0x%.2X, ", c)
+		fmt.Fprintf(w, "0x%.2X, ", c)
 	}
-	fmt.Print("\n}\n\n")
+	fmt.Fprint(w, "\n}\n\n")
 }
 
 // See forminfo.go for format.
@@ -610,13 +580,13 @@ func (m *decompSet) insert(key int, s string) {
 	m[key][s] = true
 }
 
-func printCharInfoTables() int {
+func printCharInfoTables(w io.Writer) int {
 	mkstr := func(r rune, f *FormInfo) (int, string) {
 		d := f.expandedDecomp
 		s := string([]rune(d))
 		if max := 1 << 6; len(s) >= max {
 			const msg = "%U: too many bytes in decomposition: %d >= %d"
-			logger.Fatalf(msg, r, len(s), max)
+			log.Fatalf(msg, r, len(s), max)
 		}
 		head := uint8(len(s))
 		if f.quickCheck[MComposed] != QCYes {
@@ -631,11 +601,11 @@ func printCharInfoTables() int {
 		tccc := ccc(d[len(d)-1])
 		cc := ccc(r)
 		if cc != 0 && lccc == 0 && tccc == 0 {
-			logger.Fatalf("%U: trailing and leading ccc are 0 for non-zero ccc %d", r, cc)
+			log.Fatalf("%U: trailing and leading ccc are 0 for non-zero ccc %d", r, cc)
 		}
 		if tccc < lccc && lccc != 0 {
 			const msg = "%U: lccc (%d) must be <= tcc (%d)"
-			logger.Fatalf(msg, r, lccc, tccc)
+			log.Fatalf(msg, r, lccc, tccc)
 		}
 		index := normalDecomp
 		nTrail := chars[r].nTrailingNonStarters
@@ -652,13 +622,13 @@ func printCharInfoTables() int {
 			if lccc > 0 {
 				s += string([]byte{lccc})
 				if index == firstCCC {
-					logger.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r)
+					log.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r)
 				}
 				index = firstLeadingCCC
 			}
 			if cc != lccc {
 				if cc != 0 {
-					logger.Fatalf("%U: for lccc != ccc, expected ccc to be 0; was %d", r, cc)
+					log.Fatalf("%U: for lccc != ccc, expected ccc to be 0; was %d", r, cc)
 				}
 				index = firstCCCZeroExcept
 			}
@@ -680,7 +650,7 @@ func printCharInfoTables() int {
 				continue
 			}
 			if f.combinesBackward {
-				logger.Fatalf("%U: combinesBackward and decompose", c.codePoint)
+				log.Fatalf("%U: combinesBackward and decompose", c.codePoint)
 			}
 			index, s := mkstr(c.codePoint, &f)
 			decompSet.insert(index, s)
@@ -691,7 +661,7 @@ func printCharInfoTables() int {
 	size := 0
 	positionMap := make(map[string]uint16)
 	decompositions.WriteString("\000")
-	fmt.Println("const (")
+	fmt.Fprintln(w, "const (")
 	for i, m := range decompSet {
 		sa := []string{}
 		for s := range m {
@@ -704,13 +674,13 @@ func printCharInfoTables() int {
 			positionMap[s] = uint16(p)
 		}
 		if cname[i] != "" {
-			fmt.Printf("%s = 0x%X\n", cname[i], decompositions.Len())
+			fmt.Fprintf(w, "%s = 0x%X\n", cname[i], decompositions.Len())
 		}
 	}
-	fmt.Println("maxDecomp = 0x8000")
-	fmt.Println(")")
+	fmt.Fprintln(w, "maxDecomp = 0x8000")
+	fmt.Fprintln(w, ")")
 	b := decompositions.Bytes()
-	printBytes(b, "decomps")
+	printBytes(w, b, "decomps")
 	size += len(b)
 
 	varnames := []string{"nfc", "nfkc"}
@@ -726,7 +696,7 @@ func printCharInfoTables() int {
 				if c.ccc != ccc(d[0]) {
 					// We assume the lead ccc of a decomposition !=0 in this case.
 					if ccc(d[0]) == 0 {
-						logger.Fatalf("Expected leading CCC to be non-zero; ccc is %d", c.ccc)
+						log.Fatalf("Expected leading CCC to be non-zero; ccc is %d", c.ccc)
 					}
 				}
 			} else if c.nLeadingNonStarters > 0 && len(f.expandedDecomp) == 0 && c.ccc == 0 && !f.combinesBackward {
@@ -737,9 +707,9 @@ func printCharInfoTables() int {
 				trie.Insert(c.codePoint, uint64(0x8000|v))
 			}
 		}
-		sz, err := trie.Gen(os.Stdout, triegen.Compact(&normCompacter{name: varnames[i]}))
+		sz, err := trie.Gen(w, triegen.Compact(&normCompacter{name: varnames[i]}))
 		if err != nil {
-			logger.Fatal(err)
+			log.Fatal(err)
 		}
 		size += sz
 	}
@@ -755,30 +725,9 @@ func contains(sa []string, s string) bool {
 	return false
 }
 
-// Extract the version number from the URL.
-func version() string {
-	// From http://www.unicode.org/standard/versions/#Version_Numbering:
-	// for the later Unicode versions, data files are located in
-	// versioned directories.
-	fields := strings.Split(*url, "/")
-	for _, f := range fields {
-		if match, _ := regexp.MatchString(`[0-9]\.[0-9]\.[0-9]`, f); match {
-			return f
-		}
-	}
-	logger.Fatal("unknown version")
-	return "Unknown"
-}
-
-const fileHeader = `// Generated by running
-//	maketables --tables=%s --url=%s
-// DO NOT EDIT
-
-package norm
-
-`
-
 func makeTables() {
+	w := &bytes.Buffer{}
+
 	size := 0
 	if *tablelist == "" {
 		return
@@ -787,7 +736,6 @@ func makeTables() {
 	if *tablelist == "all" {
 		list = []string{"recomp", "info"}
 	}
-	fmt.Printf(fileHeader, *tablelist, *url)
 
 	// Compute maximum decomposition size.
 	max := 0
@@ -797,30 +745,30 @@ func makeTables() {
 		}
 	}
 
-	fmt.Println("const (")
-	fmt.Println("\t// Version is the Unicode edition from which the tables are derived.")
-	fmt.Printf("\tVersion = %q\n", version())
-	fmt.Println()
-	fmt.Println("\t// MaxTransformChunkSize indicates the maximum number of bytes that Transform")
-	fmt.Println("\t// may need to write atomically for any Form. Making a destination buffer at")
-	fmt.Println("\t// least this size ensures that Transform can always make progress and that")
-	fmt.Println("\t// the user does not need to grow the buffer on an ErrShortDst.")
-	fmt.Printf("\tMaxTransformChunkSize = %d+maxNonStarters*4\n", len(string(0x034F))+max)
-	fmt.Println(")\n")
+	fmt.Fprintln(w, "const (")
+	fmt.Fprintln(w, "\t// Version is the Unicode edition from which the tables are derived.")
+	fmt.Fprintf(w, "\tVersion = %q\n", gen.UnicodeVersion())
+	fmt.Fprintln(w)
+	fmt.Fprintln(w, "\t// MaxTransformChunkSize indicates the maximum number of bytes that Transform")
+	fmt.Fprintln(w, "\t// may need to write atomically for any Form. Making a destination buffer at")
+	fmt.Fprintln(w, "\t// least this size ensures that Transform can always make progress and that")
+	fmt.Fprintln(w, "\t// the user does not need to grow the buffer on an ErrShortDst.")
+	fmt.Fprintf(w, "\tMaxTransformChunkSize = %d+maxNonStarters*4\n", len(string(0x034F))+max)
+	fmt.Fprintln(w, ")\n")
 
 	// Print the CCC remap table.
 	size += len(cccMap)
-	fmt.Printf("var ccc = [%d]uint8{", len(cccMap))
+	fmt.Fprintf(w, "var ccc = [%d]uint8{", len(cccMap))
 	for i := 0; i < len(cccMap); i++ {
 		if i%8 == 0 {
-			fmt.Println()
+			fmt.Fprintln(w)
 		}
-		fmt.Printf("%3d, ", cccMap[uint8(i)])
+		fmt.Fprintf(w, "%3d, ", cccMap[uint8(i)])
 	}
-	fmt.Println("\n}\n")
+	fmt.Fprintln(w, "\n}\n")
 
 	if contains(list, "info") {
-		size += printCharInfoTables()
+		size += printCharInfoTables(w)
 	}
 
 	if contains(list, "recomp") {
@@ -842,20 +790,21 @@ func makeTables() {
 		}
 		sz := nrentries * 8
 		size += sz
-		fmt.Printf("// recompMap: %d bytes (entries only)\n", sz)
-		fmt.Println("var recompMap = map[uint32]rune{")
+		fmt.Fprintf(w, "// recompMap: %d bytes (entries only)\n", sz)
+		fmt.Fprintln(w, "var recompMap = map[uint32]rune{")
 		for i, c := range chars {
 			f := c.forms[FCanonical]
 			d := f.decomp
 			if !f.isOneWay && len(d) > 0 {
 				key := uint32(uint16(d[0]))<<16 + uint32(uint16(d[1]))
-				fmt.Printf("0x%.8X: 0x%.4X,\n", key, i)
+				fmt.Fprintf(w, "0x%.8X: 0x%.4X,\n", key, i)
 			}
 		}
-		fmt.Printf("}\n\n")
+		fmt.Fprintf(w, "}\n\n")
 	}
 
-	fmt.Printf("// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size)
+	fmt.Fprintf(w, "// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size)
+	gen.WriteGoFile("tables.go", "norm", w.Bytes())
 }
 
 func printChars() {
@@ -901,7 +850,7 @@ func verifyComputed() {
 		nfc := c.forms[FCanonical]
 		nfkc := c.forms[FCompatibility]
 		if nfc.combinesBackward != nfkc.combinesBackward {
-			logger.Fatalf("%U: Cannot combine combinesBackward\n", c.codePoint)
+			log.Fatalf("%U: Cannot combine combinesBackward\n", c.codePoint)
 		}
 	}
 }
@@ -913,7 +862,7 @@ func verifyComputed() {
 // 0374          ; NFD_QC; N # ...
 // See http://unicode.org/reports/tr44/ for full explanation
 func testDerived() {
-	f := openReader("DerivedNormalizationProps.txt")
+	f := gen.OpenUCDFile("DerivedNormalizationProps.txt")
 	defer f.Close()
 	p := ucd.New(f)
 	for p.Next() {
@@ -946,12 +895,12 @@ func testDerived() {
 			log.Fatalf(`Unexpected quick check value "%s"`, p.String(2))
 		}
 		if got := c.forms[ftype].quickCheck[mode]; got != qr {
-			logger.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr)
+			log.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr)
 		}
 		c.forms[ftype].verified[mode] = true
 	}
 	if err := p.Err(); err != nil {
-		logger.Fatal(err)
+		log.Fatal(err)
 	}
 	// Any unspecified value must be QCYes. Verify this.
 	for i, c := range chars {
@@ -959,20 +908,14 @@ func testDerived() {
 			for k, qr := range fd.quickCheck {
 				if !fd.verified[k] && qr != QCYes {
 					m := "%U: FAIL F:%d M:%d (was %v need Yes) %s\n"
-					logger.Printf(m, i, j, k, qr, c.name)
+					log.Printf(m, i, j, k, qr, c.name)
 				}
 			}
 		}
 	}
 }
 
-var testHeader = `// Generated by running
-//   maketables --test --url=%s
-// +build test
-
-package norm
-
-const (
+var testHeader = `const (
 	Yes = iota
 	No
 	Maybe
@@ -1010,8 +953,10 @@ func printTestdata() {
 		nTrail uint8
 		f      string
 	}
+
 	last := lastInfo{}
-	fmt.Printf(testHeader, *url)
+	w := &bytes.Buffer{}
+	fmt.Fprintf(w, testHeader)
 	for r, c := range chars {
 		f := c.forms[FCanonical]
 		qc, cf, d := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp)
@@ -1025,9 +970,10 @@ func printTestdata() {
 		}
 		current := lastInfo{c.ccc, c.nLeadingNonStarters, c.nTrailingNonStarters, s}
 		if last != current {
-			fmt.Printf("\t{0x%x, %d, %d, %d, %s},\n", r, c.origCCC, c.nLeadingNonStarters, c.nTrailingNonStarters, s)
+			fmt.Fprintf(w, "\t{0x%x, %d, %d, %d, %s},\n", r, c.origCCC, c.nLeadingNonStarters, c.nTrailingNonStarters, s)
 			last = current
 		}
 	}
-	fmt.Println("}")
+	fmt.Fprintln(w, "}")
+	gen.WriteGoFile("data_test.go", "norm", w.Bytes())
 }

Godeps/_workspace/src/golang.org/x/text/unicode/norm/normalize.go

@@ -2,6 +2,9 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+//go:generate go run maketables.go triegen.go
+//go:generate go run maketables.go triegen.go -test
+
 // Package norm contains types and functions for normalizing Unicode strings.
 package norm
 

Godeps/_workspace/src/golang.org/x/text/unicode/norm/tables.go

@@ -1,6 +1,4 @@
-// Generated by running
-//	maketables --tables=all --url=http://www.unicode.org/Public/7.0.0/ucd/
-// DO NOT EDIT
+// This file was generated by go generate; DO NOT EDIT
 
 package norm
 

Godeps/_workspace/src/golang.org/x/text/unicode/norm/ucd_test.go

@@ -2,52 +2,37 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build ignore
-
-package main
+package norm
 
 import (
 	"bufio"
 	"bytes"
 	"flag"
 	"fmt"
-	"log"
-	"net/http"
-	"os"
-	"path"
 	"regexp"
 	"runtime"
 	"strconv"
 	"strings"
+	"sync"
+	"testing"
 	"time"
-	"unicode"
 	"unicode/utf8"
 
-	"golang.org/x/text/unicode/norm"
+	"golang.org/x/text/internal/gen"
 )
 
-func main() {
-	flag.Parse()
-	loadTestData()
-	CharacterByCharacterTests()
-	StandardTests()
-	PerformanceTest()
-	if errorCount == 0 {
-		fmt.Println("PASS")
+var long = flag.Bool("long", false,
+	"run time-consuming tests, such as tests that fetch data online")
+
+var once sync.Once
+
+func skipShort(t *testing.T) {
+	if !gen.IsLocal() && !*long {
+		t.Skip("skipping test to prevent downloading; to run use -long or use -local to specify a local source")
 	}
+	once.Do(func() { loadTestData(t) })
 }
 
-const file = "NormalizationTest.txt"
-
-var url = flag.String("url",
-	"http://www.unicode.org/Public/"+unicode.Version+"/ucd/"+file,
-	"URL of Unicode database directory")
-var localFiles = flag.Bool("local",
-	false,
-	"data files have been copied to the current directory; for debugging only")
-
-var logger = log.New(os.Stderr, "", log.Lshortfile)
-
 // This regression test runs the test set in NormalizationTest.txt
 // (taken from http://www.unicode.org/Public/<unicode.Version>/ucd/).
 //
@@ -124,22 +109,8 @@ var testRe = regexp.MustCompile(`^` + strings.Repeat(`([\dA-F ]+);`, 5) + ` # (.
 var counter int
 
 // Load the data form NormalizationTest.txt
-func loadTestData() {
-	if *localFiles {
-		pwd, _ := os.Getwd()
-		*url = "file://" + path.Join(pwd, file)
-	}
-	t := &http.Transport{}
-	t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/")))
-	c := &http.Client{Transport: t}
-	resp, err := c.Get(*url)
-	if err != nil {
-		logger.Fatal(err)
-	}
-	if resp.StatusCode != 200 {
-		logger.Fatal("bad GET status for "+file, resp.Status)
-	}
-	f := resp.Body
+func loadTestData(t *testing.T) {
+	f := gen.OpenUCDFile("NormalizationTest.txt")
 	defer f.Close()
 	scanner := bufio.NewScanner(f)
 	for scanner.Scan() {
@@ -150,11 +121,11 @@ func loadTestData() {
 		m := partRe.FindStringSubmatch(line)
 		if m != nil {
 			if len(m) < 3 {
-				logger.Fatal("Failed to parse Part: ", line)
+				t.Fatal("Failed to parse Part: ", line)
 			}
 			i, err := strconv.Atoi(m[1])
 			if err != nil {
-				logger.Fatal(err)
+				t.Fatal(err)
 			}
 			name := m[2]
 			part = append(part, Part{name: name[:len(name)-1], number: i})
@@ -162,7 +133,7 @@ func loadTestData() {
 		}
 		m = testRe.FindStringSubmatch(line)
 		if m == nil || len(m) < 7 {
-			logger.Fatalf(`Failed to parse: "%s" result: %#v`, line, m)
+			t.Fatalf(`Failed to parse: "%s" result: %#v`, line, m)
 		}
 		test := Test{name: m[6], partnr: len(part) - 1, number: counter}
 		counter++
@@ -170,7 +141,7 @@ func loadTestData() {
 			for _, split := range strings.Split(m[j], " ") {
 				r, err := strconv.ParseUint(split, 16, 64)
 				if err != nil {
-					logger.Fatal(err)
+					t.Fatal(err)
 				}
 				if test.r == 0 {
 					// save for CharacterByCharacterTests
@@ -185,50 +156,38 @@ func loadTestData() {
 		part.tests = append(part.tests, test)
 	}
 	if scanner.Err() != nil {
-		logger.Fatal(scanner.Err())
+		t.Fatal(scanner.Err())
 	}
 }
 
-var fstr = []string{"NFC", "NFD", "NFKC", "NFKD"}
-
-var errorCount int
-
-func cmpResult(t *Test, name string, f norm.Form, gold, test, result string) {
+func cmpResult(t *testing.T, tc *Test, name string, f Form, gold, test, result string) {
 	if gold != result {
-		errorCount++
-		if errorCount > 20 {
-			return
-		}
-		logger.Printf("%s:%s: %s(%+q)=%+q; want %+q: %s",
-			t.Name(), name, fstr[f], test, result, gold, t.name)
+		t.Errorf("%s:%s: %s(%+q)=%+q; want %+q: %s",
+			tc.Name(), name, fstr[f], test, result, gold, tc.name)
 	}
 }
 
-func cmpIsNormal(t *Test, name string, f norm.Form, test string, result, want bool) {
+func cmpIsNormal(t *testing.T, tc *Test, name string, f Form, test string, result, want bool) {
 	if result != want {
-		errorCount++
-		if errorCount > 20 {
-			return
-		}
-		logger.Printf("%s:%s: %s(%+q)=%v; want %v", t.Name(), name, fstr[f], test, result, want)
+		t.Errorf("%s:%s: %s(%+q)=%v; want %v", tc.Name(), name, fstr[f], test, result, want)
 	}
 }
 
-func doTest(t *Test, f norm.Form, gold, test string) {
+func doTest(t *testing.T, tc *Test, f Form, gold, test string) {
 	testb := []byte(test)
 	result := f.Bytes(testb)
-	cmpResult(t, "Bytes", f, gold, test, string(result))
+	cmpResult(t, tc, "Bytes", f, gold, test, string(result))
 
 	sresult := f.String(test)
-	cmpResult(t, "String", f, gold, test, sresult)
+	cmpResult(t, tc, "String", f, gold, test, sresult)
 
 	acc := []byte{}
-	i := norm.Iter{}
+	i := Iter{}
 	i.InitString(f, test)
 	for !i.Done() {
 		acc = append(acc, i.Next()...)
 	}
-	cmpResult(t, "Iter.Next", f, gold, test, string(acc))
+	cmpResult(t, tc, "Iter.Next", f, gold, test, string(acc))
 
 	buf := make([]byte, 128)
 	acc = nil
@@ -237,32 +196,33 @@ func doTest(t *Test, f norm.Form, gold, test string) {
 		acc = append(acc, buf[:nDst]...)
 		p += nSrc
 	}
-	cmpResult(t, "Transform", f, gold, test, string(acc))
+	cmpResult(t, tc, "Transform", f, gold, test, string(acc))
 
 	for i := range test {
 		out := f.Append(f.Bytes([]byte(test[:i])), []byte(test[i:])...)
-		cmpResult(t, fmt.Sprintf(":Append:%d", i), f, gold, test, string(out))
+		cmpResult(t, tc, fmt.Sprintf(":Append:%d", i), f, gold, test, string(out))
 	}
-	cmpIsNormal(t, "IsNormal", f, test, f.IsNormal([]byte(test)), test == gold)
-	cmpIsNormal(t, "IsNormalString", f, test, f.IsNormalString(test), test == gold)
+	cmpIsNormal(t, tc, "IsNormal", f, test, f.IsNormal([]byte(test)), test == gold)
+	cmpIsNormal(t, tc, "IsNormalString", f, test, f.IsNormalString(test), test == gold)
 }
 
-func doConformanceTests(t *Test, partn int) {
+func doConformanceTests(t *testing.T, tc *Test, partn int) {
 	for i := 0; i <= 2; i++ {
-		doTest(t, norm.NFC, t.cols[1], t.cols[i])
-		doTest(t, norm.NFD, t.cols[2], t.cols[i])
-		doTest(t, norm.NFKC, t.cols[3], t.cols[i])
-		doTest(t, norm.NFKD, t.cols[4], t.cols[i])
+		doTest(t, tc, NFC, tc.cols[1], tc.cols[i])
+		doTest(t, tc, NFD, tc.cols[2], tc.cols[i])
+		doTest(t, tc, NFKC, tc.cols[3], tc.cols[i])
+		doTest(t, tc, NFKD, tc.cols[4], tc.cols[i])
 	}
 	for i := 3; i <= 4; i++ {
-		doTest(t, norm.NFC, t.cols[3], t.cols[i])
-		doTest(t, norm.NFD, t.cols[4], t.cols[i])
-		doTest(t, norm.NFKC, t.cols[3], t.cols[i])
-		doTest(t, norm.NFKD, t.cols[4], t.cols[i])
+		doTest(t, tc, NFC, tc.cols[3], tc.cols[i])
+		doTest(t, tc, NFD, tc.cols[4], tc.cols[i])
+		doTest(t, tc, NFKC, tc.cols[3], tc.cols[i])
+		doTest(t, tc, NFKD, tc.cols[4], tc.cols[i])
 	}
 }
 
-func CharacterByCharacterTests() {
+func TestCharacterByCharacter(t *testing.T) {
+	skipShort(t)
 	tests := part[1].tests
 	var last rune = 0
 	for i := 0; i <= len(tests); i++ { // last one is special case
@@ -274,37 +234,39 @@ func CharacterByCharacterTests() {
 		}
 		for last++; last < r; last++ {
 			// Check all characters that were not explicitly listed in the test.
-			t := &Test{partnr: 1, number: -1}
+			tc := &Test{partnr: 1, number: -1}
 			char := string(last)
-			doTest(t, norm.NFC, char, char)
-			doTest(t, norm.NFD, char, char)
-			doTest(t, norm.NFKC, char, char)
-			doTest(t, norm.NFKD, char, char)
+			doTest(t, tc, NFC, char, char)
+			doTest(t, tc, NFD, char, char)
+			doTest(t, tc, NFKC, char, char)
+			doTest(t, tc, NFKD, char, char)
 		}
 		if i < len(tests) {
-			doConformanceTests(&tests[i], 1)
+			doConformanceTests(t, &tests[i], 1)
 		}
 	}
 }
 
-func StandardTests() {
+func TestStandardTests(t *testing.T) {
+	skipShort(t)
 	for _, j := range []int{0, 2, 3} {
 		for _, test := range part[j].tests {
-			doConformanceTests(&test, j)
+			doConformanceTests(t, &test, j)
 		}
 	}
 }
 
-// PerformanceTest verifies that normalization is O(n). If any of the
+// TestPerformance verifies that normalization is O(n). If any of the
 // code does not properly check for maxCombiningChars, normalization
 // may exhibit O(n**2) behavior.
-func PerformanceTest() {
+func TestPerformance(t *testing.T) {
+	skipShort(t)
 	runtime.GOMAXPROCS(2)
 	success := make(chan bool, 1)
 	go func() {
 		buf := bytes.Repeat([]byte("\u035D"), 1024*1024)
 		buf = append(buf, "\u035B"...)
-		norm.NFC.Append(nil, buf...)
+		NFC.Append(nil, buf...)
 		success <- true
 	}()
 	timeout := time.After(1 * time.Second)
@@ -312,7 +274,6 @@ func PerformanceTest() {
 	case <-success:
 		// test completed before the timeout
 	case <-timeout:
-		errorCount++
-		logger.Printf(`unexpectedly long time to complete PerformanceTest`)
+		t.Errorf(`unexpectedly long time to complete PerformanceTest`)
 	}
 }

NICKS

@@ -3,6 +3,8 @@
 AudriusButkevicius	<audrius.butkevicius@gmail.com>
 Cathryne	<cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
 KayoticSully	<kayoticsully@gmail.com>
+LordLandon	<lordlandon@gmail.com>
+Moter8		<moter8@gmail.com>
 Nutomic		<me@nutomic.com>
 Rewt0r		<rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
 Vilbrekin	<vilbrekin@gmail.com>
@@ -10,31 +12,41 @@ Zillode		<zillode@zillode.be>
 alex2108	<register-github@alex-graf.de>
 andrew-d	<andrew@du.nham.ca>
 asdil12		<dominik@heidler.eu>
+bencurthoys	<ben@bencurthoys.com>
 bigbear2nd	<bigbear2nd@gmail.com>
 brendanlong	<self@brendanlong.com>
 bsidhom		<bsidhom@gmail.com>
 calmh		<jakob@nym.se>
 cdata		<chris@scriptolo.gy>
+cdhowie	<me@chrishowie.com>
 ceh		<emil@hessman.se>
 cqcallaw	<enlightened.despot@gmail.com>
+dzarda		<dzardacz@gmail.com>
 facastagnini	<federico.castagnini@gmail.com>
 filoozoom	<philippe@schommers.be>
 frioux		<frew@afoolishmanifesto.com> <frioux@gmail.com>
 gillisig	<gilli@vx.is>
+jarlebring	<jarlebring@gmail.com>
 jedie		<github.com@jensdiemer.de> <git@jensdiemer.de>
 jpjp		<jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
+kamadak		<kamada@nanohz.org>
+kilburn		<kilburn@la3.org>
 kozec		<kozec@kozec.com>
 krozycki	<rozycki.karol@gmail.com>
 marcindziadus	<dziadus.marcin@gmail.com>
 marclaporte <marc@marclaporte.com>
+moshen		<moshen.colin@gmail.com>
 mvdan		<mvdan@mvdan.cc>
+pascalj		<github@pascalj.com> <mail@pascal-jungblut.com>
 peterhoeg	<peter@speartail.com>
 philips		<brandon@ifup.org>
 piobpl		<piotrb10@gmail.com>
 pluby		<phill.luby@newredo.com>
 pyfisch		<pyfisch@gmail.com>
+ralder		<ralder@yandex.ru>
 rumpelsepp	<stefan@sevenbyte.org>
 qbit		<qbit@deftly.net>
+sciurius	<jvromans@squirrel.nl>
 seehuhn		<voss@seehuhn.de>
 snnd		<dw@risu.io>
 timabell	<tim@timwise.co.uk>
@@ -42,3 +54,4 @@ tnn2		<tnn@nygren.pp.se>
 tojrobinson	<tully@tojr.org>
 uok		<ueomkail@gmail.com> <uok@users.noreply.github.com>
 veeti		<veeti.paananen@rojekti.fi>
+zukoo		<fxgsell@gmail.com>

README.md

@@ -3,20 +3,19 @@ syncthing
 
 [![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing.svg?style=flat-square)](http://build.syncthing.net/job/syncthing/lastBuild/)
 [![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/syncthing)
-[![GPL License](http://img.shields.io/badge/license-GPL-blue.svg?style=flat-square)](http://opensource.org/licenses/GPL-3.0)
+[![MPLv2 License](http://img.shields.io/badge/license-MPLv2-blue.svg?style=flat-square)](https://www.mozilla.org/MPL/2.0/)
 
-This is the `syncthing` project. The following are the project goals:
+This is the `syncthing` project which pursues the following goals:
 
  1. Define a protocol for synchronization of a folder between a number of
-    collaborating devices. The protocol should be well defined, unambiguous,
+    collaborating devices. This protocol should be well defined, unambiguous,
     easily understood, free to use, efficient, secure and language neutral.
-    This is the [Block Exchange
+    This is called the [Block Exchange
     Protocol](https://github.com/syncthing/specs/blob/master/BEPv1.md).
 
  2. Provide the reference implementation to demonstrate the usability of
-    said protocol. This is the `syncthing` utility. It is the hope that
-    alternative, compatible implementations of the protocol will come to
-    exist.
+    said protocol. This is the `syncthing` utility. We hope that
+    alternative, compatible implementations of the protocol will arise.
 
 The two are evolving together; the protocol is not to be considered
 stable until syncthing 1.0 is released, at which point it is locked down
@@ -32,20 +31,20 @@ There are a few examples for keeping syncthing running in the background
 on your system in [the etc directory](https://github.com/syncthing/syncthing/blob/master/etc).
 
 There is an IRC channel, `#syncthing` on Freenode, for talking directly
-to developers and users (when awake and present, etc.).
+to developers and users.
 
 Building
 --------
 
 Building Syncthing from source is easy, and there's a
 [guide](https://github.com/syncthing/syncthing/wiki/Building).
-that describes it for both Unix and Windows.
+that describes it for both Unix and Windows systems.
 
 Signed Releases
 ---------------
 
 As of v0.10.15 and onwards, git tags and release binaries are GPG signed
-with the key D26E6ED000654A3E (see http://syncthing.net/security.html).
+with the key D26E6ED000654A3E (see https://syncthing.net/security.html).
 For release binaries, MD5 and SHA1 checksums are calculated and signed,
 available in the md5sum.txt.asc and sha1sum.txt.asc files.
 
@@ -57,5 +56,4 @@ documentation](https://github.com/syncthing/syncthing/wiki/) is on the
 Github wiki.
 
 All code is licensed under the
-[GPL](https://github.com/syncthing/syncthing/blob/master/LICENSE), v3 or
-later.
+[MPLv2 License](https://github.com/syncthing/syncthing/blob/master/LICENSE).

authors.go

@@ -0,0 +1,58 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build ignore
+
+// Generates the list of contributors in gui/index.html based on contents of
+// AUTHORS.
+
+package main
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+func main() {
+	bs := readAll("AUTHORS")
+	lines := strings.Split(string(bs), "\n")
+	nameRe := regexp.MustCompile(`(.+?)\s+<`)
+	authors := make([]string, 0, len(lines))
+	for _, line := range lines {
+		if m := nameRe.FindStringSubmatch(line); len(m) == 2 {
+			authors = append(authors, "          <li class=\"auto-generated\">"+m[1]+"</li>")
+		}
+	}
+	sort.Strings(authors)
+	replacement := strings.Join(authors, "\n")
+
+	authorsRe := regexp.MustCompile(`(?s)id="contributor-list">.*?</ul>`)
+	bs = readAll("gui/index.html")
+	bs = authorsRe.ReplaceAll(bs, []byte("id=\"contributor-list\">\n"+replacement+"\n        </ul>"))
+
+	if err := ioutil.WriteFile("gui/index.html", bs, 0644); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func readAll(path string) []byte {
+	fd, err := os.Open(path)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer fd.Close()
+
+	bs, err := ioutil.ReadAll(fd)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return bs
+}

benchfilter.go

@@ -0,0 +1,50 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build ignore
+
+// Neatly format benchmarking output which otherwise looks like crap.
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"os"
+	"regexp"
+	"text/tabwriter"
+)
+
+var (
+	benchRe   = regexp.MustCompile(`^Bench`)
+	spacesRe  = regexp.MustCompile(`\s+`)
+	numbersRe = regexp.MustCompile(`\b[\d\.]+\b`)
+)
+
+func main() {
+	tw := tabwriter.NewWriter(os.Stdout, 1, 1, 1, ' ', 0)
+	br := bufio.NewScanner(os.Stdin)
+	n := 0
+
+	for br.Scan() {
+		line := br.Bytes()
+
+		if benchRe.Match(line) {
+			n++
+			line = spacesRe.ReplaceAllLiteral(line, []byte("\t"))
+			line = numbersRe.ReplaceAllFunc(line, func(n []byte) []byte {
+				return []byte(fmt.Sprintf("%12s", n))
+			})
+			tw.Write(line)
+			tw.Write([]byte("\n"))
+		} else if n > 0 && bytes.HasPrefix(line, []byte("ok")) {
+			n = 0
+			tw.Flush()
+			fmt.Printf("%s\n\n", line)
+		}
+	}
+	tw.Flush()
+}

build.go

@@ -1,17 +1,8 @@
 // Copyright (C) 2014 The Syncthing Authors.
 //
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
 
 // +build ignore
 
@@ -87,6 +78,11 @@ func main() {
 			tags = []string{"noupgrade"}
 		}
 		install("./cmd/...", tags)
+
+		vet("./cmd/syncthing")
+		vet("./internal/...")
+		lint("./cmd/syncthing")
+		lint("./internal/...")
 		return
 	}
 
@@ -112,8 +108,10 @@ func main() {
 			build(pkg, tags)
 
 		case "test":
-			pkg := "./..."
-			test(pkg)
+			test("./...")
+
+		case "bench":
+			bench("./...")
 
 		case "assets":
 			assets()
@@ -136,9 +134,20 @@ func main() {
 		case "zip":
 			buildZip()
 
+		case "deb":
+			buildDeb()
+
 		case "clean":
 			clean()
 
+		case "vet":
+			vet("./cmd/syncthing")
+			vet("./internal/...")
+
+		case "lint":
+			lint("./cmd/syncthing")
+			lint("./internal/...")
+
 		default:
 			log.Fatalf("Unknown command %q", cmd)
 		}
@@ -176,6 +185,11 @@ func test(pkg string) {
 	runPrint("go", "test", "-short", "-timeout", "60s", pkg)
 }
 
+func bench(pkg string) {
+	setBuildEnv()
+	runPrint("go", "test", "-run", "NONE", "-bench", ".", pkg)
+}
+
 func install(pkg string, tags []string) {
 	os.Setenv("GOBIN", "./bin")
 	args := []string{"install", "-v", "-ldflags", ldflags()}
@@ -269,6 +283,85 @@ func buildZip() {
 	log.Println(filename)
 }
 
+func buildDeb() {
+	os.RemoveAll("deb")
+
+	// "goarch" here is set to whatever the Debian packages expect. We correct
+	// "it to what we actually know how to build and keep the Debian variant
+	// "name in "debarch".
+	debarch := goarch
+	switch goarch {
+	case "i386":
+		goarch = "386"
+	case "armel", "armhf":
+		goarch = "arm"
+	}
+
+	build("./cmd/syncthing", []string{"noupgrade"})
+
+	files := []archiveFile{
+		{src: "README.md", dst: "deb/usr/share/doc/syncthing/README.txt", perm: 0644},
+		{src: "LICENSE", dst: "deb/usr/share/doc/syncthing/LICENSE.txt", perm: 0644},
+		{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing/AUTHORS.txt", perm: 0644},
+		{src: "syncthing", dst: "deb/usr/bin/syncthing", perm: 0755},
+	}
+
+	for _, file := range listFiles("extra") {
+		files = append(files, archiveFile{src: file, dst: "deb/usr/share/doc/syncthing/" + filepath.Base(file), perm: 0644})
+	}
+
+	for _, af := range files {
+		if err := copyFile(af.src, af.dst, af.perm); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	control := `Package: syncthing
+Architecture: {{arch}}
+Depends: libc6
+Version: {{version}}
+Maintainer: Syncthing Release Management <release@syncthing.net>
+Description: Open Source Continuous File Synchronization
+	Syncthing does bidirectional synchronization of files between two or
+	more computers.
+`
+	changelog := `syncthing ({{version}}); urgency=medium
+
+  * Packaging of {{version}}.
+
+ -- Jakob Borg <jakob@nym.se>  {{date}}
+`
+
+	control = strings.Replace(control, "{{arch}}", debarch, -1)
+	control = strings.Replace(control, "{{version}}", version[1:], -1)
+	changelog = strings.Replace(changelog, "{{arch}}", debarch, -1)
+	changelog = strings.Replace(changelog, "{{version}}", version[1:], -1)
+	changelog = strings.Replace(changelog, "{{date}}", time.Now().Format(time.RFC1123), -1)
+
+	os.MkdirAll("deb/DEBIAN", 0755)
+	ioutil.WriteFile("deb/DEBIAN/control", []byte(control), 0644)
+	ioutil.WriteFile("deb/DEBIAN/compat", []byte("9\n"), 0644)
+	ioutil.WriteFile("deb/DEBIAN/changelog", []byte(changelog), 0644)
+
+}
+
+func copyFile(src, dst string, perm os.FileMode) error {
+	dstDir := filepath.Dir(dst)
+	os.MkdirAll(dstDir, 0755) // ignore error
+	srcFd, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer srcFd.Close()
+	dstFd, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perm)
+	if err != nil {
+		return err
+	}
+	defer dstFd.Close()
+	_, err = io.Copy(dstFd, srcFd)
+	return err
+}
+
 func listFiles(dir string) []string {
 	var res []string
 	filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
@@ -350,16 +443,43 @@ func rmr(paths ...string) {
 	}
 }
 
-func getVersion() string {
+func getReleaseVersion() (string, error) {
+	fd, err := os.Open("RELEASE")
+	if err != nil {
+		return "", err
+	}
+	defer fd.Close()
+
+	bs, err := ioutil.ReadAll(fd)
+	if err != nil {
+		return "", err
+	}
+	return string(bytes.TrimSpace(bs)), nil
+}
+
+func getGitVersion() (string, error) {
 	v, err := runError("git", "describe", "--always", "--dirty")
 	if err != nil {
-		return "unknown-dev"
+		return "", err
 	}
 	v = versionRe.ReplaceAllFunc(v, func(s []byte) []byte {
 		s[0] = '+'
 		return s
 	})
-	return string(v)
+	return string(v), nil
+}
+
+func getVersion() string {
+	// First try for a RELEASE file,
+	if ver, err := getReleaseVersion(); err == nil {
+		return ver
+	}
+	// ... then see if we have a Git tag.
+	if ver, err := getGitVersion(); err == nil {
+		return ver
+	}
+	// This seems to be a dev build.
+	return "unknown-dev"
 }
 
 func buildStamp() int64 {
@@ -419,10 +539,7 @@ func run(cmd string, args ...string) []byte {
 func runError(cmd string, args ...string) ([]byte, error) {
 	ecmd := exec.Command(cmd, args...)
 	bs, err := ecmd.CombinedOutput()
-	if err != nil {
-		return nil, err
-	}
-	return bytes.TrimSpace(bs), nil
+	return bytes.TrimSpace(bs), err
 }
 
 func runPrint(cmd string, args ...string) {
@@ -453,8 +570,9 @@ func runPipe(file, cmd string, args ...string) {
 }
 
 type archiveFile struct {
-	src string
-	dst string
+	src  string
+	dst  string
+	perm os.FileMode
 }
 
 func tarGz(out string, files []archiveFile) {
@@ -597,3 +715,37 @@ func md5File(file string) error {
 
 	return out.Close()
 }
+
+func vet(pkg string) {
+	bs, err := runError("go", "vet", pkg)
+	if err != nil && err.Error() == "exit status 3" || bytes.Contains(bs, []byte("no such tool \"vet\"")) {
+		// Go said there is no go vet
+		log.Println(`- No go vet, no vetting. Try "go get -u golang.org/x/tools/cmd/vet".`)
+		return
+	}
+
+	falseAlarmComposites := regexp.MustCompile("composite literal uses unkeyed fields")
+	exitStatus := regexp.MustCompile("exit status 1")
+	for _, line := range bytes.Split(bs, []byte("\n")) {
+		if falseAlarmComposites.Match(line) || exitStatus.Match(line) {
+			continue
+		}
+		log.Printf("%s", line)
+	}
+}
+
+func lint(pkg string) {
+	bs, err := runError("golint", pkg)
+	if err != nil {
+		log.Println(`- No golint, not linting. Try "go get -u github.com/golang/lint/golint".`)
+		return
+	}
+
+	analCommentPolicy := regexp.MustCompile(`exported (function|method|const|type|var) [^\s]+ should have comment`)
+	for _, line := range bytes.Split(bs, []byte("\n")) {
+		if analCommentPolicy.Match(line) {
+			continue
+		}
+		log.Printf("%s", line)
+	}
+}

build.sh

@@ -17,8 +17,11 @@ case "${1:-default}" in
 		ulimit -t 60 &>/dev/null || true
 		ulimit -d 512000 &>/dev/null || true
 		ulimit -m 512000 &>/dev/null || true
+		go run build.go test
+		;;
 
-		go run build.go "$1"
+	bench)
+		LOGGER_DISCARD=1 go run build.go bench | go run benchfilter.go
 		;;
 
 	tar)
@@ -50,20 +53,29 @@ case "${1:-default}" in
 		;;
 
 	all)
-		go run build.go -goos linux -goarch amd64 tar
+		go run build.go -goos darwin -goarch amd64 tar
+		go run build.go -goos darwin -goarch 386 tar
+
+		go run build.go -goos dragonfly -goarch 386 tar
+		go run build.go -goos dragonfly -goarch amd64 tar
+
+		go run build.go -goos freebsd -goarch 386 tar
+		go run build.go -goos freebsd -goarch amd64 tar
+
 		go run build.go -goos linux -goarch 386 tar
+		go run build.go -goos linux -goarch amd64 tar
 		go run build.go -goos linux -goarch arm tar
 
-		go run build.go -goos freebsd -goarch amd64 tar
-		go run build.go -goos freebsd -goarch 386 tar
+		go run build.go -goos netbsd -goarch 386 tar
+		go run build.go -goos netbsd -goarch amd64 tar
 
-		go run build.go -goos openbsd -goarch amd64 tar
 		go run build.go -goos openbsd -goarch 386 tar
+		go run build.go -goos openbsd -goarch amd64 tar
 
-		go run build.go -goos darwin -goarch amd64 tar
+		go run build.go -goos solaris -goarch amd64 tar
 
-		go run build.go -goos windows -goarch amd64 zip
 		go run build.go -goos windows -goarch 386 zip
+		go run build.go -goos windows -goarch amd64 zip
 		;;
 
 	setup)
@@ -109,10 +121,9 @@ case "${1:-default}" in
 			-e "STTRACE=$STTRACE" \
 			syncthing/build:latest \
 			sh -c './build.sh clean \
-				&& go vet ./cmd/... ./internal/... \
-				&& ( golint ./cmd/... ; golint ./internal/... ) | egrep -v "comment on exported|should have comment" \
-				&& ./build.sh all \
-				&& STTRACE=all ./build.sh test-cov'
+				&& ./build.sh test-cov \
+				&& ./build.sh bench \
+				&& ./build.sh all'
 		;;
 
 	docker-test)
@@ -125,10 +136,29 @@ case "${1:-default}" in
 				&& go run build.go -race \
 				&& export GOPATH=$(pwd)/Godeps/_workspace:$GOPATH \
 				&& cd test \
-				&& go test -tags integration -v -timeout 60m -short \
+				&& go test -tags integration -v -timeout 90m -short \
 				&& git clean -fxd .'
 		;;
 
+	docker-lint)
+		docker run --rm -h syncthing-builder -u $(id -u) -t \
+			-v $(pwd):/go/src/github.com/syncthing/syncthing \
+			-w /go/src/github.com/syncthing/syncthing \
+			-e "STTRACE=$STTRACE" \
+			syncthing/build:latest \
+			sh -euxc 'go run build.go lint'
+		;;
+
+
+	docker-vet)
+		docker run --rm -h syncthing-builder -u $(id -u) -t \
+			-v $(pwd):/go/src/github.com/syncthing/syncthing \
+			-w /go/src/github.com/syncthing/syncthing \
+			-e "STTRACE=$STTRACE" \
+			syncthing/build:latest \
+			sh -euxc 'go run build.go vet'
+		;;
+
 	*)
 		echo "Unknown build command $1"
 		;;

changelog.go

@@ -0,0 +1,72 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+// +build ignore
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"log"
+	"os/exec"
+	"regexp"
+)
+
+var (
+	subjectIssues = regexp.MustCompile(`^([^(]+)\s+\((?:fixes|ref) ([^)]+)\)(?:[^\w])?$`)
+	issueNumbers  = regexp.MustCompile(`(#\d+)`)
+)
+
+func main() {
+	flag.Parse()
+
+	// Display changelog since the version given on the command line, or
+	// figure out the last release if there were no arguments.
+	var prevRel string
+	if flag.NArg() > 0 {
+		prevRel = flag.Arg(0)
+	} else {
+		bs, err := runError("git", "describe", "--abbrev=0", "HEAD^")
+		if err != nil {
+			log.Fatal(err)
+		}
+		prevRel = string(bs)
+	}
+
+	// Get the git log with subject and author nickname
+	bs, err := runError("git", "log", "--reverse", "--pretty=format:%s|%aN", prevRel+"..")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Split into lines
+	for _, line := range bytes.Split(bs, []byte{'\n'}) {
+		// Split into subject and author
+		fields := bytes.Split(line, []byte{'|'})
+		subj := fields[0]
+		author := fields[1]
+
+		// Check if subject contains a "(fixes ...)" or "(ref ...)""
+		if m := subjectIssues.FindSubmatch(subj); len(m) > 0 {
+			// Find all issue numbers
+			issues := issueNumbers.FindAll(m[2], -1)
+
+			// Format a changelog entry
+			fmt.Printf("* %s (%s, @%s)\n", m[1], bytes.Join(issues, []byte(", ")), author)
+		}
+	}
+}
+
+func runError(cmd string, args ...string) ([]byte, error) {
+	ecmd := exec.Command(cmd, args...)
+	bs, err := ecmd.CombinedOutput()
+	if err != nil {
+		return nil, err
+	}
+	return bytes.TrimSpace(bs), nil
+}

changelog.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-since="$1"
-if [[ -z $since ]] ; then
-	since="$(git describe --abbrev=0 HEAD^).."
-fi
-
-git log --reverse --pretty=format:'* %s, @%aN)' "$since" | egrep 'fixes #\d|ref #\d' | sed 's/),/,/' | sed 's/fixes #/#/g' | sed 's/ref #/#/g'
-

check-contrib.sh

@@ -17,7 +17,9 @@ no-docs-typos() {
 	grep -v a9339d0627fff439879d157c75077f02c9fac61b |\
 	grep -v 254c63763a3ad42fd82259f1767db526cff94a14 |\
 	grep -v 4b76ec40c07078beaa2c5e250ed7d9bd6276a718 |\
-	grep -v ffc39dfbcb34eacc3ea12327a02b6e7741a2c207
+	grep -v ffc39dfbcb34eacc3ea12327a02b6e7741a2c207 |\
+	grep -v 32a76901a91ff0f663db6f0830e0aedec946e4d0 |\
+	grep -v af3288043a49bcc28f8ae3060852a09de552fe5f
 }
 
 print-missing-authors() {
@@ -27,7 +29,7 @@ print-missing-authors() {
 }
 
 print-missing-copyright() {
-	find . -name \*.go | xargs egrep -L 'Copyright \(C\)|automatically generated' | grep -v Godeps | grep -v internal/auto/
+	find . -name \*.go | xargs egrep -L 'Copyright|automatically generated' | grep -v Godeps | grep -v internal/auto/
 }
 
 authors=$(print-missing-authors)

cmd/genassets/main.go

@@ -1,17 +1,8 @@
 // Copyright (C) 2014 The Syncthing Authors.
 //
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
 
 // +build ignore
 
@@ -24,33 +15,32 @@ import (
 	"flag"
 	"go/format"
 	"io"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
 	"text/template"
+	"time"
 )
 
 var tpl = template.Must(template.New("assets").Parse(`package auto
 
 import (
-	"bytes"
-	"compress/gzip"
 	"encoding/base64"
-	"io/ioutil"
+)
+
+const (
+	AssetsBuildDate = "{{.BuildDate}}"
 )
 
 func Assets() map[string][]byte {
-	var assets = make(map[string][]byte, {{.assets | len}})
-	var bs []byte
-	var gr *gzip.Reader
-{{range $asset := .assets}}
-	bs, _ = base64.StdEncoding.DecodeString("{{$asset.Data}}")
-	gr, _ = gzip.NewReader(bytes.NewReader(bs))
-	bs, _ = ioutil.ReadAll(gr)
-	assets["{{$asset.Name}}"] = bs
+	var assets = make(map[string][]byte, {{.Assets | len}})
+{{range $asset := .Assets}}
+	assets["{{$asset.Name}}"], _ = base64.StdEncoding.DecodeString("{{$asset.Data}}")
 {{end}}
 	return assets
 }
+
 `))
 
 type asset struct {
@@ -95,12 +85,20 @@ func walkerFor(basePath string) filepath.WalkFunc {
 	}
 }
 
+type templateVars struct {
+	Assets    []asset
+	BuildDate string
+}
+
 func main() {
 	flag.Parse()
 
 	filepath.Walk(flag.Arg(0), walkerFor(flag.Arg(0)))
 	var buf bytes.Buffer
-	tpl.Execute(&buf, map[string][]asset{"assets": assets})
+	tpl.Execute(&buf, templateVars{
+		Assets:    assets,
+		BuildDate: time.Now().UTC().Format(http.TimeFormat),
+	})
 	bs, err := format.Source(buf.Bytes())
 	if err != nil {
 		panic(err)

cmd/stcompdirs/main.go

@@ -1,17 +1,8 @@
 // Copyright (C) 2014 The Syncthing Authors.
 //
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/stevents/main.go

@@ -1,17 +1,8 @@
 // Copyright (C) 2014 The Syncthing Authors.
 //
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -36,7 +27,7 @@ func main() {
 	log.SetOutput(os.Stdout)
 	log.SetFlags(0)
 
-	target := flag.String("target", "localhost:8080", "Target Syncthing instance")
+	target := flag.String("target", "localhost:8384", "Target Syncthing instance")
 	apikey := flag.String("apikey", "", "Syncthing API key")
 	flag.Parse()
 

cmd/stfileinfo/main.go

@@ -1,17 +1,8 @@
 // Copyright (C) 2014 The Syncthing Authors.
 //
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/stfinddevice/main.go

@@ -1,17 +1,8 @@
 // Copyright (C) 2014 The Syncthing Authors.
 //
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/stindex/main.go

@@ -1,21 +1,13 @@
 // Copyright (C) 2014 The Syncthing Authors.
 //
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
 
 package main
 
 import (
+	"encoding/binary"
 	"flag"
 	"fmt"
 	"log"
@@ -24,41 +16,71 @@ import (
 	"github.com/syncthing/protocol"
 	"github.com/syncthing/syncthing/internal/db"
 	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
 )
 
 func main() {
 	log.SetFlags(0)
 	log.SetOutput(os.Stdout)
 
-	folder := flag.String("folder", "default", "Folder ID")
-	device := flag.String("device", "", "Device ID (blank for global)")
 	flag.Parse()
 
-	ldb, err := leveldb.OpenFile(flag.Arg(0), nil)
+	ldb, err := leveldb.OpenFile(flag.Arg(0), &opt.Options{
+		ErrorIfMissing:         true,
+		Strict:                 opt.StrictAll,
+		OpenFilesCacheCapacity: 100,
+	})
 	if err != nil {
 		log.Fatal(err)
 	}
 
-	fs := db.NewFileSet(*folder, ldb)
+	it := ldb.NewIterator(nil, nil)
+	var dev protocol.DeviceID
+	for it.Next() {
+		key := it.Key()
+		switch key[0] {
+		case db.KeyTypeDevice:
+			folder := nulString(key[1 : 1+64])
+			devBytes := key[1+64 : 1+64+32]
+			name := nulString(key[1+64+32:])
+			copy(dev[:], devBytes)
+			fmt.Printf("[device] F:%q N:%q D:%v\n", folder, name, dev)
 
-	if *device == "" {
-		log.Printf("*** Global index for folder %q", *folder)
-		fs.WithGlobalTruncated(func(fi db.FileIntf) bool {
-			f := fi.(db.FileInfoTruncated)
-			fmt.Println(f)
-			fmt.Println("\t", fs.Availability(f.Name))
-			return true
-		})
-	} else {
-		n, err := protocol.DeviceIDFromString(*device)
-		if err != nil {
-			log.Fatal(err)
+			var f protocol.FileInfo
+			err := f.UnmarshalXDR(it.Value())
+			if err != nil {
+				log.Fatal(err)
+			}
+			fmt.Printf("  N:%q\n  F:%#o\n  M:%d\n  V:%v\n  S:%d\n  B:%d\n", f.Name, f.Flags, f.Modified, f.Version, f.Size(), len(f.Blocks))
+
+		case db.KeyTypeGlobal:
+			folder := nulString(key[1 : 1+64])
+			name := nulString(key[1+64:])
+			fmt.Printf("[global] F:%q N:%q V:%x\n", folder, name, it.Value())
+
+		case db.KeyTypeBlock:
+			folder := nulString(key[1 : 1+64])
+			hash := key[1+64 : 1+64+32]
+			name := nulString(key[1+64+32:])
+			fmt.Printf("[block] F:%q H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
+
+		case db.KeyTypeDeviceStatistic:
+			fmt.Printf("[dstat]\n  %x\n  %x\n", it.Key(), it.Value())
+
+		case db.KeyTypeFolderStatistic:
+			fmt.Printf("[fstat]\n  %x\n  %x\n", it.Key(), it.Value())
+
+		default:
+			fmt.Printf("[???]\n  %x\n  %x\n", it.Key(), it.Value())
 		}
-		log.Printf("*** Have index for folder %q device %q", *folder, n)
-		fs.WithHaveTruncated(n, func(fi db.FileIntf) bool {
-			f := fi.(db.FileInfoTruncated)
-			fmt.Println(f)
-			return true
-		})
 	}
 }
+
+func nulString(bs []byte) string {
+	for i := range bs {
+		if bs[i] == 0 {
+			return string(bs[:i])
+		}
+	}
+	return string(bs)
+}

cmd/syncthing/audit.go

@@ -0,0 +1,69 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"encoding/json"
+	"io"
+
+	"github.com/syncthing/syncthing/internal/events"
+)
+
+// The auditSvc subscribes to events and writes these in JSON format, one
+// event per line, to the specified writer.
+type auditSvc struct {
+	w       io.Writer     // audit destination
+	stop    chan struct{} // signals time to stop
+	started chan struct{} // signals startup complete
+	stopped chan struct{} // signals stop complete
+}
+
+func newAuditSvc(w io.Writer) *auditSvc {
+	return &auditSvc{
+		w:       w,
+		stop:    make(chan struct{}),
+		started: make(chan struct{}),
+		stopped: make(chan struct{}),
+	}
+}
+
+// Serve runs the audit service.
+func (s *auditSvc) Serve() {
+	defer close(s.stopped)
+	sub := events.Default.Subscribe(events.AllEvents)
+	defer events.Default.Unsubscribe(sub)
+	enc := json.NewEncoder(s.w)
+
+	// We're ready to start processing events.
+	close(s.started)
+
+	for {
+		select {
+		case ev := <-sub.C():
+			enc.Encode(ev)
+		case <-s.stop:
+			return
+		}
+	}
+}
+
+// Stop stops the audit service.
+func (s *auditSvc) Stop() {
+	close(s.stop)
+}
+
+// WaitForStart returns once the audit service is ready to receive events, or
+// immediately if it's already running.
+func (s *auditSvc) WaitForStart() {
+	<-s.started
+}
+
+// WaitForStop returns once the audit service has stopped.
+// (Needed by the tests.)
+func (s *auditSvc) WaitForStop() {
+	<-s.stopped
+}

cmd/syncthing/auditsvc_test.go

@@ -0,0 +1,54 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/syncthing/syncthing/internal/events"
+)
+
+func TestAuditService(t *testing.T) {
+	buf := new(bytes.Buffer)
+	svc := newAuditSvc(buf)
+
+	// Event sent before start, will not be logged
+	events.Default.Log(events.Ping, "the first event")
+
+	go svc.Serve()
+	svc.WaitForStart()
+
+	// Event that should end up in the audit log
+	events.Default.Log(events.Ping, "the second event")
+
+	// We need to give the events time to arrive, since the channels are buffered etc.
+	time.Sleep(10 * time.Millisecond)
+
+	svc.Stop()
+	svc.WaitForStop()
+
+	// This event should not be logged, since we have stopped.
+	events.Default.Log(events.Ping, "the third event")
+
+	result := string(buf.Bytes())
+	t.Log(result)
+
+	if strings.Contains(result, "first event") {
+		t.Error("Unexpected first event")
+	}
+
+	if !strings.Contains(result, "second event") {
+		t.Error("Missing second event")
+	}
+
+	if strings.Contains(result, "third event") {
+		t.Error("Missing third event")
+	}
+}

cmd/syncthing/blockprof.go

@@ -1,17 +1,8 @@
 // Copyright (C) 2014 The Syncthing Authors.
 //
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/syncthing/connections.go

@@ -0,0 +1,355 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net"
+	"strings"
+	"time"
+
+	"github.com/syncthing/protocol"
+	"github.com/syncthing/syncthing/internal/config"
+	"github.com/syncthing/syncthing/internal/events"
+	"github.com/syncthing/syncthing/internal/model"
+	"github.com/thejerf/suture"
+)
+
+// The connection service listens on TLS and dials configured unconnected
+// devices. Successful connections are handed to the model.
+type connectionSvc struct {
+	*suture.Supervisor
+	cfg    *config.Wrapper
+	myID   protocol.DeviceID
+	model  *model.Model
+	tlsCfg *tls.Config
+	conns  chan *tls.Conn
+}
+
+func newConnectionSvc(cfg *config.Wrapper, myID protocol.DeviceID, model *model.Model, tlsCfg *tls.Config) *connectionSvc {
+	svc := &connectionSvc{
+		Supervisor: suture.NewSimple("connectionSvc"),
+		cfg:        cfg,
+		myID:       myID,
+		model:      model,
+		tlsCfg:     tlsCfg,
+		conns:      make(chan *tls.Conn),
+	}
+
+	// There are several moving parts here; one routine per listening address
+	// to handle incoming connections, one routine to periodically attempt
+	// outgoing connections, and lastly one routine to the the common handling
+	// regardless of whether the connection was incoming or outgoing. It ends
+	// up as in the diagram below. We embed a Supervisor to manage the
+	// routines (i.e. log and restart if they crash or exit, etc).
+	//
+	//                +-----------------+
+	//    Incoming    | +---------------+-+      +-----------------+
+	//   Connections  | |                 |      |                 |   Outgoing
+	// -------------->| |   svc.listen    |      |                 |  Connections
+	//                | |  (1 per listen  |      |   svc.connect   |-------------->
+	//                | |    address)     |      |                 |
+	//                +-+                 |      |                 |
+	//                  +-----------------+      +-----------------+
+	//                           v                        v
+	//                           |                        |
+	//                           |                        |
+	//                           +------------+-----------+
+	//                                        |
+	//                                        | svc.conns
+	//                                        v
+	//                               +-----------------+
+	//                               |                 |
+	//                               |                 |
+	//                               |   svc.handle    |------> model.AddConnection()
+	//                               |                 |
+	//                               |                 |
+	//                               +-----------------+
+	//
+	// TODO: Clean shutdown, and/or handling config changes on the fly. We
+	// partly do this now - new devices and addresses will be picked up, but
+	// not new listen addresses and we don't support disconnecting devices
+	// that are removed and so on...
+
+	svc.Add(serviceFunc(svc.connect))
+	for _, addr := range svc.cfg.Options().ListenAddress {
+		addr := addr
+		listener := serviceFunc(func() {
+			svc.listen(addr)
+		})
+		svc.Add(listener)
+	}
+	svc.Add(serviceFunc(svc.handle))
+
+	return svc
+}
+
+func (s *connectionSvc) handle() {
+next:
+	for conn := range s.conns {
+		cs := conn.ConnectionState()
+
+		// We should have negotiated the next level protocol "bep/1.0" as part
+		// of the TLS handshake. Unfortunately this can't be a hard error,
+		// because there are implementations out there that don't support
+		// protocol negotiation (iOS for one...).
+		if !cs.NegotiatedProtocolIsMutual || cs.NegotiatedProtocol != bepProtocolName {
+			l.Infof("Peer %s did not negotiate bep/1.0", conn.RemoteAddr())
+		}
+
+		// We should have received exactly one certificate from the other
+		// side. If we didn't, they don't have a device ID and we drop the
+		// connection.
+		certs := cs.PeerCertificates
+		if cl := len(certs); cl != 1 {
+			l.Infof("Got peer certificate list of length %d != 1 from %s; protocol error", cl, conn.RemoteAddr())
+			conn.Close()
+			continue
+		}
+		remoteCert := certs[0]
+		remoteID := protocol.NewDeviceID(remoteCert.Raw)
+
+		// The device ID should not be that of ourselves. It can happen
+		// though, especially in the presence of NAT hairpinning, multiple
+		// clients between the same NAT gateway, and global discovery.
+		if remoteID == myID {
+			l.Infof("Connected to myself (%s) - should not happen", remoteID)
+			conn.Close()
+			continue
+		}
+
+		// We should not already be connected to the other party. TODO: This
+		// could use some better handling. If the old connection is dead but
+		// hasn't timed out yet we may want to drop *that* connection and keep
+		// this one. But in case we are two devices connecting to each other
+		// in parallel we don't want to do that or we end up with no
+		// connections still established...
+		if s.model.ConnectedTo(remoteID) {
+			l.Infof("Connected to already connected device (%s)", remoteID)
+			conn.Close()
+			continue
+		}
+
+		for deviceID, deviceCfg := range s.cfg.Devices() {
+			if deviceID == remoteID {
+				// Verify the name on the certificate. By default we set it to
+				// "syncthing" when generating, but the user may have replaced
+				// the certificate and used another name.
+				certName := deviceCfg.CertName
+				if certName == "" {
+					certName = tlsDefaultCommonName
+				}
+				err := remoteCert.VerifyHostname(certName)
+				if err != nil {
+					// Incorrect certificate name is something the user most
+					// likely wants to know about, since it's an advanced
+					// config. Warn instead of Info.
+					l.Warnf("Bad certificate from %s (%v): %v", remoteID, conn.RemoteAddr(), err)
+					conn.Close()
+					continue next
+				}
+
+				// If rate limiting is set, and based on the address we should
+				// limit the connection, then we wrap it in a limiter.
+
+				limit := s.shouldLimit(conn.RemoteAddr())
+
+				wr := io.Writer(conn)
+				if limit && writeRateLimit != nil {
+					wr = &limitedWriter{conn, writeRateLimit}
+				}
+
+				rd := io.Reader(conn)
+				if limit && readRateLimit != nil {
+					rd = &limitedReader{conn, readRateLimit}
+				}
+
+				name := fmt.Sprintf("%s-%s", conn.LocalAddr(), conn.RemoteAddr())
+				protoConn := protocol.NewConnection(remoteID, rd, wr, s.model, name, deviceCfg.Compression)
+
+				l.Infof("Established secure connection to %s at %s", remoteID, name)
+				if debugNet {
+					l.Debugf("cipher suite: %04X in lan: %t", conn.ConnectionState().CipherSuite, !limit)
+				}
+				events.Default.Log(events.DeviceConnected, map[string]string{
+					"id":   remoteID.String(),
+					"addr": conn.RemoteAddr().String(),
+				})
+
+				s.model.AddConnection(conn, protoConn)
+				continue next
+			}
+		}
+
+		if !s.cfg.IgnoredDevice(remoteID) {
+			events.Default.Log(events.DeviceRejected, map[string]string{
+				"device":  remoteID.String(),
+				"address": conn.RemoteAddr().String(),
+			})
+			l.Infof("Connection from %s with unknown device ID %s", conn.RemoteAddr(), remoteID)
+		} else {
+			l.Infof("Connection from %s with ignored device ID %s", conn.RemoteAddr(), remoteID)
+		}
+
+		conn.Close()
+	}
+}
+
+func (s *connectionSvc) listen(addr string) {
+	if debugNet {
+		l.Debugln("listening on", addr)
+	}
+
+	tcaddr, err := net.ResolveTCPAddr("tcp", addr)
+	if err != nil {
+		l.Fatalln("listen (BEP):", err)
+	}
+	listener, err := net.ListenTCP("tcp", tcaddr)
+	if err != nil {
+		l.Fatalln("listen (BEP):", err)
+	}
+
+	for {
+		conn, err := listener.Accept()
+		if err != nil {
+			l.Warnln("Accepting connection:", err)
+			continue
+		}
+
+		if debugNet {
+			l.Debugln("connect from", conn.RemoteAddr())
+		}
+
+		tcpConn := conn.(*net.TCPConn)
+		s.setTCPOptions(tcpConn)
+
+		tc := tls.Server(conn, s.tlsCfg)
+		err = tc.Handshake()
+		if err != nil {
+			l.Infoln("TLS handshake:", err)
+			tc.Close()
+			continue
+		}
+
+		s.conns <- tc
+	}
+}
+
+func (s *connectionSvc) connect() {
+	delay := time.Second
+	for {
+	nextDevice:
+		for deviceID, deviceCfg := range s.cfg.Devices() {
+			if deviceID == myID {
+				continue
+			}
+
+			if s.model.ConnectedTo(deviceID) {
+				continue
+			}
+
+			var addrs []string
+			for _, addr := range deviceCfg.Addresses {
+				if addr == "dynamic" {
+					if discoverer != nil {
+						t := discoverer.Lookup(deviceID)
+						if len(t) == 0 {
+							continue
+						}
+						addrs = append(addrs, t...)
+					}
+				} else {
+					addrs = append(addrs, addr)
+				}
+			}
+
+			for _, addr := range addrs {
+				host, port, err := net.SplitHostPort(addr)
+				if err != nil && strings.HasPrefix(err.Error(), "missing port") {
+					// addr is on the form "1.2.3.4"
+					addr = net.JoinHostPort(addr, "22000")
+				} else if err == nil && port == "" {
+					// addr is on the form "1.2.3.4:"
+					addr = net.JoinHostPort(host, "22000")
+				}
+				if debugNet {
+					l.Debugln("dial", deviceCfg.DeviceID, addr)
+				}
+
+				raddr, err := net.ResolveTCPAddr("tcp", addr)
+				if err != nil {
+					if debugNet {
+						l.Debugln(err)
+					}
+					continue
+				}
+
+				conn, err := net.DialTCP("tcp", nil, raddr)
+				if err != nil {
+					if debugNet {
+						l.Debugln(err)
+					}
+					continue
+				}
+
+				s.setTCPOptions(conn)
+
+				tc := tls.Client(conn, s.tlsCfg)
+				err = tc.Handshake()
+				if err != nil {
+					l.Infoln("TLS handshake:", err)
+					tc.Close()
+					continue
+				}
+
+				s.conns <- tc
+				continue nextDevice
+			}
+		}
+
+		time.Sleep(delay)
+		delay *= 2
+		if maxD := time.Duration(s.cfg.Options().ReconnectIntervalS) * time.Second; delay > maxD {
+			delay = maxD
+		}
+	}
+}
+
+func (*connectionSvc) setTCPOptions(conn *net.TCPConn) {
+	var err error
+	if err = conn.SetLinger(0); err != nil {
+		l.Infoln(err)
+	}
+	if err = conn.SetNoDelay(false); err != nil {
+		l.Infoln(err)
+	}
+	if err = conn.SetKeepAlivePeriod(60 * time.Second); err != nil {
+		l.Infoln(err)
+	}
+	if err = conn.SetKeepAlive(true); err != nil {
+		l.Infoln(err)
+	}
+}
+
+func (s *connectionSvc) shouldLimit(addr net.Addr) bool {
+	if s.cfg.Options().LimitBandwidthInLan {
+		return true
+	}
+
+	tcpaddr, ok := addr.(*net.TCPAddr)
+	if !ok {
+		return true
+	}
+	for _, lan := range lans {
+		if lan.Contains(tcpaddr.IP) {
+			return false
+		}
+	}
+	return !tcpaddr.IP.IsLoopback()
+}

cmd/syncthing/debug.go

@@ -1,17 +1,8 @@
 // Copyright (C) 2014 The Syncthing Authors.
 //
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -21,5 +12,6 @@ import (
 )
 
 var (
-	debugNet = strings.Contains(os.Getenv("STTRACE"), "net") || os.Getenv("STTRACE") == "all"
+	debugNet  = strings.Contains(os.Getenv("STTRACE"), "net") || os.Getenv("STTRACE") == "all"
+	debugHTTP = strings.Contains(os.Getenv("STTRACE"), "http") || os.Getenv("STTRACE") == "all"
 )

cmd/syncthing/gui_auth.go

@@ -1,17 +1,8 @@
 // Copyright (C) 2014 The Syncthing Authors.
 //
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -21,16 +12,16 @@ import (
 	"math/rand"
 	"net/http"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/syncthing/syncthing/internal/config"
+	"github.com/syncthing/syncthing/internal/sync"
 	"golang.org/x/crypto/bcrypt"
 )
 
 var (
 	sessions    = make(map[string]bool)
-	sessionsMut sync.Mutex
+	sessionsMut = sync.NewMutex()
 )
 
 func basicAuthAndSessionMiddleware(cfg config.GUIConfiguration, next http.Handler) http.Handler {
@@ -51,6 +42,10 @@ func basicAuthAndSessionMiddleware(cfg config.GUIConfiguration, next http.Handle
 			}
 		}
 
+		if debugHTTP {
+			l.Debugln("Sessionless HTTP request with authentication; this is expensive.")
+		}
+
 		error := func() {
 			time.Sleep(time.Duration(rand.Intn(100)+100) * time.Millisecond)
 			w.Header().Set("WWW-Authenticate", "Basic realm=\"Authorization Required\"")

cmd/syncthing/gui_csrf.go

@@ -1,17 +1,8 @@
 // Copyright (C) 2014 The Syncthing Authors.
 //
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -20,16 +11,15 @@ import (
 	"fmt"
 	"net/http"
 	"os"
-	"path/filepath"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/syncthing/syncthing/internal/osutil"
+	"github.com/syncthing/syncthing/internal/sync"
 )
 
 var csrfTokens []string
-var csrfMut sync.Mutex
+var csrfMut = sync.NewMutex()
 
 // Check for CSRF token on /rest/ URLs. If a correct one is not given, reject
 // the request with 403. For / and /index.html, set a new CSRF cookie if none
@@ -101,7 +91,7 @@ func newCsrfToken() string {
 }
 
 func saveCsrfTokens() {
-	name := filepath.Join(confDir, "csrftokens.txt")
+	name := locations[locCsrfTokens]
 	tmp := fmt.Sprintf("%s.tmp.%d", name, time.Now().UnixNano())
 
 	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
@@ -126,8 +116,7 @@ func saveCsrfTokens() {
 }
 
 func loadCsrfTokens() {
-	name := filepath.Join(confDir, "csrftokens.txt")
-	f, err := os.Open(name)
+	f, err := os.Open(locations[locCsrfTokens])
 	if err != nil {
 		return
 	}

cmd/syncthing/gui_solaris.go

@@ -1,17 +1,8 @@
 // Copyright (C) 2014 The Syncthing Authors.
 //
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
 
 //+build solaris
 

cmd/syncthing/gui_unix.go

@@ -1,17 +1,8 @@
 // Copyright (C) 2014 The Syncthing Authors.
 //
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
 
 //+build !windows,!solaris
 

cmd/syncthing/gui_windows.go

@@ -1,17 +1,8 @@
 // Copyright (C) 2014 The Syncthing Authors.
 //
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
 
 //+build windows
 

cmd/syncthing/heapprof.go

@@ -1,17 +1,8 @@
 // Copyright (C) 2014 The Syncthing Authors.
 //
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/syncthing/limitedreader.go

@@ -1,17 +1,8 @@
 // Copyright (C) 2014 The Syncthing Authors.
 //
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/syncthing/limitedwriter.go

@@ -1,17 +1,8 @@
 // Copyright (C) 2014 The Syncthing Authors.
 //
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
 
 package main
 

cmd/syncthing/locations.go

@@ -0,0 +1,123 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/syncthing/syncthing/internal/osutil"
+)
+
+type locationEnum string
+
+// Use strings as keys to make printout and serialization of the locations map
+// more meaningful.
+const (
+	locConfigFile    locationEnum = "config"
+	locCertFile                   = "certFile"
+	locKeyFile                    = "keyFile"
+	locHTTPSCertFile              = "httpsCertFile"
+	locHTTPSKeyFile               = "httpsKeyFile"
+	locDatabase                   = "database"
+	locLogFile                    = "logFile"
+	locCsrfTokens                 = "csrfTokens"
+	locPanicLog                   = "panicLog"
+	locAuditLog                   = "auditLog"
+	locDefFolder                  = "defFolder"
+)
+
+// Platform dependent directories
+var baseDirs = map[string]string{
+	"config": defaultConfigDir(), // Overridden by -home flag
+	"home":   homeDir(),          // User's home directory, *not* -home flag
+}
+
+// Use the variables from baseDirs here
+var locations = map[locationEnum]string{
+	locConfigFile:    "${config}/config.xml",
+	locCertFile:      "${config}/cert.pem",
+	locKeyFile:       "${config}/key.pem",
+	locHTTPSCertFile: "${config}/https-cert.pem",
+	locHTTPSKeyFile:  "${config}/https-key.pem",
+	locDatabase:      "${config}/index-v0.11.0.db",
+	locLogFile:       "${config}/syncthing.log", // -logfile on Windows
+	locCsrfTokens:    "${config}/csrftokens.txt",
+	locPanicLog:      "${config}/panic-${timestamp}.log",
+	locAuditLog:      "${config}/audit-${timestamp}.log",
+	locDefFolder:     "${home}/Sync",
+}
+
+// expandLocations replaces the variables in the location map with actual
+// directory locations.
+func expandLocations() error {
+	for key, dir := range locations {
+		for varName, value := range baseDirs {
+			dir = strings.Replace(dir, "${"+varName+"}", value, -1)
+		}
+		var err error
+		dir, err = osutil.ExpandTilde(dir)
+		if err != nil {
+			return err
+		}
+		locations[key] = dir
+	}
+	return nil
+}
+
+// defaultConfigDir returns the default configuration directory, as figured
+// out by various the environment variables present on each platform, or dies
+// trying.
+func defaultConfigDir() string {
+	switch runtime.GOOS {
+	case "windows":
+		if p := os.Getenv("LocalAppData"); p != "" {
+			return filepath.Join(p, "Syncthing")
+		}
+		return filepath.Join(os.Getenv("AppData"), "Syncthing")
+
+	case "darwin":
+		dir, err := osutil.ExpandTilde("~/Library/Application Support/Syncthing")
+		if err != nil {
+			l.Fatalln(err)
+		}
+		return dir
+
+	default:
+		if xdgCfg := os.Getenv("XDG_CONFIG_HOME"); xdgCfg != "" {
+			return filepath.Join(xdgCfg, "syncthing")
+		}
+		dir, err := osutil.ExpandTilde("~/.config/syncthing")
+		if err != nil {
+			l.Fatalln(err)
+		}
+		return dir
+	}
+}
+
+// homeDir returns the user's home directory, or dies trying.
+func homeDir() string {
+	home, err := osutil.ExpandTilde("~")
+	if err != nil {
+		l.Fatalln(err)
+	}
+	return home
+}
+
+func timestampedLoc(key locationEnum) string {
+	// We take the roundtrip via "${timestamp}" instead of passing the path
+	// directly through time.Format() to avoid issues when the path we are
+	// expanding contains numbers; otherwise for example
+	// /home/user2006/.../panic-20060102-150405.log would get both instances of
+	// 2006 replaced by 2015...
+	tpl := locations[key]
+	now := time.Now().Format("20060102-150405")
+	return strings.Replace(tpl, "${timestamp}", now, -1)
+}

cmd/syncthing/main_test.go

@@ -1,17 +1,8 @@
 // Copyright (C) 2014 The Syncthing Authors.
 //
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
 
 package main
 
@@ -28,19 +19,22 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/storage"
 )
 
-func TestSanityCheck(t *testing.T) {
+func TestFolderErrors(t *testing.T) {
+	// This test intentionally avoids starting the folders. If they are
+	// started, they will perform an initial scan, which will create missing
+	// folder markers and race with the stuff we do in the test.
+
 	fcfg := config.FolderConfiguration{
-		ID:   "folder",
-		Path: "testdata/testfolder",
+		ID:      "folder",
+		RawPath: "testdata/testfolder",
 	}
 	cfg := config.Wrap("/tmp/test", config.Configuration{
 		Folders: []config.FolderConfiguration{fcfg},
 	})
 
-	for _, file := range []string{".stfolder", "testfolder", "testfolder/.stfolder"} {
-		_, err := os.Stat("testdata/" + file)
-		if err == nil {
-			t.Error("Found unexpected file")
+	for _, file := range []string{".stfolder", "testfolder/.stfolder", "testfolder"} {
+		if err := os.Remove("testdata/" + file); err != nil && !os.IsNotExist(err) {
+			t.Fatal(err)
 		}
 	}
 
@@ -48,10 +42,10 @@ func TestSanityCheck(t *testing.T) {
 
 	// Case 1 - new folder, directory and marker created
 
-	m := model.NewModel(cfg, "device", "syncthing", "dev", ldb)
-	sanityCheckFolders(cfg, m)
+	m := model.NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
+	m.AddFolder(fcfg)
 
-	if cfg.Folders()["folder"].Invalid != "" {
+	if err := m.CheckFolderHealth("folder"); err != nil {
 		t.Error("Unexpected error", cfg.Folders()["folder"].Invalid)
 	}
 
@@ -65,20 +59,24 @@ func TestSanityCheck(t *testing.T) {
 		t.Error(err)
 	}
 
-	os.Remove("testdata/testfolder/.stfolder")
-	os.Remove("testdata/testfolder/")
+	if err := os.Remove("testdata/testfolder/.stfolder"); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Remove("testdata/testfolder/"); err != nil {
+		t.Fatal(err)
+	}
 
 	// Case 2 - new folder, marker created
 
-	fcfg.Path = "testdata/"
+	fcfg.RawPath = "testdata/"
 	cfg = config.Wrap("/tmp/test", config.Configuration{
 		Folders: []config.FolderConfiguration{fcfg},
 	})
 
-	m = model.NewModel(cfg, "device", "syncthing", "dev", ldb)
-	sanityCheckFolders(cfg, m)
+	m = model.NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
+	m.AddFolder(fcfg)
 
-	if cfg.Folders()["folder"].Invalid != "" {
+	if err := m.CheckFolderHealth("folder"); err != nil {
 		t.Error("Unexpected error", cfg.Folders()["folder"].Invalid)
 	}
 
@@ -87,33 +85,96 @@ func TestSanityCheck(t *testing.T) {
 		t.Error(err)
 	}
 
-	os.Remove("testdata/.stfolder")
+	if err := os.Remove("testdata/.stfolder"); err != nil {
+		t.Fatal(err)
+	}
 
-	// Case 3 - marker missing
+	// Case 3 - Folder marker missing
 
 	set := db.NewFileSet("folder", ldb)
 	set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
 		{Name: "dummyfile"},
 	})
 
-	m = model.NewModel(cfg, "device", "syncthing", "dev", ldb)
-	sanityCheckFolders(cfg, m)
+	m = model.NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
+	m.AddFolder(fcfg)
 
-	if cfg.Folders()["folder"].Invalid != "folder marker missing" {
-		t.Error("Incorrect error")
+	if err := m.CheckFolderHealth("folder"); err == nil || err.Error() != "folder marker missing" {
+		t.Error("Incorrect error: Folder marker missing !=", m.CheckFolderHealth("folder"))
 	}
 
-	// Case 4 - path missing
+	// Case 3.1 - recover after folder marker missing
 
-	fcfg.Path = "testdata/testfolder"
-	cfg = config.Wrap("/tmp/test", config.Configuration{
+	if err = fcfg.CreateMarker(); err != nil {
+		t.Error(err)
+	}
+
+	if err := m.CheckFolderHealth("folder"); err != nil {
+		t.Error("Unexpected error", cfg.Folders()["folder"].Invalid)
+	}
+
+	// Case 4 - Folder path missing
+
+	if err := os.Remove("testdata/testfolder/.stfolder"); err != nil && !os.IsNotExist(err) {
+		t.Fatal(err)
+	}
+	if err := os.Remove("testdata/testfolder"); err != nil && !os.IsNotExist(err) {
+		t.Fatal(err)
+	}
+
+	fcfg.RawPath = "testdata/testfolder"
+	cfg = config.Wrap("testdata/subfolder", config.Configuration{
 		Folders: []config.FolderConfiguration{fcfg},
 	})
 
-	m = model.NewModel(cfg, "device", "syncthing", "dev", ldb)
-	sanityCheckFolders(cfg, m)
+	m = model.NewModel(cfg, protocol.LocalDeviceID, "device", "syncthing", "dev", ldb)
+	m.AddFolder(fcfg)
 
-	if cfg.Folders()["folder"].Invalid != "folder path missing" {
-		t.Error("Incorrect error")
+	if err := m.CheckFolderHealth("folder"); err == nil || err.Error() != "folder path missing" {
+		t.Error("Incorrect error: Folder path missing !=", m.CheckFolderHealth("folder"))
+	}
+
+	// Case 4.1 - recover after folder path missing
+
+	if err := os.Mkdir("testdata/testfolder", 0700); err != nil {
+		t.Fatal(err)
+	}
+
+	if err := m.CheckFolderHealth("folder"); err == nil || err.Error() != "folder marker missing" {
+		t.Error("Incorrect error: Folder marker missing !=", m.CheckFolderHealth("folder"))
+	}
+
+	// Case 4.2 - recover after missing marker
+
+	if err = fcfg.CreateMarker(); err != nil {
+		t.Error(err)
+	}
+
+	if err := m.CheckFolderHealth("folder"); err != nil {
+		t.Error("Unexpected error", cfg.Folders()["folder"].Invalid)
+	}
+}
+
+func TestShortIDCheck(t *testing.T) {
+	cfg := config.Wrap("/tmp/test", config.Configuration{
+		Devices: []config.DeviceConfiguration{
+			{DeviceID: protocol.DeviceID{8, 16, 24, 32, 40, 48, 56, 0, 0}},
+			{DeviceID: protocol.DeviceID{8, 16, 24, 32, 40, 48, 56, 1, 1}}, // first 56 bits same, differ in the first 64 bits
+		},
+	})
+
+	if err := checkShortIDs(cfg); err != nil {
+		t.Error("Unexpected error:", err)
+	}
+
+	cfg = config.Wrap("/tmp/test", config.Configuration{
+		Devices: []config.DeviceConfiguration{
+			{DeviceID: protocol.DeviceID{8, 16, 24, 32, 40, 48, 56, 64, 0}},
+			{DeviceID: protocol.DeviceID{8, 16, 24, 32, 40, 48, 56, 64, 1}}, // first 64 bits same
+		},
+	})
+
+	if err := checkShortIDs(cfg); err == nil {
+		t.Error("Should have gotten an error")
 	}
 }

cmd/syncthing/memsize_darwin.go

@@ -1,17 +1,8 @@
 // Copyright (C) 2014 The Syncthing Authors.
 //
-// This program is free software: you can redistribute it and/or modify it
-// under the terms of the GNU General Public License as published by the Free
-// Software Foundation, either version 3 of the License, or (at your option)
-// any later version.
-//
-// This program is distributed in the hope that it will be useful, but WITHOUT
-// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
-// more details.
-//
-// You should have received a copy of the GNU General Public License along
-// with this program. If not, see <http://www.gnu.org/licenses/>.
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
 
 package main