Translation & docs update

Retain standard streams over restart (fixes #2155)

.gitignore

@@ -13,3 +13,4 @@ coverage.xml
 syncthing.md5
 syncthing.exe.md5
 RELEASE
+deb

AUTHORS

@@ -3,28 +3,37 @@
 Aaron Bieber <qbit@deftly.net>
 Alexander Graf <register-github@alex-graf.de>
 Andrew Dunham <andrew@du.nham.ca>
-Audrius Butkevicius <audrius.butkevicius@gmail.com>
+Antony Male <antony.male@gmail.com>
 Arthur Axel fREW Schmidt <frew@afoolishmanifesto.com> <frioux@gmail.com>
+Audrius Butkevicius <audrius.butkevicius@gmail.com>
+Bart De Vries <devriesb@gmail.com>
 Ben Curthoys <ben@bencurthoys.com>
 Ben Schulz <ueomkail@gmail.com> <uok@users.noreply.github.com>
 Ben Sidhom <bsidhom@gmail.com>
 Brandon Philips <brandon@ifup.org>
 Brendan Long <self@brendanlong.com>
+Brian R. Becker <brbecker@gmail.com>
 Caleb Callaway <enlightened.despot@gmail.com>
 Carsten Hagemann <moter8@gmail.com>
 Cathryne Linenweaver <cathryne.linenweaver@gmail.com> <Cathryne@users.noreply.github.com>
+Chris Howie <me@chrishowie.com>
 Chris Joel <chris@scriptolo.gy>
 Colin Kennedy <moshen.colin@gmail.com>
+Daniel Bergmann <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
 Daniel Martí <mvdan@mvdan.cc>
+Denis A. <denisva@gmail.com>
 Dennis Wilson <dw@risu.io>
 Dominik Heidler <dominik@heidler.eu>
 Elias Jarlebring <jarlebring@gmail.com>
 Emil Hessman <emil@hessman.se>
+Erik Meitner <e.meitner@willystreet.coop>
 Federico Castagnini <federico.castagnini@gmail.com>
 Felix Ableitner <me@nutomic.com>
 Felix Unterpaintner <bigbear2nd@gmail.com>
 Francois-Xavier Gsell <fxgsell@gmail.com>
+Frank Isemann <frank@isemann.name>
 Gilli Sigurdsson <gilli@vx.is>
+Jacek Szafarkiewicz <szafar@linux.pl>
 Jakob Borg <jakob@nym.se>
 James Patterson <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
 Jaroslav Malec <dzardacz@gmail.com>
@@ -35,9 +44,9 @@ Karol Różycki <rozycki.karol@gmail.com>
 Ken'ichi Kamada <kamada@nanohz.org>
 Lode Hoste <zillode@zillode.be>
 Lord Landon Agahnim <lordlandon@gmail.com>
-Marcin Dziadus <dziadus.marcin@gmail.com>
 Marc Laporte <marc@marclaporte.com> <marc@laporte.name>
 Marc Pujol <kilburn@la3.org>
+Marcin Dziadus <dziadus.marcin@gmail.com>
 Michael Jephcote <rewt0r@gmx.com> <Rewt0r@users.noreply.github.com>
 Michael Tilli <pyfisch@gmail.com>
 Pascal Jungblut <github@pascalj.com> <mail@pascal-jungblut.com>

CONTRIBUTING.md

@@ -32,64 +32,15 @@ latest info on Transifex.
 ## Contributing Code
 
 Every contribution is welcome. If you want to contribute but are unsure
-where to start, any open issues are fair game! Be prepared for a
-[certain amount of review](https://github.com/syncthing/syncthing/wiki/FAQ#why-are-you-being-so-hard-on-my-pull-request);
-it's all in the name of quality. :) Following the points below will make this
-a smoother process.
+where to start, any open issues are fair game! See the [Contribution
+Guidelines](http://docs.syncthing.net/dev/contributing.html) for the full
+story on committing code.
 
-Individuals making significant and valuable contributions are given
-commit-access to the project. If you make a significant contribution and
-are not considered for commit-access, please contact any of the
-Syncthing core team members.
+## Contributing Documentation
 
-All nontrivial contributions should go through the pull request
-mechanism for internal review. Determining what is "nontrivial" is left
-at the discretion of the contributor.
-
-### Authorship
-
-All code authors are listed in the AUTHORS file. Commits must be made
-with the same name and email as listed in the AUTHORS file. To
-accomplish this, ensure that your git configuration is set correctly
-prior to making your first commit;
-
-    $ git config --global user.name "Jane Doe"
-    $ git config --global user.email janedoe@example.com
-
-You must be reachable on the given email address. If you do not wish to
-use your real name for whatever reason, using a nickname or pseudonym is
-perfectly acceptable.
-
-### Core Team
-
-The Syncthing core team currently consists of the following members;
-
- - Jakob Borg (@calmh)
- - Audrius Butkevicius (@AudriusButkevicius)
-
-## Coding Style
-
-- Follow the conventions laid out in [Effective Go](https://golang.org/doc/effective_go.html)
-  as much as makes sense.
-
-- All text files use Unix line endings.
-
-- Each commit should be `go fmt` clean.
-
-- The commit message subject should be a single short sentence
-  describing the change, starting with a capital letter.
-
-- Commits that resolve an existing issue must include the issue number
-  as `(fixes #123)` at the end of the commit message subject.
-
-- Imports are grouped per `goimports` standard; that is, standard
-  library first, then third party libraries after a blank line.
-
-- A contribution solving a single issue or introducing a single new
-  feature should probably be a single commit based on the current
-  `master` branch. You may be asked to "rebase" or "squash" your pull
-  request to make sure this is the case, especially if there have been
-  amendments during review.
+Updates to the [documentation site](http://docs.syncthing.net/) can be
+made as pull requests on the [documentation
+repository](https://github.com/syncthing/docs).
 
 ## Licensing
 
@@ -99,42 +50,3 @@ strings which are licensed under the Creative Commons Attribution 4.0
 International License. You retain the copyright to code you have
 written.
 
-When accepting your first contribution, the maintainer of the project
-will ensure that you are added to the AUTHORS file. You are welcome to
-add yourself as a separate commit in your first pull request.
-
-## Building
-
-[See the documentation](https://github.com/syncthing/syncthing/wiki/Building)
-on how to get started with a build environment.
-
-## Branches
-
- - `master` is the main branch containing good code that will end up in
-   the next release. You should base your work on it. It won't ever be
-   rebased or force-pushed to.
-
- - `vx.y` branches exist to make patch releases on otherwise obsolete
-   minor releases. Should only contain fixes cherry picked from master.
-   Don't base any work on them.
-
- - Other branches are probably topic branches and may be subject to
-   rebasing. Don't base any work on them unless you specifically know
-   otherwise.
-
-## Tags
-
-All releases are tagged semver style as `vx.y.z`. Release tags are
-signed by GPG key BCE524C7.
-
-## Tests
-
-Yes please!
-
-## Documentation
-
-[Over here!](https://github.com/syncthing/syncthing/wiki)
-
-## License
-
-MPLv2

Godeps/Godeps.json

@@ -1,17 +1,21 @@
 {
 	"ImportPath": "github.com/syncthing/syncthing",
-	"GoVersion": "go1.4",
+	"GoVersion": "go1.5rc1",
 	"Packages": [
 		"./cmd/..."
 	],
 	"Deps": [
 		{
 			"ImportPath": "github.com/bkaradzic/go-lz4",
-			"Rev": "93a831dcee242be64a9cc9803dda84af25932de7"
+			"Rev": "d47913b1412890a261b9fefae99d72d2bf5aebd8"
+		},
+		{
+			"ImportPath": "github.com/calmh/du",
+			"Rev": "3c0690cca16228b97741327b1b6781397afbdb24"
 		},
 		{
 			"ImportPath": "github.com/calmh/logger",
-			"Rev": "4d4e2801954c5581e4c2a80a3d3beb3b3645fd04"
+			"Rev": "c96f6a1a8c7b6bf2f4860c667867d90174799eb2"
 		},
 		{
 			"ImportPath": "github.com/calmh/luhn",
@@ -21,29 +25,29 @@
 			"ImportPath": "github.com/calmh/xdr",
 			"Rev": "5f7208e86762911861c94f1849eddbfc0a60cbf0"
 		},
+		{
+			"ImportPath": "github.com/golang/snappy",
+			"Rev": "723cc1e459b8eea2dea4583200fd60757d40097a"
+		},
 		{
 			"ImportPath": "github.com/juju/ratelimit",
-			"Rev": "c5abe513796336ee2869745bff0638508450e9c5"
+			"Rev": "772f5c38e468398c4511514f4f6aa9a4185bc0a0"
 		},
 		{
 			"ImportPath": "github.com/kardianos/osext",
-			"Rev": "efacde03154693404c65e7aa7d461ac9014acd0c"
+			"Rev": "6e7f843663477789fac7c02def0d0909e969b4e5"
 		},
 		{
 			"ImportPath": "github.com/syncthing/protocol",
-			"Rev": "e7db2648034fb71b051902a02bc25d4468ed492e"
+			"Rev": "388a29bbe21d8772ee4c29f4520aa8040309607d"
 		},
 		{
 			"ImportPath": "github.com/syndtr/goleveldb/leveldb",
-			"Rev": "87e4e645d80ae9c537e8f2dee52b28036a5dd75e"
-		},
-		{
-			"ImportPath": "github.com/syndtr/gosnappy/snappy",
-			"Rev": "156a073208e131d7d2e212cb749feae7c339e846"
+			"Rev": "b743d92d3215f11c9b5ce8830fafe1f16786adf4"
 		},
 		{
 			"ImportPath": "github.com/thejerf/suture",
-			"Rev": "ff19fb384c3fe30f42717967eaa69da91e5f317c"
+			"Rev": "fc7aaeabdc43fe41c5328efa1479ffea0b820978"
 		},
 		{
 			"ImportPath": "github.com/vitrun/qart/coding",
@@ -59,19 +63,19 @@
 		},
 		{
 			"ImportPath": "golang.org/x/crypto/bcrypt",
-			"Rev": "c57d4a71915a248dbad846d60825145062b4c18e"
+			"Rev": "c16968172724c0b5e8bdc6ad33f5a79443a44cd7"
 		},
 		{
 			"ImportPath": "golang.org/x/crypto/blowfish",
-			"Rev": "c57d4a71915a248dbad846d60825145062b4c18e"
+			"Rev": "c16968172724c0b5e8bdc6ad33f5a79443a44cd7"
 		},
 		{
 			"ImportPath": "golang.org/x/text/transform",
-			"Rev": "2076e9cab4147459c82bc81169e46c139d358547"
+			"Rev": "723492b65e225eafcba054e76ba18bb9c5ac1ea2"
 		},
 		{
 			"ImportPath": "golang.org/x/text/unicode/norm",
-			"Rev": "2076e9cab4147459c82bc81169e46c139d358547"
+			"Rev": "723492b65e225eafcba054e76ba18bb9c5ac1ea2"
 		}
 	]
 }

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/.travis.yml

@@ -4,4 +4,5 @@ go:
   - 1.1
   - 1.2
   - 1.3
+  - 1.4
   - tip

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/README.md

@@ -4,7 +4,7 @@ go-lz4
 go-lz4 is port of LZ4 lossless compression algorithm to Go. The original C code
 is located at:
 
-https://code.google.com/p/lz4/
+https://github.com/Cyan4973/lz4
 
 Status
 ------

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/fuzz.go

@@ -0,0 +1,23 @@
+// +build gofuzz
+
+package lz4
+
+import "encoding/binary"
+
+func Fuzz(data []byte) int {
+
+	if len(data) < 4 {
+		return 0
+	}
+
+	ln := binary.LittleEndian.Uint32(data)
+	if ln > (1 << 21) {
+		return 0
+	}
+
+	if _, err := Decode(nil, data); err != nil {
+		return 0
+	}
+
+	return 1
+}

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/reader.go

@@ -141,7 +141,7 @@ func Decode(dst, src []byte) ([]byte, error) {
 			length += ln
 		}
 
-		if int(d.spos+length) > len(d.src) {
+		if int(d.spos+length) > len(d.src) || int(d.dpos+length) > len(d.dst) {
 			return nil, ErrCorrupt
 		}
 
@@ -179,7 +179,12 @@ func Decode(dst, src []byte) ([]byte, error) {
 		}
 
 		literal := d.dpos - d.ref
+
 		if literal < 4 {
+			if int(d.dpos+4) > len(d.dst) {
+				return nil, ErrCorrupt
+			}
+
 			d.cp(4, decr[literal])
 		} else {
 			length += 4

Godeps/_workspace/src/github.com/bkaradzic/go-lz4/writer.go

@@ -25,8 +25,10 @@
 
 package lz4
 
-import "encoding/binary"
-import "errors"
+import (
+	"encoding/binary"
+	"errors"
+)
 
 const (
 	minMatch              = 4

Godeps/_workspace/src/github.com/calmh/du/LICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org>

Godeps/_workspace/src/github.com/calmh/du/README.md

@@ -0,0 +1,14 @@
+du
+==
+
+Get total and available disk space on a given volume.
+
+Documentation
+-------------
+
+http://godoc.org/github.com/calmh/du
+
+License
+-------
+
+Public Domain

Godeps/_workspace/src/github.com/calmh/du/cmd/du/main.go

@@ -0,0 +1,21 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/calmh/du"
+)
+
+var KB = int64(1024)
+
+func main() {
+	usage, err := du.Get(os.Args[1])
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println("Free:", usage.FreeBytes/(KB*KB), "MiB")
+	fmt.Println("Available:", usage.AvailBytes/(KB*KB), "MiB")
+	fmt.Println("Size:", usage.TotalBytes/(KB*KB), "MiB")
+}

Godeps/_workspace/src/github.com/calmh/du/diskusage.go

@@ -0,0 +1,8 @@
+package du
+
+// Usage holds information about total and available storage on a volume.
+type Usage struct {
+	TotalBytes int64 // Size of volume
+	FreeBytes  int64 // Unused size
+	AvailBytes int64 // Available to a non-privileged user
+}

Godeps/_workspace/src/github.com/calmh/du/diskusage_posix.go

@@ -0,0 +1,24 @@
+// +build !windows,!netbsd,!openbsd,!solaris
+
+package du
+
+import (
+	"path/filepath"
+	"syscall"
+)
+
+// Get returns the Usage of a given path, or an error if usage data is
+// unavailable.
+func Get(path string) (Usage, error) {
+	var stat syscall.Statfs_t
+	err := syscall.Statfs(filepath.Clean(path), &stat)
+	if err != nil {
+		return Usage{}, err
+	}
+	u := Usage{
+		FreeBytes:  int64(stat.Bfree) * int64(stat.Bsize),
+		TotalBytes: int64(stat.Blocks) * int64(stat.Bsize),
+		AvailBytes: int64(stat.Bavail) * int64(stat.Bsize),
+	}
+	return u, nil
+}

Godeps/_workspace/src/github.com/calmh/du/diskusage_unsupported.go

@@ -0,0 +1,13 @@
+// +build netbsd openbsd solaris
+
+package du
+
+import "errors"
+
+var ErrUnsupported = errors.New("unsupported platform")
+
+// Get returns the Usage of a given path, or an error if usage data is
+// unavailable.
+func Get(path string) (Usage, error) {
+	return Usage{}, ErrUnsupported
+}

Godeps/_workspace/src/github.com/calmh/du/diskusage_windows.go

@@ -0,0 +1,27 @@
+package du
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+// Get returns the Usage of a given path, or an error if usage data is
+// unavailable.
+func Get(path string) (Usage, error) {
+	h := syscall.MustLoadDLL("kernel32.dll")
+	c := h.MustFindProc("GetDiskFreeSpaceExW")
+
+	var u Usage
+
+	ret, _, err := c.Call(
+		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
+		uintptr(unsafe.Pointer(&u.FreeBytes)),
+		uintptr(unsafe.Pointer(&u.TotalBytes)),
+		uintptr(unsafe.Pointer(&u.AvailBytes)))
+
+	if ret == 0 {
+		return u, err
+	}
+
+	return u, nil
+}

Godeps/_workspace/src/github.com/calmh/logger/logger.go

@@ -6,6 +6,7 @@ package logger
 
 import (
 	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
 	"strings"
@@ -37,6 +38,13 @@ type Logger struct {
 var DefaultLogger = New()
 
 func New() *Logger {
+	if os.Getenv("LOGGER_DISCARD") != "" {
+		// Hack to completely disable logging, for example when running benchmarks.
+		return &Logger{
+			logger: log.New(ioutil.Discard, "", 0),
+		}
+	}
+
 	return &Logger{
 		logger: log.New(os.Stdout, "", log.Ltime),
 	}

Godeps/_workspace/src/github.com/golang/snappy/AUTHORS

@@ -0,0 +1,14 @@
+# This is the official list of Snappy-Go authors for copyright purposes.
+# This file is distinct from the CONTRIBUTORS files.
+# See the latter for an explanation.
+
+# Names should be added to this file as
+#	Name or Organization <email address>
+# The email address is not required for organizations.
+
+# Please keep the list sorted.
+
+Damian Gryski <dgryski@gmail.com>
+Google Inc.
+Jan Mercl <0xjnml@gmail.com>
+Sebastien Binet <seb.binet@gmail.com>

Godeps/_workspace/src/github.com/golang/snappy/CONTRIBUTORS

@@ -0,0 +1,36 @@
+# This is the official list of people who can contribute
+# (and typically have contributed) code to the Snappy-Go repository.
+# The AUTHORS file lists the copyright holders; this file
+# lists people.  For example, Google employees are listed here
+# but not in AUTHORS, because Google holds the copyright.
+#
+# The submission process automatically checks to make sure
+# that people submitting code are listed in this file (by email address).
+#
+# Names should be added to this file only after verifying that
+# the individual or the individual's organization has agreed to
+# the appropriate Contributor License Agreement, found here:
+#
+#     http://code.google.com/legal/individual-cla-v1.0.html
+#     http://code.google.com/legal/corporate-cla-v1.0.html
+#
+# The agreement for individuals can be filled out on the web.
+#
+# When adding J Random Contributor's name to this file,
+# either J's name or J's organization's name should be
+# added to the AUTHORS file, depending on whether the
+# individual or corporate CLA was used.
+
+# Names should be added to this file like so:
+#     Name <email address>
+
+# Please keep the list sorted.
+
+Damian Gryski <dgryski@gmail.com>
+Jan Mercl <0xjnml@gmail.com>
+Kai Backman <kaib@golang.org>
+Marc-Antoine Ruel <maruel@chromium.org>
+Nigel Tao <nigeltao@golang.org>
+Rob Pike <r@golang.org>
+Russ Cox <rsc@golang.org>
+Sebastien Binet <seb.binet@gmail.com>

Godeps/_workspace/src/github.com/golang/snappy/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2011 The Snappy-Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

Godeps/_workspace/src/github.com/golang/snappy/README

@@ -0,0 +1,7 @@
+The Snappy compression format in the Go programming language.
+
+To download and install from source:
+$ go get github.com/golang/snappy
+
+Unless otherwise noted, the Snappy-Go source files are distributed
+under the BSD-style license found in the LICENSE file.

Godeps/_workspace/src/github.com/golang/snappy/decode.go

@@ -13,6 +13,8 @@ import (
 var (
 	// ErrCorrupt reports that the input is invalid.
 	ErrCorrupt = errors.New("snappy: corrupt input")
+	// ErrTooLarge reports that the uncompressed length is too large.
+	ErrTooLarge = errors.New("snappy: decoded block is too large")
 	// ErrUnsupported reports that the input isn't supported.
 	ErrUnsupported = errors.New("snappy: unsupported input")
 )
@@ -27,11 +29,13 @@ func DecodedLen(src []byte) (int, error) {
 // that the length header occupied.
 func decodedLen(src []byte) (blockLen, headerLen int, err error) {
 	v, n := binary.Uvarint(src)
-	if n == 0 {
+	if n <= 0 || v > 0xffffffff {
 		return 0, 0, ErrCorrupt
 	}
-	if uint64(int(v)) != v {
-		return 0, 0, errors.New("snappy: decoded block is too large")
+
+	const wordSize = 32 << (^uint(0) >> 32 & 1)
+	if wordSize == 32 && v > 0x7fffffff {
+		return 0, 0, ErrTooLarge
 	}
 	return int(v), n, nil
 }
@@ -56,7 +60,7 @@ func Decode(dst, src []byte) ([]byte, error) {
 			x := uint(src[s] >> 2)
 			switch {
 			case x < 60:
-				s += 1
+				s++
 			case x == 60:
 				s += 2
 				if s > len(src) {
@@ -130,7 +134,7 @@ func Decode(dst, src []byte) ([]byte, error) {
 
 // NewReader returns a new Reader that decompresses from r, using the framing
 // format described at
-// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
+// https://github.com/google/snappy/blob/master/framing_format.txt
 func NewReader(r io.Reader) *Reader {
 	return &Reader{
 		r:       r,
@@ -200,7 +204,7 @@ func (r *Reader) Read(p []byte) (int, error) {
 		}
 
 		// The chunk types are specified at
-		// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
+		// https://github.com/google/snappy/blob/master/framing_format.txt
 		switch chunkType {
 		case chunkTypeCompressedData:
 			// Section 4.2. Compressed data (chunk type 0x00).
@@ -280,13 +284,11 @@ func (r *Reader) Read(p []byte) (int, error) {
 			// Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f).
 			r.err = ErrUnsupported
 			return 0, r.err
-
-		} else {
-			// Section 4.4 Padding (chunk type 0xfe).
-			// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
-			if !r.readFull(r.buf[:chunkLen]) {
-				return 0, r.err
-			}
+		}
+		// Section 4.4 Padding (chunk type 0xfe).
+		// Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd).
+		if !r.readFull(r.buf[:chunkLen]) {
+			return 0, r.err
 		}
 	}
 }

Godeps/_workspace/src/github.com/golang/snappy/encode.go

@@ -79,7 +79,7 @@ func emitCopy(dst []byte, offset, length int) int {
 // slice of dst if dst was large enough to hold the entire encoded block.
 // Otherwise, a newly allocated slice will be returned.
 // It is valid to pass a nil dst.
-func Encode(dst, src []byte) ([]byte, error) {
+func Encode(dst, src []byte) []byte {
 	if n := MaxEncodedLen(len(src)); len(dst) < n {
 		dst = make([]byte, n)
 	}
@@ -92,7 +92,7 @@ func Encode(dst, src []byte) ([]byte, error) {
 		if len(src) != 0 {
 			d += emitLiteral(dst[d:], src)
 		}
-		return dst[:d], nil
+		return dst[:d]
 	}
 
 	// Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive.
@@ -145,7 +145,7 @@ func Encode(dst, src []byte) ([]byte, error) {
 	if lit != len(src) {
 		d += emitLiteral(dst[d:], src[lit:])
 	}
-	return dst[:d], nil
+	return dst[:d]
 }
 
 // MaxEncodedLen returns the maximum length of a snappy block, given its
@@ -176,7 +176,7 @@ func MaxEncodedLen(srcLen int) int {
 
 // NewWriter returns a new Writer that compresses to w, using the framing
 // format described at
-// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
+// https://github.com/google/snappy/blob/master/framing_format.txt
 func NewWriter(w io.Writer) *Writer {
 	return &Writer{
 		w:   w,
@@ -226,11 +226,7 @@ func (w *Writer) Write(p []byte) (n int, errRet error) {
 		// Compress the buffer, discarding the result if the improvement
 		// isn't at least 12.5%.
 		chunkType := uint8(chunkTypeCompressedData)
-		chunkBody, err := Encode(w.enc, uncompressed)
-		if err != nil {
-			w.err = err
-			return n, err
-		}
+		chunkBody := Encode(w.enc, uncompressed)
 		if len(chunkBody) >= len(uncompressed)-len(uncompressed)/8 {
 			chunkType, chunkBody = chunkTypeUncompressedData, uncompressed
 		}
@@ -244,11 +240,11 @@ func (w *Writer) Write(p []byte) (n int, errRet error) {
 		w.buf[5] = uint8(checksum >> 8)
 		w.buf[6] = uint8(checksum >> 16)
 		w.buf[7] = uint8(checksum >> 24)
-		if _, err = w.w.Write(w.buf[:]); err != nil {
+		if _, err := w.w.Write(w.buf[:]); err != nil {
 			w.err = err
 			return n, err
 		}
-		if _, err = w.w.Write(chunkBody); err != nil {
+		if _, err := w.w.Write(chunkBody); err != nil {
 			w.err = err
 			return n, err
 		}

Godeps/_workspace/src/github.com/golang/snappy/snappy.go

@@ -5,7 +5,7 @@
 // Package snappy implements the snappy block-based compression format.
 // It aims for very high speeds and reasonable compression.
 //
-// The C++ snappy implementation is at http://code.google.com/p/snappy/
+// The C++ snappy implementation is at https://github.com/google/snappy
 package snappy
 
 import (
@@ -46,7 +46,7 @@ const (
 	chunkHeaderSize = 4
 	magicChunk      = "\xff\x06\x00\x00" + magicBody
 	magicBody       = "sNaPpY"
-	// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt says
+	// https://github.com/google/snappy/blob/master/framing_format.txt says
 	// that "the uncompressed data in a chunk must be no longer than 65536 bytes".
 	maxUncompressedChunkLen = 65536
 )
@@ -61,7 +61,7 @@ const (
 var crcTable = crc32.MakeTable(crc32.Castagnoli)
 
 // crc implements the checksum specified in section 3 of
-// https://code.google.com/p/snappy/source/browse/trunk/framing_format.txt
+// https://github.com/google/snappy/blob/master/framing_format.txt
 func crc(b []byte) uint32 {
 	c := crc32.Update(0, crcTable, b)
 	return uint32(c>>15|c<<17) + 0xa282ead8

Godeps/_workspace/src/github.com/golang/snappy/snappy_test.go

@@ -24,11 +24,7 @@ var (
 )
 
 func roundtrip(b, ebuf, dbuf []byte) error {
-	e, err := Encode(ebuf, b)
-	if err != nil {
-		return fmt.Errorf("encoding error: %v", err)
-	}
-	d, err := Decode(dbuf, e)
+	d, err := Decode(dbuf, Encode(ebuf, b))
 	if err != nil {
 		return fmt.Errorf("decoding error: %v", err)
 	}
@@ -82,6 +78,26 @@ func TestSmallRegular(t *testing.T) {
 	}
 }
 
+func TestInvalidVarint(t *testing.T) {
+	data := []byte("\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x00")
+	if _, err := DecodedLen(data); err != ErrCorrupt {
+		t.Errorf("DecodedLen: got %v, want ErrCorrupt", err)
+	}
+	if _, err := Decode(nil, data); err != ErrCorrupt {
+		t.Errorf("Decode: got %v, want ErrCorrupt", err)
+	}
+
+	// The encoded varint overflows 32 bits
+	data = []byte("\xff\xff\xff\xff\xff\x00")
+
+	if _, err := DecodedLen(data); err != ErrCorrupt {
+		t.Errorf("DecodedLen: got %v, want ErrCorrupt", err)
+	}
+	if _, err := Decode(nil, data); err != ErrCorrupt {
+		t.Errorf("Decode: got %v, want ErrCorrupt", err)
+	}
+}
+
 func cmp(a, b []byte) error {
 	if len(a) != len(b) {
 		return fmt.Errorf("got %d bytes, want %d", len(a), len(b))
@@ -197,10 +213,7 @@ func TestWriterReset(t *testing.T) {
 }
 
 func benchDecode(b *testing.B, src []byte) {
-	encoded, err := Encode(nil, src)
-	if err != nil {
-		b.Fatal(err)
-	}
+	encoded := Encode(nil, src)
 	// Bandwidth is in amount of uncompressed data.
 	b.SetBytes(int64(len(src)))
 	b.ResetTimer()
@@ -222,7 +235,7 @@ func benchEncode(b *testing.B, src []byte) {
 func readFile(b testing.TB, filename string) []byte {
 	src, err := ioutil.ReadFile(filename)
 	if err != nil {
-		b.Fatalf("failed reading %s: %s", filename, err)
+		b.Skipf("skipping benchmark: %v", err)
 	}
 	if len(src) == 0 {
 		b.Fatalf("%s has zero length", filename)
@@ -284,14 +297,14 @@ var testFiles = []struct {
 // The test data files are present at this canonical URL.
 const baseURL = "https://raw.githubusercontent.com/google/snappy/master/testdata/"
 
-func downloadTestdata(basename string) (errRet error) {
+func downloadTestdata(b *testing.B, basename string) (errRet error) {
 	filename := filepath.Join(*testdata, basename)
 	if stat, err := os.Stat(filename); err == nil && stat.Size() != 0 {
 		return nil
 	}
 
 	if !*download {
-		return fmt.Errorf("test data not found; skipping benchmark without the -download flag")
+		b.Skipf("test data not found; skipping benchmark without the -download flag")
 	}
 	// Download the official snappy C++ implementation reference test data
 	// files for benchmarking.
@@ -326,7 +339,7 @@ func downloadTestdata(basename string) (errRet error) {
 }
 
 func benchFile(b *testing.B, n int, decode bool) {
-	if err := downloadTestdata(testFiles[n].filename); err != nil {
+	if err := downloadTestdata(b, testFiles[n].filename); err != nil {
 		b.Fatalf("failed to download testdata: %s", err)
 	}
 	data := readFile(b, filepath.Join(*testdata, testFiles[n].filename))

Godeps/_workspace/src/github.com/juju/ratelimit/LICENSE

@@ -1,5 +1,8 @@
-This package contains an efficient token-bucket-based rate limiter.
-Copyright (C) 2015 Canonical Ltd.
+All files in this repository are licensed as follows. If you contribute
+to this repository, it is assumed that you license your contribution
+under the same license unless you state otherwise.
+
+All files Copyright (C) 2015 Canonical Ltd. unless otherwise specified in the file.
 
 This software is licensed under the LGPLv3, included below.
 

Godeps/_workspace/src/github.com/juju/ratelimit/README.md

@@ -20,7 +20,7 @@ token in the bucket represents one byte.
 ```go
 func Writer(w io.Writer, bucket *Bucket) io.Writer
 ```
-Writer returns a reader that is rate limited by the given token bucket. Each
+Writer returns a writer that is rate limited by the given token bucket. Each
 token in the bucket represents one byte.
 
 #### type Bucket

Godeps/_workspace/src/github.com/juju/ratelimit/ratelimit.go

@@ -2,7 +2,8 @@
 // Licensed under the LGPLv3 with static-linking exception.
 // See LICENCE file for details.
 
-// The ratelimit package provides an efficient token bucket implementation.
+// The ratelimit package provides an efficient token bucket implementation
+// that can be used to limit the rate of arbitrary things.
 // See http://en.wikipedia.org/wiki/Token_bucket.
 package ratelimit
 
@@ -10,6 +11,7 @@ import (
 	"strconv"
 	"sync"
 	"time"
+	"math"
 )
 
 // Bucket represents a token bucket that fills at a predetermined rate.
@@ -55,7 +57,7 @@ func NewBucketWithRate(rate float64, capacity int64) *Bucket {
 			continue
 		}
 		tb := NewBucketWithQuantum(fillInterval, capacity, quantum)
-		if diff := abs(tb.Rate() - rate); diff/rate <= rateMargin {
+		if diff := math.Abs(tb.Rate() - rate); diff/rate <= rateMargin {
 			return tb
 		}
 	}
@@ -217,10 +219,3 @@ func (tb *Bucket) adjust(now time.Time) (currentTick int64) {
 	tb.availTick = currentTick
 	return
 }
-
-func abs(f float64) float64 {
-	if f < 0 {
-		return -f
-	}
-	return f
-}

Godeps/_workspace/src/github.com/kardianos/osext/README.md

@@ -4,7 +4,9 @@
 
 There is sometimes utility in finding the current executable file
 that is running. This can be used for upgrading the current executable
-or finding resources located relative to the executable file.
+or finding resources located relative to the executable file. Both
+working directory and the os.Args[0] value are arbitrary and cannot
+be relied on; os.Args[0] can be "faked".
 
 Multi-platform and supports:
  * Linux

Godeps/_workspace/src/github.com/kardianos/osext/osext.go

@@ -16,12 +16,12 @@ func Executable() (string, error) {
 }
 
 // Returns same path as Executable, returns just the folder
-// path. Excludes the executable name.
+// path. Excludes the executable name and any trailing slash.
 func ExecutableFolder() (string, error) {
 	p, err := Executable()
 	if err != nil {
 		return "", err
 	}
-	folder, _ := filepath.Split(p)
-	return folder, nil
+
+	return filepath.Dir(p), nil
 }

Godeps/_workspace/src/github.com/kardianos/osext/osext_procfs.go

@@ -17,12 +17,14 @@ import (
 func executable() (string, error) {
 	switch runtime.GOOS {
 	case "linux":
-		const deletedSuffix = " (deleted)"
+		const deletedTag = " (deleted)"
 		execpath, err := os.Readlink("/proc/self/exe")
 		if err != nil {
 			return execpath, err
 		}
-		return strings.TrimSuffix(execpath, deletedSuffix), nil
+		execpath = strings.TrimSuffix(execpath, deletedTag)
+		execpath = strings.TrimPrefix(execpath, deletedTag)
+		return execpath, nil
 	case "netbsd":
 		return os.Readlink("/proc/curproc/exe")
 	case "openbsd", "dragonfly":

Godeps/_workspace/src/github.com/kardianos/osext/osext_test.go

@@ -24,6 +24,29 @@ const (
 	executableEnvValueDelete = "delete"
 )
 
+func TestPrintExecutable(t *testing.T) {
+	ef, err := Executable()
+	if err != nil {
+		t.Fatalf("Executable failed: %v", err)
+	}
+	t.Log("Executable:", ef)
+}
+func TestPrintExecutableFolder(t *testing.T) {
+	ef, err := ExecutableFolder()
+	if err != nil {
+		t.Fatalf("ExecutableFolder failed: %v", err)
+	}
+	t.Log("Executable Folder:", ef)
+}
+func TestExecutableFolder(t *testing.T) {
+	ef, err := ExecutableFolder()
+	if err != nil {
+		t.Fatalf("ExecutableFolder failed: %v", err)
+	}
+	if ef[len(ef)-1] == filepath.Separator {
+		t.Fatal("ExecutableFolder ends with a trailing slash.")
+	}
+}
 func TestExecutableMatch(t *testing.T) {
 	ep, err := Executable()
 	if err != nil {

Godeps/_workspace/src/github.com/syncthing/protocol/common_test.go

@@ -31,15 +31,16 @@ func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo, fl
 func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
 }
 
-func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
+func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
 	t.folder = folder
 	t.name = name
 	t.offset = offset
-	t.size = size
+	t.size = len(buf)
 	t.hash = hash
 	t.flags = flags
 	t.options = options
-	return t.data, nil
+	copy(buf, t.data)
+	return nil
 }
 
 func (t *TestModel) Close(deviceID DeviceID, err error) {

Godeps/_workspace/src/github.com/syncthing/protocol/conflict_test.go

@@ -0,0 +1,23 @@
+// Copyright (C) 2015 The Protocol Authors.
+
+package protocol
+
+import "testing"
+
+func TestWinsConflict(t *testing.T) {
+	testcases := [][2]FileInfo{
+		// The first should always win over the second
+		{{Modified: 42}, {Modified: 41}},
+		{{Modified: 41}, {Modified: 42, Flags: FlagDeleted}},
+		{{Modified: 41, Version: Vector{{42, 2}, {43, 1}}}, {Modified: 41, Version: Vector{{42, 1}, {43, 2}}}},
+	}
+
+	for _, tc := range testcases {
+		if !tc[0].WinsConflict(tc[1]) {
+			t.Errorf("%v should win over %v", tc[0], tc[1])
+		}
+		if tc[1].WinsConflict(tc[0]) {
+			t.Errorf("%v should not win over %v", tc[1], tc[0])
+		}
+	}
+}

Godeps/_workspace/src/github.com/syncthing/protocol/fuzz.go

@@ -0,0 +1,70 @@
+// Copyright (C) 2015 The Protocol Authors.
+
+// +build gofuzz
+
+package protocol
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"reflect"
+	"sync"
+)
+
+func Fuzz(data []byte) int {
+	// Regenerate the length, or we'll most commonly exit quickly due to an
+	// unexpected eof which is unintestering.
+	if len(data) > 8 {
+		binary.BigEndian.PutUint32(data[4:], uint32(len(data))-8)
+	}
+
+	// Setup a rawConnection we'll use to parse the message.
+	c := rawConnection{
+		cr:     &countingReader{Reader: bytes.NewReader(data)},
+		closed: make(chan struct{}),
+		pool: sync.Pool{
+			New: func() interface{} {
+				return make([]byte, BlockSize)
+			},
+		},
+	}
+
+	// Attempt to parse the message.
+	hdr, msg, err := c.readMessage()
+	if err != nil {
+		return 0
+	}
+
+	// If parsing worked, attempt to encode it again.
+	newBs, err := msg.AppendXDR(nil)
+	if err != nil {
+		panic("not encodable")
+	}
+
+	// Create an appriate header for the re-encoding.
+	newMsg := make([]byte, 8)
+	binary.BigEndian.PutUint32(newMsg, encodeHeader(hdr))
+	binary.BigEndian.PutUint32(newMsg[4:], uint32(len(newBs)))
+	newMsg = append(newMsg, newBs...)
+
+	// Use the rawConnection to parse the re-encoding.
+	c.cr = &countingReader{Reader: bytes.NewReader(newMsg)}
+	hdr2, msg2, err := c.readMessage()
+	if err != nil {
+		fmt.Println("Initial:\n" + hex.Dump(data))
+		fmt.Println("New:\n" + hex.Dump(newMsg))
+		panic("not parseable after re-encode: " + err.Error())
+	}
+
+	// Make sure the data is the same as it was before.
+	if hdr != hdr2 {
+		panic("headers differ")
+	}
+	if !reflect.DeepEqual(msg, msg2) {
+		panic("contents differ")
+	}
+
+	return 1
+}

Godeps/_workspace/src/github.com/syncthing/protocol/fuzz_test.go

@@ -0,0 +1,89 @@
+// Copyright (C) 2015 The Protocol Authors.
+
+// +build gofuzz
+
+package protocol
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+	"testing/quick"
+)
+
+// This can be used to generate a corpus of valid messages as a starting point
+// for the fuzzer.
+func TestGenerateCorpus(t *testing.T) {
+	t.Skip("Use to generate initial corpus only")
+
+	n := 0
+	check := func(idx IndexMessage) bool {
+		for i := range idx.Options {
+			if len(idx.Options[i].Key) > 64 {
+				idx.Options[i].Key = idx.Options[i].Key[:64]
+			}
+		}
+		hdr := header{
+			version:     0,
+			msgID:       42,
+			msgType:     messageTypeIndex,
+			compression: false,
+		}
+
+		msgBs := idx.MustMarshalXDR()
+
+		buf := make([]byte, 8)
+		binary.BigEndian.PutUint32(buf, encodeHeader(hdr))
+		binary.BigEndian.PutUint32(buf[4:], uint32(len(msgBs)))
+		buf = append(buf, msgBs...)
+
+		ioutil.WriteFile(fmt.Sprintf("testdata/corpus/test-%03d.xdr", n), buf, 0644)
+		n++
+		return true
+	}
+
+	if err := quick.Check(check, &quick.Config{MaxCount: 1000}); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// Tests any crashers found by the fuzzer, for closer investigation.
+func TestCrashers(t *testing.T) {
+	testFiles(t, "testdata/crashers")
+}
+
+// Tests the entire corpus, which should PASS before the fuzzer starts
+// fuzzing.
+func TestCorpus(t *testing.T) {
+	testFiles(t, "testdata/corpus")
+}
+
+func testFiles(t *testing.T, dir string) {
+	fd, err := os.Open(dir)
+	if err != nil {
+		t.Fatal(err)
+	}
+	crashers, err := fd.Readdirnames(-1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, name := range crashers {
+		if strings.HasSuffix(name, ".output") {
+			continue
+		}
+		if strings.HasSuffix(name, ".quoted") {
+			continue
+		}
+
+		t.Log(name)
+		crasher, err := ioutil.ReadFile(dir + "/" + name)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		Fuzz(crasher)
+	}
+}

Godeps/_workspace/src/github.com/syncthing/protocol/message.go

@@ -9,7 +9,7 @@ import "fmt"
 
 type IndexMessage struct {
 	Folder  string
-	Files   []FileInfo
+	Files   []FileInfo // max:1000000
 	Flags   uint32
 	Options []Option // max:64
 }
@@ -20,7 +20,7 @@ type FileInfo struct {
 	Modified     int64
 	Version      Vector
 	LocalVersion int64
-	Blocks       []BlockInfo
+	Blocks       []BlockInfo // max:1000000
 }
 
 func (f FileInfo) String() string {
@@ -58,6 +58,31 @@ func (f FileInfo) HasPermissionBits() bool {
 	return f.Flags&FlagNoPermBits == 0
 }
 
+// WinsConflict returns true if "f" is the one to choose when it is in
+// conflict with "other".
+func (f FileInfo) WinsConflict(other FileInfo) bool {
+	// If a modification is in conflict with a delete, we pick the
+	// modification.
+	if !f.IsDeleted() && other.IsDeleted() {
+		return true
+	}
+	if f.IsDeleted() && !other.IsDeleted() {
+		return false
+	}
+
+	// The one with the newer modification time wins.
+	if f.Modified > other.Modified {
+		return true
+	}
+	if f.Modified < other.Modified {
+		return false
+	}
+
+	// The modification times were equal. Use the device ID in the version
+	// vector as tie breaker.
+	return f.Version.Compare(other.Version) == ConcurrentGreater
+}
+
 type BlockInfo struct {
 	Offset int64 // noencode (cache only)
 	Size   int32
@@ -84,9 +109,9 @@ type ResponseMessage struct {
 }
 
 type ClusterConfigMessage struct {
-	ClientName    string // max:64
-	ClientVersion string // max:64
-	Folders       []Folder
+	ClientName    string   // max:64
+	ClientVersion string   // max:64
+	Folders       []Folder // max:1000000
 	Options       []Option // max:64
 }
 
@@ -100,8 +125,8 @@ func (o *ClusterConfigMessage) GetOption(key string) string {
 }
 
 type Folder struct {
-	ID      string // max:64
-	Devices []Device
+	ID      string   // max:64
+	Devices []Device // max:1000000
 	Flags   uint32
 	Options []Option // max:64
 }

Godeps/_workspace/src/github.com/syncthing/protocol/message_xdr.go

@@ -42,7 +42,7 @@ IndexMessage Structure:
 
 struct IndexMessage {
 	string Folder<>;
-	FileInfo Files<>;
+	FileInfo Files<1000000>;
 	unsigned int Flags;
 	Option Options<64>;
 }
@@ -75,6 +75,9 @@ func (o IndexMessage) AppendXDR(bs []byte) ([]byte, error) {
 
 func (o IndexMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 	xw.WriteString(o.Folder)
+	if l := len(o.Files); l > 1000000 {
+		return xw.Tot(), xdr.ElementSizeExceeded("Files", l, 1000000)
+	}
 	xw.WriteUint32(uint32(len(o.Files)))
 	for i := range o.Files {
 		_, err := o.Files[i].EncodeXDRInto(xw)
@@ -111,7 +114,10 @@ func (o *IndexMessage) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.Folder = xr.ReadString()
 	_FilesSize := int(xr.ReadUint32())
 	if _FilesSize < 0 {
-		return xdr.ElementSizeExceeded("Files", _FilesSize, 0)
+		return xdr.ElementSizeExceeded("Files", _FilesSize, 1000000)
+	}
+	if _FilesSize > 1000000 {
+		return xdr.ElementSizeExceeded("Files", _FilesSize, 1000000)
 	}
 	o.Files = make([]FileInfo, _FilesSize)
 	for i := range o.Files {
@@ -173,7 +179,7 @@ struct FileInfo {
 	hyper Modified;
 	Vector Version;
 	hyper LocalVersion;
-	BlockInfo Blocks<>;
+	BlockInfo Blocks<1000000>;
 }
 
 */
@@ -214,6 +220,9 @@ func (o FileInfo) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 		return xw.Tot(), err
 	}
 	xw.WriteUint64(uint64(o.LocalVersion))
+	if l := len(o.Blocks); l > 1000000 {
+		return xw.Tot(), xdr.ElementSizeExceeded("Blocks", l, 1000000)
+	}
 	xw.WriteUint32(uint32(len(o.Blocks)))
 	for i := range o.Blocks {
 		_, err := o.Blocks[i].EncodeXDRInto(xw)
@@ -243,7 +252,10 @@ func (o *FileInfo) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.LocalVersion = int64(xr.ReadUint64())
 	_BlocksSize := int(xr.ReadUint32())
 	if _BlocksSize < 0 {
-		return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 0)
+		return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 1000000)
+	}
+	if _BlocksSize > 1000000 {
+		return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 1000000)
 	}
 	o.Blocks = make([]BlockInfo, _BlocksSize)
 	for i := range o.Blocks {
@@ -571,7 +583,7 @@ ClusterConfigMessage Structure:
 struct ClusterConfigMessage {
 	string ClientName<64>;
 	string ClientVersion<64>;
-	Folder Folders<>;
+	Folder Folders<1000000>;
 	Option Options<64>;
 }
 
@@ -610,6 +622,9 @@ func (o ClusterConfigMessage) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 		return xw.Tot(), xdr.ElementSizeExceeded("ClientVersion", l, 64)
 	}
 	xw.WriteString(o.ClientVersion)
+	if l := len(o.Folders); l > 1000000 {
+		return xw.Tot(), xdr.ElementSizeExceeded("Folders", l, 1000000)
+	}
 	xw.WriteUint32(uint32(len(o.Folders)))
 	for i := range o.Folders {
 		_, err := o.Folders[i].EncodeXDRInto(xw)
@@ -646,7 +661,10 @@ func (o *ClusterConfigMessage) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.ClientVersion = xr.ReadStringMax(64)
 	_FoldersSize := int(xr.ReadUint32())
 	if _FoldersSize < 0 {
-		return xdr.ElementSizeExceeded("Folders", _FoldersSize, 0)
+		return xdr.ElementSizeExceeded("Folders", _FoldersSize, 1000000)
+	}
+	if _FoldersSize > 1000000 {
+		return xdr.ElementSizeExceeded("Folders", _FoldersSize, 1000000)
 	}
 	o.Folders = make([]Folder, _FoldersSize)
 	for i := range o.Folders {
@@ -697,7 +715,7 @@ Folder Structure:
 
 struct Folder {
 	string ID<64>;
-	Device Devices<>;
+	Device Devices<1000000>;
 	unsigned int Flags;
 	Option Options<64>;
 }
@@ -733,6 +751,9 @@ func (o Folder) EncodeXDRInto(xw *xdr.Writer) (int, error) {
 		return xw.Tot(), xdr.ElementSizeExceeded("ID", l, 64)
 	}
 	xw.WriteString(o.ID)
+	if l := len(o.Devices); l > 1000000 {
+		return xw.Tot(), xdr.ElementSizeExceeded("Devices", l, 1000000)
+	}
 	xw.WriteUint32(uint32(len(o.Devices)))
 	for i := range o.Devices {
 		_, err := o.Devices[i].EncodeXDRInto(xw)
@@ -769,7 +790,10 @@ func (o *Folder) DecodeXDRFrom(xr *xdr.Reader) error {
 	o.ID = xr.ReadStringMax(64)
 	_DevicesSize := int(xr.ReadUint32())
 	if _DevicesSize < 0 {
-		return xdr.ElementSizeExceeded("Devices", _DevicesSize, 0)
+		return xdr.ElementSizeExceeded("Devices", _DevicesSize, 1000000)
+	}
+	if _DevicesSize > 1000000 {
+		return xdr.ElementSizeExceeded("Devices", _DevicesSize, 1000000)
 	}
 	o.Devices = make([]Device, _DevicesSize)
 	for i := range o.Devices {

Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_darwin.go

@@ -26,9 +26,9 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI
 	m.next.IndexUpdate(deviceID, folder, files, flags, options)
 }
 
-func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
+func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
 	name = norm.NFD.String(name)
-	return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options)
+	return m.next.Request(deviceID, folder, name, offset, hash, flags, options, buf)
 }
 
 func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {

Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_unix.go

@@ -18,8 +18,8 @@ func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileI
 	m.next.IndexUpdate(deviceID, folder, files, flags, options)
 }
 
-func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
-	return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options)
+func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
+	return m.next.Request(deviceID, folder, name, offset, hash, flags, options, buf)
 }
 
 func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {

Godeps/_workspace/src/github.com/syncthing/protocol/nativemodel_windows.go

@@ -25,18 +25,18 @@ type nativeModel struct {
 }
 
 func (m nativeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
-	fixupFiles(files)
+	fixupFiles(folder, files)
 	m.next.Index(deviceID, folder, files, flags, options)
 }
 
 func (m nativeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
-	fixupFiles(files)
+	fixupFiles(folder, files)
 	m.next.IndexUpdate(deviceID, folder, files, flags, options)
 }
 
-func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error) {
+func (m nativeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
 	name = filepath.FromSlash(name)
-	return m.next.Request(deviceID, folder, name, offset, size, hash, flags, options)
+	return m.next.Request(deviceID, folder, name, offset, hash, flags, options, buf)
 }
 
 func (m nativeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
@@ -47,7 +47,7 @@ func (m nativeModel) Close(deviceID DeviceID, err error) {
 	m.next.Close(deviceID, err)
 }
 
-func fixupFiles(files []FileInfo) {
+func fixupFiles(folder string, files []FileInfo) {
 	for i, f := range files {
 		if strings.ContainsAny(f.Name, disallowedCharacters) {
 			if f.IsDeleted() {
@@ -56,7 +56,7 @@ func fixupFiles(files []FileInfo) {
 				continue
 			}
 			files[i].Flags |= FlagInvalid
-			l.Warnf("File name %q contains invalid characters; marked as invalid.", f.Name)
+			l.Warnf("File name %q (folder %q) contains invalid characters; marked as invalid.", f.Name, folder)
 		}
 		files[i].Name = filepath.FromSlash(files[i].Name)
 	}

Godeps/_workspace/src/github.com/syncthing/protocol/protocol.go

@@ -15,7 +15,11 @@ import (
 )
 
 const (
-	BlockSize = 128 * 1024
+	// Data block size (128 KiB)
+	BlockSize = 128 << 10
+
+	// We reject messages larger than this when encountered on the wire. (64 MiB)
+	MaxMessageLen = 64 << 20
 )
 
 const (
@@ -31,8 +35,7 @@ const (
 
 const (
 	stateInitial = iota
-	stateCCRcvd
-	stateIdxRcvd
+	stateReady
 )
 
 // FileInfo flags
@@ -82,7 +85,7 @@ type Model interface {
 	// An index update was received from the peer device
 	IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option)
 	// A request was made by the peer device
-	Request(deviceID DeviceID, folder string, name string, offset int64, size int, hash []byte, flags uint32, options []Option) ([]byte, error)
+	Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error
 	// A cluster configuration message was received
 	ClusterConfig(deviceID DeviceID, config ClusterConfigMessage)
 	// The peer device closed the connection
@@ -90,6 +93,7 @@ type Model interface {
 }
 
 type Connection interface {
+	Start()
 	ID() DeviceID
 	Name() string
 	Index(folder string, files []FileInfo, flags uint32, options []Option) error
@@ -103,7 +107,6 @@ type rawConnection struct {
 	id       DeviceID
 	name     string
 	receiver Model
-	state    int
 
 	cr *countingReader
 	cw *countingWriter
@@ -113,11 +116,11 @@ type rawConnection struct {
 
 	idxMut sync.Mutex // ensures serialization of Index calls
 
-	nextID chan int
-	outbox chan hdrMsg
-	closed chan struct{}
-	once   sync.Once
-
+	nextID      chan int
+	outbox      chan hdrMsg
+	closed      chan struct{}
+	once        sync.Once
+	pool        sync.Pool
 	compression Compression
 
 	rdbuf0 []byte // used & reused by readMessage
@@ -130,8 +133,9 @@ type asyncResult struct {
 }
 
 type hdrMsg struct {
-	hdr header
-	msg encodable
+	hdr  header
+	msg  encodable
+	done chan struct{}
 }
 
 type encodable interface {
@@ -142,9 +146,9 @@ type isEofer interface {
 	IsEOF() bool
 }
 
-const (
-	pingTimeout  = 30 * time.Second
-	pingIdleTime = 60 * time.Second
+var (
+	PingTimeout  = 30 * time.Second
+	PingIdleTime = 60 * time.Second
 )
 
 func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiver Model, name string, compress Compression) Connection {
@@ -152,24 +156,32 @@ func NewConnection(deviceID DeviceID, reader io.Reader, writer io.Writer, receiv
 	cw := &countingWriter{Writer: writer}
 
 	c := rawConnection{
-		id:          deviceID,
-		name:        name,
-		receiver:    nativeModel{receiver},
-		state:       stateInitial,
-		cr:          cr,
-		cw:          cw,
-		outbox:      make(chan hdrMsg),
-		nextID:      make(chan int),
-		closed:      make(chan struct{}),
+		id:       deviceID,
+		name:     name,
+		receiver: nativeModel{receiver},
+		cr:       cr,
+		cw:       cw,
+		outbox:   make(chan hdrMsg),
+		nextID:   make(chan int),
+		closed:   make(chan struct{}),
+		pool: sync.Pool{
+			New: func() interface{} {
+				return make([]byte, BlockSize)
+			},
+		},
 		compression: compress,
 	}
 
+	return wireFormatConnection{&c}
+}
+
+// Start creates the goroutines for sending and receiving of messages. It must
+// be called exactly once after creating a connection.
+func (c *rawConnection) Start() {
 	go c.readerLoop()
 	go c.writerLoop()
 	go c.pingerLoop()
 	go c.idGenerator()
-
-	return wireFormatConnection{&c}
 }
 
 func (c *rawConnection) ID() DeviceID {
@@ -193,7 +205,7 @@ func (c *rawConnection) Index(folder string, idx []FileInfo, flags uint32, optio
 		Files:   idx,
 		Flags:   flags,
 		Options: options,
-	})
+	}, nil)
 	c.idxMut.Unlock()
 	return nil
 }
@@ -211,7 +223,7 @@ func (c *rawConnection) IndexUpdate(folder string, idx []FileInfo, flags uint32,
 		Files:   idx,
 		Flags:   flags,
 		Options: options,
-	})
+	}, nil)
 	c.idxMut.Unlock()
 	return nil
 }
@@ -241,7 +253,7 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i
 		Hash:    hash,
 		Flags:   flags,
 		Options: options,
-	})
+	}, nil)
 	if !ok {
 		return nil, ErrClosed
 	}
@@ -255,7 +267,7 @@ func (c *rawConnection) Request(folder string, name string, offset int64, size i
 
 // ClusterConfig send the cluster configuration message to the peer and returns any error
 func (c *rawConnection) ClusterConfig(config ClusterConfigMessage) {
-	c.send(-1, messageTypeClusterConfig, config)
+	c.send(-1, messageTypeClusterConfig, config, nil)
 }
 
 func (c *rawConnection) ping() bool {
@@ -271,7 +283,7 @@ func (c *rawConnection) ping() bool {
 	c.awaiting[id] = rc
 	c.awaitingMut.Unlock()
 
-	ok := c.send(id, messageTypePing, nil)
+	ok := c.send(id, messageTypePing, nil, nil)
 	if !ok {
 		return false
 	}
@@ -285,6 +297,7 @@ func (c *rawConnection) readerLoop() (err error) {
 		c.close(err)
 	}()
 
+	state := stateInitial
 	for {
 		select {
 		case <-c.closed:
@@ -298,47 +311,54 @@ func (c *rawConnection) readerLoop() (err error) {
 		}
 
 		switch msg := msg.(type) {
+		case ClusterConfigMessage:
+			if state != stateInitial {
+				return fmt.Errorf("protocol error: cluster config message in state %d", state)
+			}
+			go c.receiver.ClusterConfig(c.id, msg)
+			state = stateReady
+
 		case IndexMessage:
 			switch hdr.msgType {
 			case messageTypeIndex:
-				if c.state < stateCCRcvd {
-					return fmt.Errorf("protocol error: index message in state %d", c.state)
+				if state != stateReady {
+					return fmt.Errorf("protocol error: index message in state %d", state)
 				}
 				c.handleIndex(msg)
-				c.state = stateIdxRcvd
+				state = stateReady
 
 			case messageTypeIndexUpdate:
-				if c.state < stateIdxRcvd {
-					return fmt.Errorf("protocol error: index update message in state %d", c.state)
+				if state != stateReady {
+					return fmt.Errorf("protocol error: index update message in state %d", state)
 				}
 				c.handleIndexUpdate(msg)
+				state = stateReady
 			}
 
 		case RequestMessage:
-			if c.state < stateIdxRcvd {
-				return fmt.Errorf("protocol error: request message in state %d", c.state)
+			if state != stateReady {
+				return fmt.Errorf("protocol error: request message in state %d", state)
 			}
 			// Requests are handled asynchronously
 			go c.handleRequest(hdr.msgID, msg)
 
 		case ResponseMessage:
-			if c.state < stateIdxRcvd {
-				return fmt.Errorf("protocol error: response message in state %d", c.state)
+			if state != stateReady {
+				return fmt.Errorf("protocol error: response message in state %d", state)
 			}
 			c.handleResponse(hdr.msgID, msg)
 
 		case pingMessage:
-			c.send(hdr.msgID, messageTypePong, pongMessage{})
+			if state != stateReady {
+				return fmt.Errorf("protocol error: ping message in state %d", state)
+			}
+			c.send(hdr.msgID, messageTypePong, pongMessage{}, nil)
 
 		case pongMessage:
-			c.handlePong(hdr.msgID)
-
-		case ClusterConfigMessage:
-			if c.state != stateInitial {
-				return fmt.Errorf("protocol error: cluster config message in state %d", c.state)
+			if state != stateReady {
+				return fmt.Errorf("protocol error: pong message in state %d", state)
 			}
-			go c.receiver.ClusterConfig(c.id, msg)
-			c.state = stateCCRcvd
+			c.handlePong(hdr.msgID)
 
 		case CloseMessage:
 			return errors.New(msg.Reason)
@@ -367,6 +387,11 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) {
 		l.Debugf("read header %v (msglen=%d)", hdr, msglen)
 	}
 
+	if msglen > MaxMessageLen {
+		err = fmt.Errorf("message length %d exceeds maximum %d", msglen, MaxMessageLen)
+		return
+	}
+
 	if hdr.version != 0 {
 		err = fmt.Errorf("unknown protocol version 0x%x", hdr.version)
 		return
@@ -387,7 +412,7 @@ func (c *rawConnection) readMessage() (hdr header, msg encodable, err error) {
 	}
 
 	msgBuf := c.rdbuf0
-	if hdr.compression {
+	if hdr.compression && msglen > 0 {
 		c.rdbuf1 = c.rdbuf1[:cap(c.rdbuf1)]
 		c.rdbuf1, err = lz4.Decode(c.rdbuf1, c.rdbuf0)
 		if err != nil {
@@ -509,12 +534,36 @@ func filterIndexMessageFiles(fs []FileInfo) []FileInfo {
 }
 
 func (c *rawConnection) handleRequest(msgID int, req RequestMessage) {
-	data, err := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), int(req.Size), req.Hash, req.Flags, req.Options)
+	size := int(req.Size)
+	usePool := size <= BlockSize
 
-	c.send(msgID, messageTypeResponse, ResponseMessage{
-		Data: data,
-		Code: errorToCode(err),
-	})
+	var buf []byte
+	var done chan struct{}
+
+	if usePool {
+		buf = c.pool.Get().([]byte)[:size]
+		done = make(chan struct{})
+	} else {
+		buf = make([]byte, size)
+	}
+
+	err := c.receiver.Request(c.id, req.Folder, req.Name, int64(req.Offset), req.Hash, req.Flags, req.Options, buf)
+	if err != nil {
+		c.send(msgID, messageTypeResponse, ResponseMessage{
+			Data: nil,
+			Code: errorToCode(err),
+		}, done)
+	} else {
+		c.send(msgID, messageTypeResponse, ResponseMessage{
+			Data: buf,
+			Code: errorToCode(err),
+		}, done)
+	}
+
+	if usePool {
+		<-done
+		c.pool.Put(buf)
+	}
 }
 
 func (c *rawConnection) handleResponse(msgID int, resp ResponseMessage) {
@@ -537,7 +586,7 @@ func (c *rawConnection) handlePong(msgID int) {
 	c.awaitingMut.Unlock()
 }
 
-func (c *rawConnection) send(msgID int, msgType int, msg encodable) bool {
+func (c *rawConnection) send(msgID int, msgType int, msg encodable, done chan struct{}) bool {
 	if msgID < 0 {
 		select {
 		case id := <-c.nextID:
@@ -554,7 +603,7 @@ func (c *rawConnection) send(msgID int, msgType int, msg encodable) bool {
 	}
 
 	select {
-	case c.outbox <- hdrMsg{hdr, msg}:
+	case c.outbox <- hdrMsg{hdr, msg, done}:
 		return true
 	case <-c.closed:
 		return false
@@ -573,6 +622,9 @@ func (c *rawConnection) writerLoop() {
 			if hm.msg != nil {
 				// Uncompressed message in uncBuf
 				uncBuf, err = hm.msg.AppendXDR(uncBuf[:0])
+				if hm.done != nil {
+					close(hm.done)
+				}
 				if err != nil {
 					c.close(err)
 					return
@@ -679,17 +731,17 @@ func (c *rawConnection) idGenerator() {
 
 func (c *rawConnection) pingerLoop() {
 	var rc = make(chan bool, 1)
-	ticker := time.Tick(pingIdleTime / 2)
+	ticker := time.Tick(PingIdleTime / 2)
 	for {
 		select {
 		case <-ticker:
-			if d := time.Since(c.cr.Last()); d < pingIdleTime {
+			if d := time.Since(c.cr.Last()); d < PingIdleTime {
 				if debug {
 					l.Debugln(c.id, "ping skipped after rd", d)
 				}
 				continue
 			}
-			if d := time.Since(c.cw.Last()); d < pingIdleTime {
+			if d := time.Since(c.cw.Last()); d < PingIdleTime {
 				if debug {
 					l.Debugln(c.id, "ping skipped after wr", d)
 				}
@@ -709,7 +761,7 @@ func (c *rawConnection) pingerLoop() {
 				if !ok {
 					c.close(fmt.Errorf("ping failure"))
 				}
-			case <-time.After(pingTimeout):
+			case <-time.After(PingTimeout):
 				c.close(fmt.Errorf("ping timeout"))
 			case <-c.closed:
 				return

Godeps/_workspace/src/github.com/syncthing/protocol/protocol_test.go

@@ -67,8 +67,12 @@ func TestPing(t *testing.T) {
 	ar, aw := io.Pipe()
 	br, bw := io.Pipe()
 
-	c0 := NewConnection(c0ID, ar, bw, nil, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
-	c1 := NewConnection(c1ID, br, aw, nil, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
+	c0 := NewConnection(c0ID, ar, bw, newTestModel(), "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
+	c0.Start()
+	c1 := NewConnection(c1ID, br, aw, newTestModel(), "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
+	c1.Start()
+	c0.ClusterConfig(ClusterConfigMessage{})
+	c1.ClusterConfig(ClusterConfigMessage{})
 
 	if ok := c0.ping(); !ok {
 		t.Error("c0 ping failed")
@@ -81,8 +85,8 @@ func TestPing(t *testing.T) {
 func TestPingErr(t *testing.T) {
 	e := errors.New("something broke")
 
-	for i := 0; i < 16; i++ {
-		for j := 0; j < 16; j++ {
+	for i := 0; i < 32; i++ {
+		for j := 0; j < 32; j++ {
 			m0 := newTestModel()
 			m1 := newTestModel()
 
@@ -92,12 +96,18 @@ func TestPingErr(t *testing.T) {
 			ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e}
 
 			c0 := NewConnection(c0ID, ar, ebw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
-			NewConnection(c1ID, br, eaw, m1, "name", CompressAlways)
+			c0.Start()
+			c1 := NewConnection(c1ID, br, eaw, m1, "name", CompressAlways)
+			c1.Start()
+			c0.ClusterConfig(ClusterConfigMessage{})
+			c1.ClusterConfig(ClusterConfigMessage{})
 
 			res := c0.ping()
 			if (i < 8 || j < 8) && res {
+				// This should have resulted in failure, as there is no way an empty ClusterConfig plus a Ping message fits in eight bytes.
 				t.Errorf("Unexpected ping success; i=%d, j=%d", i, j)
-			} else if (i >= 12 && j >= 12) && !res {
+			} else if (i >= 28 && j >= 28) && !res {
+				// This should have worked though, as 28 bytes is plenty for both.
 				t.Errorf("Unexpected ping fail; i=%d, j=%d", i, j)
 			}
 		}
@@ -168,7 +178,11 @@ func TestVersionErr(t *testing.T) {
 	br, bw := io.Pipe()
 
 	c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
-	NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
+	c0.Start()
+	c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
+	c1.Start()
+	c0.ClusterConfig(ClusterConfigMessage{})
+	c1.ClusterConfig(ClusterConfigMessage{})
 
 	w := xdr.NewWriter(c0.cw)
 	w.WriteUint32(encodeHeader(header{
@@ -191,7 +205,11 @@ func TestTypeErr(t *testing.T) {
 	br, bw := io.Pipe()
 
 	c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
-	NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
+	c0.Start()
+	c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
+	c1.Start()
+	c0.ClusterConfig(ClusterConfigMessage{})
+	c1.ClusterConfig(ClusterConfigMessage{})
 
 	w := xdr.NewWriter(c0.cw)
 	w.WriteUint32(encodeHeader(header{
@@ -214,7 +232,11 @@ func TestClose(t *testing.T) {
 	br, bw := io.Pipe()
 
 	c0 := NewConnection(c0ID, ar, bw, m0, "name", CompressAlways).(wireFormatConnection).next.(*rawConnection)
-	NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
+	c0.Start()
+	c1 := NewConnection(c1ID, br, aw, m1, "name", CompressAlways)
+	c1.Start()
+	c0.ClusterConfig(ClusterConfigMessage{})
+	c1.ClusterConfig(ClusterConfigMessage{})
 
 	c0.close(nil)
 

Godeps/_workspace/src/github.com/syncthing/protocol/vector_xdr.go

@@ -2,6 +2,8 @@
 
 package protocol
 
+import "github.com/calmh/xdr"
+
 // This stuff is hacked up manually because genxdr doesn't support 'type
 // Vector []Counter' declarations and it was tricky when I tried to add it...
 
@@ -28,6 +30,9 @@ func (v Vector) EncodeXDRInto(w xdrWriter) (int, error) {
 // DecodeXDRFrom decodes the XDR objects from the given reader into itself.
 func (v *Vector) DecodeXDRFrom(r xdrReader) error {
 	l := int(r.ReadUint32())
+	if l > 1e6 {
+		return xdr.ElementSizeExceeded("number of counters", l, 1e6)
+	}
 	n := make(Vector, l)
 	for i := range n {
 		n[i].ID = r.ReadUint64()

Godeps/_workspace/src/github.com/syncthing/protocol/wireformat.go

@@ -12,6 +12,10 @@ type wireFormatConnection struct {
 	next Connection
 }
 
+func (c wireFormatConnection) Start() {
+	c.next.Start()
+}
+
 func (c wireFormatConnection) ID() DeviceID {
 	return c.next.ID()
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go

@@ -63,13 +63,14 @@ type DB struct {
 	journalAckC  chan error
 
 	// Compaction.
-	tcompCmdC   chan cCmd
-	tcompPauseC chan chan<- struct{}
-	mcompCmdC   chan cCmd
-	compErrC    chan error
-	compPerErrC chan error
-	compErrSetC chan error
-	compStats   []cStats
+	tcompCmdC        chan cCmd
+	tcompPauseC      chan chan<- struct{}
+	mcompCmdC        chan cCmd
+	compErrC         chan error
+	compPerErrC      chan error
+	compErrSetC      chan error
+	compWriteLocking bool
+	compStats        []cStats
 
 	// Close.
 	closeW sync.WaitGroup
@@ -108,28 +109,44 @@ func openDB(s *session) (*DB, error) {
 		closeC: make(chan struct{}),
 	}
 
-	if err := db.recoverJournal(); err != nil {
-		return nil, err
-	}
+	// Read-only mode.
+	readOnly := s.o.GetReadOnly()
 
-	// Remove any obsolete files.
-	if err := db.checkAndCleanFiles(); err != nil {
-		// Close journal.
-		if db.journal != nil {
-			db.journal.Close()
-			db.journalWriter.Close()
+	if readOnly {
+		// Recover journals (read-only mode).
+		if err := db.recoverJournalRO(); err != nil {
+			return nil, err
 		}
-		return nil, err
+	} else {
+		// Recover journals.
+		if err := db.recoverJournal(); err != nil {
+			return nil, err
+		}
+
+		// Remove any obsolete files.
+		if err := db.checkAndCleanFiles(); err != nil {
+			// Close journal.
+			if db.journal != nil {
+				db.journal.Close()
+				db.journalWriter.Close()
+			}
+			return nil, err
+		}
+
 	}
 
 	// Doesn't need to be included in the wait group.
 	go db.compactionError()
 	go db.mpoolDrain()
 
-	db.closeW.Add(3)
-	go db.tCompaction()
-	go db.mCompaction()
-	go db.jWriter()
+	if readOnly {
+		db.SetReadOnly()
+	} else {
+		db.closeW.Add(3)
+		go db.tCompaction()
+		go db.mCompaction()
+		go db.jWriter()
+	}
 
 	s.logf("db@open done T·%v", time.Since(start))
 
@@ -274,8 +291,9 @@ func recoverTable(s *session, o *opt.Options) error {
 
 		// We will drop corrupted table.
 		strict = o.GetStrict(opt.StrictRecovery)
+		noSync = o.GetNoSync()
 
-		rec   = &sessionRecord{numLevel: o.GetNumLevel()}
+		rec   = &sessionRecord{}
 		bpool = util.NewBufferPool(o.GetBlockSize() + 5)
 	)
 	buildTable := func(iter iterator.Iterator) (tmp storage.File, size int64, err error) {
@@ -311,9 +329,11 @@ func recoverTable(s *session, o *opt.Options) error {
 		if err != nil {
 			return
 		}
-		err = writer.Sync()
-		if err != nil {
-			return
+		if !noSync {
+			err = writer.Sync()
+			if err != nil {
+				return
+			}
 		}
 		size = int64(tw.BytesLen())
 		return
@@ -450,132 +470,136 @@ func recoverTable(s *session, o *opt.Options) error {
 }
 
 func (db *DB) recoverJournal() error {
-	// Get all tables and sort it by file number.
-	journalFiles_, err := db.s.getFiles(storage.TypeJournal)
+	// Get all journals and sort it by file number.
+	allJournalFiles, err := db.s.getFiles(storage.TypeJournal)
 	if err != nil {
 		return err
 	}
-	journalFiles := files(journalFiles_)
-	journalFiles.sort()
+	files(allJournalFiles).sort()
 
-	// Discard older journal.
-	prev := -1
-	for i, file := range journalFiles {
-		if file.Num() >= db.s.stJournalNum {
-			if prev >= 0 {
-				i--
-				journalFiles[i] = journalFiles[prev]
-			}
-			journalFiles = journalFiles[i:]
-			break
-		} else if file.Num() == db.s.stPrevJournalNum {
-			prev = i
+	// Journals that will be recovered.
+	var recJournalFiles []storage.File
+	for _, jf := range allJournalFiles {
+		if jf.Num() >= db.s.stJournalNum || jf.Num() == db.s.stPrevJournalNum {
+			recJournalFiles = append(recJournalFiles, jf)
 		}
 	}
 
-	var jr *journal.Reader
-	var of storage.File
-	var mem *memdb.DB
-	batch := new(Batch)
-	cm := newCMem(db.s)
-	buf := new(util.Buffer)
-	// Options.
-	strict := db.s.o.GetStrict(opt.StrictJournal)
-	checksum := db.s.o.GetStrict(opt.StrictJournalChecksum)
-	writeBuffer := db.s.o.GetWriteBuffer()
-	recoverJournal := func(file storage.File) error {
-		db.logf("journal@recovery recovering @%d", file.Num())
-		reader, err := file.Open()
-		if err != nil {
-			return err
-		}
-		defer reader.Close()
+	var (
+		of  storage.File // Obsolete file.
+		rec = &sessionRecord{}
+	)
 
-		// Create/reset journal reader instance.
-		if jr == nil {
-			jr = journal.NewReader(reader, dropper{db.s, file}, strict, checksum)
-		} else {
-			jr.Reset(reader, dropper{db.s, file}, strict, checksum)
-		}
-
-		// Flush memdb and remove obsolete journal file.
-		if of != nil {
-			if mem.Len() > 0 {
-				if err := cm.flush(mem, 0); err != nil {
-					return err
-				}
-			}
-			if err := cm.commit(file.Num(), db.seq); err != nil {
-				return err
-			}
-			cm.reset()
-			of.Remove()
-			of = nil
-		}
-
-		// Replay journal to memdb.
-		mem.Reset()
-		for {
-			r, err := jr.Next()
-			if err != nil {
-				if err == io.EOF {
-					break
-				}
-				return errors.SetFile(err, file)
-			}
-
-			buf.Reset()
-			if _, err := buf.ReadFrom(r); err != nil {
-				if err == io.ErrUnexpectedEOF {
-					// This is error returned due to corruption, with strict == false.
-					continue
-				} else {
-					return errors.SetFile(err, file)
-				}
-			}
-			if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mem); err != nil {
-				if strict || !errors.IsCorrupted(err) {
-					return errors.SetFile(err, file)
-				} else {
-					db.s.logf("journal error: %v (skipped)", err)
-					// We won't apply sequence number as it might be corrupted.
-					continue
-				}
-			}
-
-			// Save sequence number.
-			db.seq = batch.seq + uint64(batch.Len())
-
-			// Flush it if large enough.
-			if mem.Size() >= writeBuffer {
-				if err := cm.flush(mem, 0); err != nil {
-					return err
-				}
-				mem.Reset()
-			}
-		}
-
-		of = file
-		return nil
-	}
-
-	// Recover all journals.
-	if len(journalFiles) > 0 {
-		db.logf("journal@recovery F·%d", len(journalFiles))
+	// Recover journals.
+	if len(recJournalFiles) > 0 {
+		db.logf("journal@recovery F·%d", len(recJournalFiles))
 
 		// Mark file number as used.
-		db.s.markFileNum(journalFiles[len(journalFiles)-1].Num())
+		db.s.markFileNum(recJournalFiles[len(recJournalFiles)-1].Num())
 
-		mem = memdb.New(db.s.icmp, writeBuffer)
-		for _, file := range journalFiles {
-			if err := recoverJournal(file); err != nil {
+		var (
+			// Options.
+			strict      = db.s.o.GetStrict(opt.StrictJournal)
+			checksum    = db.s.o.GetStrict(opt.StrictJournalChecksum)
+			writeBuffer = db.s.o.GetWriteBuffer()
+
+			jr    *journal.Reader
+			mdb   = memdb.New(db.s.icmp, writeBuffer)
+			buf   = &util.Buffer{}
+			batch = &Batch{}
+		)
+
+		for _, jf := range recJournalFiles {
+			db.logf("journal@recovery recovering @%d", jf.Num())
+
+			fr, err := jf.Open()
+			if err != nil {
 				return err
 			}
+
+			// Create or reset journal reader instance.
+			if jr == nil {
+				jr = journal.NewReader(fr, dropper{db.s, jf}, strict, checksum)
+			} else {
+				jr.Reset(fr, dropper{db.s, jf}, strict, checksum)
+			}
+
+			// Flush memdb and remove obsolete journal file.
+			if of != nil {
+				if mdb.Len() > 0 {
+					if _, err := db.s.flushMemdb(rec, mdb, -1); err != nil {
+						fr.Close()
+						return err
+					}
+				}
+
+				rec.setJournalNum(jf.Num())
+				rec.setSeqNum(db.seq)
+				if err := db.s.commit(rec); err != nil {
+					fr.Close()
+					return err
+				}
+				rec.resetAddedTables()
+
+				of.Remove()
+				of = nil
+			}
+
+			// Replay journal to memdb.
+			mdb.Reset()
+			for {
+				r, err := jr.Next()
+				if err != nil {
+					if err == io.EOF {
+						break
+					}
+
+					fr.Close()
+					return errors.SetFile(err, jf)
+				}
+
+				buf.Reset()
+				if _, err := buf.ReadFrom(r); err != nil {
+					if err == io.ErrUnexpectedEOF {
+						// This is error returned due to corruption, with strict == false.
+						continue
+					}
+
+					fr.Close()
+					return errors.SetFile(err, jf)
+				}
+				if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mdb); err != nil {
+					if !strict && errors.IsCorrupted(err) {
+						db.s.logf("journal error: %v (skipped)", err)
+						// We won't apply sequence number as it might be corrupted.
+						continue
+					}
+
+					fr.Close()
+					return errors.SetFile(err, jf)
+				}
+
+				// Save sequence number.
+				db.seq = batch.seq + uint64(batch.Len())
+
+				// Flush it if large enough.
+				if mdb.Size() >= writeBuffer {
+					if _, err := db.s.flushMemdb(rec, mdb, 0); err != nil {
+						fr.Close()
+						return err
+					}
+
+					mdb.Reset()
+				}
+			}
+
+			fr.Close()
+			of = jf
 		}
 
-		// Flush the last journal.
-		if mem.Len() > 0 {
-			if err := cm.flush(mem, 0); err != nil {
+		// Flush the last memdb.
+		if mdb.Len() > 0 {
+			if _, err := db.s.flushMemdb(rec, mdb, 0); err != nil {
 				return err
 			}
 		}
@@ -587,8 +611,10 @@ func (db *DB) recoverJournal() error {
 	}
 
 	// Commit.
-	if err := cm.commit(db.journalFile.Num(), db.seq); err != nil {
-		// Close journal.
+	rec.setJournalNum(db.journalFile.Num())
+	rec.setSeqNum(db.seq)
+	if err := db.s.commit(rec); err != nil {
+		// Close journal on error.
 		if db.journal != nil {
 			db.journal.Close()
 			db.journalWriter.Close()
@@ -604,6 +630,103 @@ func (db *DB) recoverJournal() error {
 	return nil
 }
 
+func (db *DB) recoverJournalRO() error {
+	// Get all journals and sort it by file number.
+	allJournalFiles, err := db.s.getFiles(storage.TypeJournal)
+	if err != nil {
+		return err
+	}
+	files(allJournalFiles).sort()
+
+	// Journals that will be recovered.
+	var recJournalFiles []storage.File
+	for _, jf := range allJournalFiles {
+		if jf.Num() >= db.s.stJournalNum || jf.Num() == db.s.stPrevJournalNum {
+			recJournalFiles = append(recJournalFiles, jf)
+		}
+	}
+
+	var (
+		// Options.
+		strict      = db.s.o.GetStrict(opt.StrictJournal)
+		checksum    = db.s.o.GetStrict(opt.StrictJournalChecksum)
+		writeBuffer = db.s.o.GetWriteBuffer()
+
+		mdb = memdb.New(db.s.icmp, writeBuffer)
+	)
+
+	// Recover journals.
+	if len(recJournalFiles) > 0 {
+		db.logf("journal@recovery RO·Mode F·%d", len(recJournalFiles))
+
+		var (
+			jr    *journal.Reader
+			buf   = &util.Buffer{}
+			batch = &Batch{}
+		)
+
+		for _, jf := range recJournalFiles {
+			db.logf("journal@recovery recovering @%d", jf.Num())
+
+			fr, err := jf.Open()
+			if err != nil {
+				return err
+			}
+
+			// Create or reset journal reader instance.
+			if jr == nil {
+				jr = journal.NewReader(fr, dropper{db.s, jf}, strict, checksum)
+			} else {
+				jr.Reset(fr, dropper{db.s, jf}, strict, checksum)
+			}
+
+			// Replay journal to memdb.
+			for {
+				r, err := jr.Next()
+				if err != nil {
+					if err == io.EOF {
+						break
+					}
+
+					fr.Close()
+					return errors.SetFile(err, jf)
+				}
+
+				buf.Reset()
+				if _, err := buf.ReadFrom(r); err != nil {
+					if err == io.ErrUnexpectedEOF {
+						// This is error returned due to corruption, with strict == false.
+						continue
+					}
+
+					fr.Close()
+					return errors.SetFile(err, jf)
+				}
+				if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mdb); err != nil {
+					if !strict && errors.IsCorrupted(err) {
+						db.s.logf("journal error: %v (skipped)", err)
+						// We won't apply sequence number as it might be corrupted.
+						continue
+					}
+
+					fr.Close()
+					return errors.SetFile(err, jf)
+				}
+
+				// Save sequence number.
+				db.seq = batch.seq + uint64(batch.Len())
+			}
+
+			fr.Close()
+		}
+	}
+
+	// Set memDB.
+	db.mem = &memDB{db: db, DB: mdb, ref: 1}
+
+	return nil
+}
+
 func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, err error) {
 	ikey := newIkey(key, seq, ktSeek)
 
@@ -614,7 +737,7 @@ func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, er
 		}
 		defer m.decref()
 
-		mk, mv, me := m.mdb.Find(ikey)
+		mk, mv, me := m.Find(ikey)
 		if me == nil {
 			ukey, _, kt, kerr := parseIkey(mk)
 			if kerr != nil {
@@ -652,7 +775,7 @@ func (db *DB) has(key []byte, seq uint64, ro *opt.ReadOptions) (ret bool, err er
 		}
 		defer m.decref()
 
-		mk, _, me := m.mdb.Find(ikey)
+		mk, _, me := m.Find(ikey)
 		if me == nil {
 			ukey, _, kt, kerr := parseIkey(mk)
 			if kerr != nil {
@@ -784,7 +907,7 @@ func (db *DB) GetProperty(name string) (value string, err error) {
 
 	const prefix = "leveldb."
 	if !strings.HasPrefix(name, prefix) {
-		return "", errors.New("leveldb: GetProperty: unknown property: " + name)
+		return "", ErrNotFound
 	}
 	p := name[len(prefix):]
 
@@ -798,7 +921,7 @@ func (db *DB) GetProperty(name string) (value string, err error) {
 		var rest string
 		n, _ := fmt.Sscanf(p[len(numFilesPrefix):], "%d%s", &level, &rest)
 		if n != 1 || int(level) >= db.s.o.GetNumLevel() {
-			err = errors.New("leveldb: GetProperty: invalid property: " + name)
+			err = ErrNotFound
 		} else {
 			value = fmt.Sprint(v.tLen(int(level)))
 		}
@@ -837,7 +960,7 @@ func (db *DB) GetProperty(name string) (value string, err error) {
 	case p == "aliveiters":
 		value = fmt.Sprintf("%d", atomic.LoadInt32(&db.aliveIters))
 	default:
-		err = errors.New("leveldb: GetProperty: unknown property: " + name)
+		err = ErrNotFound
 	}
 
 	return
@@ -900,6 +1023,9 @@ func (db *DB) Close() error {
 	var err error
 	select {
 	case err = <-db.compErrC:
+		if err == ErrReadOnly {
+			err = nil
+		}
 	default:
 	}
 

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go

@@ -11,7 +11,6 @@ import (
 	"time"
 
 	"github.com/syndtr/goleveldb/leveldb/errors"
-	"github.com/syndtr/goleveldb/leveldb/memdb"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 )
 
@@ -62,58 +61,8 @@ func (p *cStatsStaging) stopTimer() {
 	}
 }
 
-type cMem struct {
-	s     *session
-	level int
-	rec   *sessionRecord
-}
-
-func newCMem(s *session) *cMem {
-	return &cMem{s: s, rec: &sessionRecord{numLevel: s.o.GetNumLevel()}}
-}
-
-func (c *cMem) flush(mem *memdb.DB, level int) error {
-	s := c.s
-
-	// Write memdb to table.
-	iter := mem.NewIterator(nil)
-	defer iter.Release()
-	t, n, err := s.tops.createFrom(iter)
-	if err != nil {
-		return err
-	}
-
-	// Pick level.
-	if level < 0 {
-		v := s.version()
-		level = v.pickLevel(t.imin.ukey(), t.imax.ukey())
-		v.release()
-	}
-	c.rec.addTableFile(level, t)
-
-	s.logf("mem@flush created L%d@%d N·%d S·%s %q:%q", level, t.file.Num(), n, shortenb(int(t.size)), t.imin, t.imax)
-
-	c.level = level
-	return nil
-}
-
-func (c *cMem) reset() {
-	c.rec = &sessionRecord{numLevel: c.s.o.GetNumLevel()}
-}
-
-func (c *cMem) commit(journal, seq uint64) error {
-	c.rec.setJournalNum(journal)
-	c.rec.setSeqNum(seq)
-
-	// Commit changes.
-	return c.s.commit(c.rec)
-}
-
 func (db *DB) compactionError() {
-	var (
-		err     error
-		wlocked bool
-	)
+	var err error
 noerr:
 	// No error.
 	for {
@@ -121,7 +70,7 @@ noerr:
 		case err = <-db.compErrSetC:
 			switch {
 			case err == nil:
-			case errors.IsCorrupted(err):
+			case err == ErrReadOnly, errors.IsCorrupted(err):
 				goto hasperr
 			default:
 				goto haserr
@@ -139,7 +88,7 @@ haserr:
 			switch {
 			case err == nil:
 				goto noerr
-			case errors.IsCorrupted(err):
+			case err == ErrReadOnly, errors.IsCorrupted(err):
 				goto hasperr
 			default:
 			}
@@ -155,9 +104,9 @@ hasperr:
 		case db.compPerErrC <- err:
 		case db.writeLockC <- struct{}{}:
 			// Hold write lock, so that write won't pass-through.
-			wlocked = true
+			db.compWriteLocking = true
 		case _, _ = <-db.closeC:
-			if wlocked {
+			if db.compWriteLocking {
 				// We should release the lock or Close will hang.
 				<-db.writeLockC
 			}
@@ -287,21 +236,18 @@ func (db *DB) compactionExitTransact() {
 }
 
 func (db *DB) memCompaction() {
-	mem := db.getFrozenMem()
-	if mem == nil {
+	mdb := db.getFrozenMem()
+	if mdb == nil {
 		return
 	}
-	defer mem.decref()
+	defer mdb.decref()
 
-	c := newCMem(db.s)
-	stats := new(cStatsStaging)
-
-	db.logf("mem@flush N·%d S·%s", mem.mdb.Len(), shortenb(mem.mdb.Size()))
+	db.logf("memdb@flush N·%d S·%s", mdb.Len(), shortenb(mdb.Size()))
 
 	// Don't compact empty memdb.
-	if mem.mdb.Len() == 0 {
-		db.logf("mem@flush skipping")
-		// drop frozen mem
+	if mdb.Len() == 0 {
+		db.logf("memdb@flush skipping")
+		// drop frozen memdb
 		db.dropFrozenMem()
 		return
 	}
@@ -317,13 +263,20 @@ func (db *DB) memCompaction() {
 		return
 	}
 
-	db.compactionTransactFunc("mem@flush", func(cnt *compactionTransactCounter) (err error) {
+	var (
+		rec        = &sessionRecord{}
+		stats      = &cStatsStaging{}
+		flushLevel int
+	)
+
+	db.compactionTransactFunc("memdb@flush", func(cnt *compactionTransactCounter) (err error) {
 		stats.startTimer()
-		defer stats.stopTimer()
-		return c.flush(mem.mdb, -1)
+		flushLevel, err = db.s.flushMemdb(rec, mdb.DB, -1)
+		stats.stopTimer()
+		return
 	}, func() error {
-		for _, r := range c.rec.addedTables {
-			db.logf("mem@flush revert @%d", r.num)
+		for _, r := range rec.addedTables {
+			db.logf("memdb@flush revert @%d", r.num)
 			f := db.s.getTableFile(r.num)
 			if err := f.Remove(); err != nil {
 				return err
@@ -332,20 +285,23 @@ func (db *DB) memCompaction() {
 		return nil
 	})
 
-	db.compactionTransactFunc("mem@commit", func(cnt *compactionTransactCounter) (err error) {
+	db.compactionTransactFunc("memdb@commit", func(cnt *compactionTransactCounter) (err error) {
 		stats.startTimer()
-		defer stats.stopTimer()
-		return c.commit(db.journalFile.Num(), db.frozenSeq)
+		rec.setJournalNum(db.journalFile.Num())
+		rec.setSeqNum(db.frozenSeq)
+		err = db.s.commit(rec)
+		stats.stopTimer()
+		return
 	}, nil)
 
-	db.logf("mem@flush committed F·%d T·%v", len(c.rec.addedTables), stats.duration)
+	db.logf("memdb@flush committed F·%d T·%v", len(rec.addedTables), stats.duration)
 
-	for _, r := range c.rec.addedTables {
+	for _, r := range rec.addedTables {
 		stats.write += r.size
 	}
-	db.compStats[c.level].add(stats)
+	db.compStats[flushLevel].add(stats)
 
-	// Drop frozen mem.
+	// Drop frozen memdb.
 	db.dropFrozenMem()
 
 	// Resume table compaction.
@@ -557,7 +513,7 @@ func (b *tableCompactionBuilder) revert() error {
 func (db *DB) tableCompaction(c *compaction, noTrivial bool) {
 	defer c.release()
 
-	rec := &sessionRecord{numLevel: db.s.o.GetNumLevel()}
+	rec := &sessionRecord{}
 	rec.addCompPtr(c.level, c.imax)
 
 	if !noTrivial && c.trivial() {

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_iter.go

@@ -8,6 +8,7 @@ package leveldb
 
 import (
 	"errors"
+	"math/rand"
 	"runtime"
 	"sync"
 	"sync/atomic"
@@ -39,11 +40,11 @@ func (db *DB) newRawIterator(slice *util.Range, ro *opt.ReadOptions) iterator.It
 	ti := v.getIterators(slice, ro)
 	n := len(ti) + 2
 	i := make([]iterator.Iterator, 0, n)
-	emi := em.mdb.NewIterator(slice)
+	emi := em.NewIterator(slice)
 	emi.SetReleaser(&memdbReleaser{m: em})
 	i = append(i, emi)
 	if fm != nil {
-		fmi := fm.mdb.NewIterator(slice)
+		fmi := fm.NewIterator(slice)
 		fmi.SetReleaser(&memdbReleaser{m: fm})
 		i = append(i, fmi)
 	}
@@ -80,6 +81,10 @@ func (db *DB) newIterator(seq uint64, slice *util.Range, ro *opt.ReadOptions) *d
 	return iter
 }
 
+func (db *DB) iterSamplingRate() int {
+	return rand.Intn(2 * db.s.o.GetIteratorSamplingRate())
+}
+
 type dir int
 
 const (
@@ -98,11 +103,21 @@ type dbIter struct {
 	seq    uint64
 	strict bool
 
-	dir      dir
-	key      []byte
-	value    []byte
-	err      error
-	releaser util.Releaser
+	smaplingGap int
+	dir         dir
+	key         []byte
+	value       []byte
+	err         error
+	releaser    util.Releaser
+}
+
+func (i *dbIter) sampleSeek() {
+	ikey := i.iter.Key()
+	i.smaplingGap -= len(ikey) + len(i.iter.Value())
+	for i.smaplingGap < 0 {
+		i.smaplingGap += i.db.iterSamplingRate()
+		i.db.sampleSeek(ikey)
+	}
 }
 
 func (i *dbIter) setErr(err error) {
@@ -175,6 +190,7 @@ func (i *dbIter) Seek(key []byte) bool {
 func (i *dbIter) next() bool {
 	for {
 		if ukey, seq, kt, kerr := parseIkey(i.iter.Key()); kerr == nil {
+			i.sampleSeek()
 			if seq <= i.seq {
 				switch kt {
 				case ktDel:
@@ -225,6 +241,7 @@ func (i *dbIter) prev() bool {
 	if i.iter.Valid() {
 		for {
 			if ukey, seq, kt, kerr := parseIkey(i.iter.Key()); kerr == nil {
+				i.sampleSeek()
 				if seq <= i.seq {
 					if !del && i.icmp.uCompare(ukey, i.key) < 0 {
 						return true
@@ -266,6 +283,7 @@ func (i *dbIter) Prev() bool {
 	case dirForward:
 		for i.iter.Prev() {
 			if ukey, _, _, kerr := parseIkey(i.iter.Key()); kerr == nil {
+				i.sampleSeek()
 				if i.icmp.uCompare(ukey, i.key) < 0 {
 					goto cont
 				}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_state.go

@@ -15,8 +15,8 @@ import (
 )
 
 type memDB struct {
-	db  *DB
-	mdb *memdb.DB
+	db *DB
+	*memdb.DB
 	ref int32
 }
 
@@ -27,12 +27,12 @@ func (m *memDB) incref() {
 func (m *memDB) decref() {
 	if ref := atomic.AddInt32(&m.ref, -1); ref == 0 {
 		// Only put back memdb with std capacity.
-		if m.mdb.Capacity() == m.db.s.o.GetWriteBuffer() {
-			m.mdb.Reset()
-			m.db.mpoolPut(m.mdb)
+		if m.Capacity() == m.db.s.o.GetWriteBuffer() {
+			m.Reset()
+			m.db.mpoolPut(m.DB)
 		}
 		m.db = nil
-		m.mdb = nil
+		m.DB = nil
 	} else if ref < 0 {
 		panic("negative memdb ref")
 	}
@@ -48,6 +48,15 @@ func (db *DB) addSeq(delta uint64) {
 	atomic.AddUint64(&db.seq, delta)
 }
 
+func (db *DB) sampleSeek(ikey iKey) {
+	v := db.s.version()
+	if v.sampleSeek(ikey) {
+		// Trigger table compaction.
+		db.compSendTrigger(db.tcompCmdC)
+	}
+	v.release()
+}
+
 func (db *DB) mpoolPut(mem *memdb.DB) {
 	defer func() {
 		recover()
@@ -117,7 +126,7 @@ func (db *DB) newMem(n int) (mem *memDB, err error) {
 	}
 	mem = &memDB{
 		db:  db,
-		mdb: mdb,
+		DB:  mdb,
 		ref: 2,
 	}
 	db.mem = mem

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_test.go

@@ -405,19 +405,21 @@ func (h *dbHarness) compactRange(min, max string) {
 	t.Log("DB range compaction done")
 }
 
-func (h *dbHarness) sizeAssert(start, limit string, low, hi uint64) {
-	t := h.t
-	db := h.db
-
-	s, err := db.SizeOf([]util.Range{
+func (h *dbHarness) sizeOf(start, limit string) uint64 {
+	sz, err := h.db.SizeOf([]util.Range{
 		{[]byte(start), []byte(limit)},
 	})
 	if err != nil {
-		t.Error("SizeOf: got error: ", err)
+		h.t.Error("SizeOf: got error: ", err)
 	}
-	if s.Sum() < low || s.Sum() > hi {
-		t.Errorf("sizeof %q to %q not in range, want %d - %d, got %d",
-			shorten(start), shorten(limit), low, hi, s.Sum())
+	return sz.Sum()
+}
+
+func (h *dbHarness) sizeAssert(start, limit string, low, hi uint64) {
+	sz := h.sizeOf(start, limit)
+	if sz < low || sz > hi {
+		h.t.Errorf("sizeOf %q to %q not in range, want %d - %d, got %d",
+			shorten(start), shorten(limit), low, hi, sz)
 	}
 }
 
@@ -2443,7 +2445,7 @@ func TestDB_TableCompactionBuilder(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		rec := &sessionRecord{numLevel: s.o.GetNumLevel()}
+		rec := &sessionRecord{}
 		rec.addTableFile(i, tf)
 		if err := s.commit(rec); err != nil {
 			t.Fatal(err)
@@ -2453,7 +2455,7 @@ func TestDB_TableCompactionBuilder(t *testing.T) {
 	// Build grandparent.
 	v := s.version()
 	c := newCompaction(s, v, 1, append(tFiles{}, v.tables[1]...))
-	rec := &sessionRecord{numLevel: s.o.GetNumLevel()}
+	rec := &sessionRecord{}
 	b := &tableCompactionBuilder{
 		s:         s,
 		c:         c,
@@ -2477,7 +2479,7 @@ func TestDB_TableCompactionBuilder(t *testing.T) {
 	// Build level-1.
 	v = s.version()
 	c = newCompaction(s, v, 0, append(tFiles{}, v.tables[0]...))
-	rec = &sessionRecord{numLevel: s.o.GetNumLevel()}
+	rec = &sessionRecord{}
 	b = &tableCompactionBuilder{
 		s:         s,
 		c:         c,
@@ -2521,7 +2523,7 @@ func TestDB_TableCompactionBuilder(t *testing.T) {
 	// Compaction with transient error.
 	v = s.version()
 	c = newCompaction(s, v, 1, append(tFiles{}, v.tables[1]...))
-	rec = &sessionRecord{numLevel: s.o.GetNumLevel()}
+	rec = &sessionRecord{}
 	b = &tableCompactionBuilder{
 		s:         s,
 		c:         c,
@@ -2577,3 +2579,123 @@ func TestDB_TableCompactionBuilder(t *testing.T) {
 	}
 	v.release()
 }
+
+func testDB_IterTriggeredCompaction(t *testing.T, limitDiv int) {
+	const (
+		vSize = 200 * opt.KiB
+		tSize = 100 * opt.MiB
+		mIter = 100
+		n     = tSize / vSize
+	)
+
+	h := newDbHarnessWopt(t, &opt.Options{
+		Compression:       opt.NoCompression,
+		DisableBlockCache: true,
+	})
+	defer h.close()
+
+	key := func(x int) string {
+		return fmt.Sprintf("v%06d", x)
+	}
+
+	// Fill.
+	value := strings.Repeat("x", vSize)
+	for i := 0; i < n; i++ {
+		h.put(key(i), value)
+	}
+	h.compactMem()
+
+	// Delete all.
+	for i := 0; i < n; i++ {
+		h.delete(key(i))
+	}
+	h.compactMem()
+
+	var (
+		limit = n / limitDiv
+
+		startKey = key(0)
+		limitKey = key(limit)
+		maxKey   = key(n)
+		slice    = &util.Range{Limit: []byte(limitKey)}
+
+		initialSize0 = h.sizeOf(startKey, limitKey)
+		initialSize1 = h.sizeOf(limitKey, maxKey)
+	)
+
+	t.Logf("inital size %s [rest %s]", shortenb(int(initialSize0)), shortenb(int(initialSize1)))
+
+	for r := 0; true; r++ {
+		if r >= mIter {
+			t.Fatal("taking too long to compact")
+		}
+
+		// Iterates.
+		iter := h.db.NewIterator(slice, h.ro)
+		for iter.Next() {
+		}
+		if err := iter.Error(); err != nil {
+			t.Fatalf("Iter err: %v", err)
+		}
+		iter.Release()
+
+		// Wait compaction.
+		h.waitCompaction()
+
+		// Check size.
+		size0 := h.sizeOf(startKey, limitKey)
+		size1 := h.sizeOf(limitKey, maxKey)
+		t.Logf("#%03d size %s [rest %s]", r, shortenb(int(size0)), shortenb(int(size1)))
+		if size0 < initialSize0/10 {
+			break
+		}
+	}
+
+	if initialSize1 > 0 {
+		h.sizeAssert(limitKey, maxKey, initialSize1/4-opt.MiB, initialSize1+opt.MiB)
+	}
+}
+
+func TestDB_IterTriggeredCompaction(t *testing.T) {
+	testDB_IterTriggeredCompaction(t, 1)
+}
+
+func TestDB_IterTriggeredCompactionHalf(t *testing.T) {
+	testDB_IterTriggeredCompaction(t, 2)
+}
+
+func TestDB_ReadOnly(t *testing.T) {
+	h := newDbHarness(t)
+	defer h.close()
+
+	h.put("foo", "v1")
+	h.put("bar", "v2")
+	h.compactMem()
+
+	h.put("xfoo", "v1")
+	h.put("xbar", "v2")
+
+	t.Log("Trigger read-only")
+	if err := h.db.SetReadOnly(); err != nil {
+		h.close()
+		t.Fatalf("SetReadOnly error: %v", err)
+	}
+
+	h.stor.SetEmuErr(storage.TypeAll, tsOpCreate, tsOpReplace, tsOpRemove, tsOpWrite, tsOpWrite, tsOpSync)
+
+	ro := func(key, value, wantValue string) {
+		if err := h.db.Put([]byte(key), []byte(value), h.wo); err != ErrReadOnly {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		h.getVal(key, wantValue)
+	}
+
+	ro("foo", "vx", "v1")
+
+	h.o.ReadOnly = true
+	h.reopenDB()
+
+	ro("foo", "vx", "v1")
+	ro("bar", "vx", "v2")
+	h.assertNumKeys(4)
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_write.go

@@ -63,24 +63,24 @@ func (db *DB) rotateMem(n int) (mem *memDB, err error) {
 	return
 }
 
-func (db *DB) flush(n int) (mem *memDB, nn int, err error) {
+func (db *DB) flush(n int) (mdb *memDB, mdbFree int, err error) {
 	delayed := false
 	flush := func() (retry bool) {
 		v := db.s.version()
 		defer v.release()
-		mem = db.getEffectiveMem()
+		mdb = db.getEffectiveMem()
 		defer func() {
 			if retry {
-				mem.decref()
-				mem = nil
+				mdb.decref()
+				mdb = nil
 			}
 		}()
-		nn = mem.mdb.Free()
+		mdbFree = mdb.Free()
 		switch {
 		case v.tLen(0) >= db.s.o.GetWriteL0SlowdownTrigger() && !delayed:
 			delayed = true
 			time.Sleep(time.Millisecond)
-		case nn >= n:
+		case mdbFree >= n:
 			return false
 		case v.tLen(0) >= db.s.o.GetWriteL0PauseTrigger():
 			delayed = true
@@ -90,15 +90,15 @@ func (db *DB) flush(n int) (mem *memDB, nn int, err error) {
 			}
 		default:
 			// Allow memdb to grow if it has no entry.
-			if mem.mdb.Len() == 0 {
-				nn = n
+			if mdb.Len() == 0 {
+				mdbFree = n
 			} else {
-				mem.decref()
-				mem, err = db.rotateMem(n)
+				mdb.decref()
+				mdb, err = db.rotateMem(n)
 				if err == nil {
-					nn = mem.mdb.Free()
+					mdbFree = mdb.Free()
 				} else {
-					nn = 0
+					mdbFree = 0
 				}
 			}
 			return false
@@ -129,7 +129,7 @@ func (db *DB) Write(b *Batch, wo *opt.WriteOptions) (err error) {
 		return
 	}
 
-	b.init(wo.GetSync())
+	b.init(wo.GetSync() && !db.s.o.GetNoSync())
 
 	// The write happen synchronously.
 	select {
@@ -157,18 +157,18 @@ func (db *DB) Write(b *Batch, wo *opt.WriteOptions) (err error) {
 		}
 	}()
 
-	mem, memFree, err := db.flush(b.size())
+	mdb, mdbFree, err := db.flush(b.size())
 	if err != nil {
 		return
 	}
-	defer mem.decref()
+	defer mdb.decref()
 
 	// Calculate maximum size of the batch.
 	m := 1 << 20
 	if x := b.size(); x <= 128<<10 {
 		m = x + (128 << 10)
 	}
-	m = minInt(m, memFree)
+	m = minInt(m, mdbFree)
 
 	// Merge with other batch.
 drain:
@@ -197,7 +197,7 @@ drain:
 		select {
 		case db.journalC <- b:
 			// Write into memdb
-			if berr := b.memReplay(mem.mdb); berr != nil {
+			if berr := b.memReplay(mdb.DB); berr != nil {
 				panic(berr)
 			}
 		case err = <-db.compPerErrC:
@@ -211,7 +211,7 @@ drain:
 		case err = <-db.journalAckC:
 			if err != nil {
 				// Revert memdb if error detected
-				if berr := b.revertMemReplay(mem.mdb); berr != nil {
+				if berr := b.revertMemReplay(mdb.DB); berr != nil {
 					panic(berr)
 				}
 				return
@@ -225,7 +225,7 @@ drain:
 		if err != nil {
 			return
 		}
-		if berr := b.memReplay(mem.mdb); berr != nil {
+		if berr := b.memReplay(mdb.DB); berr != nil {
 			panic(berr)
 		}
 	}
@@ -233,7 +233,7 @@ drain:
 	// Set last seq number.
 	db.addSeq(uint64(b.Len()))
 
-	if b.size() >= memFree {
+	if b.size() >= mdbFree {
 		db.rotateMem(0)
 	}
 	return
@@ -249,8 +249,7 @@ func (db *DB) Put(key, value []byte, wo *opt.WriteOptions) error {
 	return db.Write(b, wo)
 }
 
-// Delete deletes the value for the given key. It returns ErrNotFound if
-// the DB does not contain the key.
+// Delete deletes the value for the given key.
 //
 // It is safe to modify the contents of the arguments after Delete returns.
 func (db *DB) Delete(key []byte, wo *opt.WriteOptions) error {
@@ -290,9 +289,9 @@ func (db *DB) CompactRange(r util.Range) error {
 	}
 
 	// Check for overlaps in memdb.
-	mem := db.getEffectiveMem()
-	defer mem.decref()
-	if isMemOverlaps(db.s.icmp, mem.mdb, r.Start, r.Limit) {
+	mdb := db.getEffectiveMem()
+	defer mdb.decref()
+	if isMemOverlaps(db.s.icmp, mdb.DB, r.Start, r.Limit) {
 		// Memdb compaction.
 		if _, err := db.rotateMem(0); err != nil {
 			<-db.writeLockC
@@ -309,3 +308,31 @@ func (db *DB) CompactRange(r util.Range) error {
 	// Table compaction.
 	return db.compSendRange(db.tcompCmdC, -1, r.Start, r.Limit)
 }
+
+// SetReadOnly makes DB read-only. It will stay read-only until reopened.
+func (db *DB) SetReadOnly() error {
+	if err := db.ok(); err != nil {
+		return err
+	}
+
+	// Lock writer.
+	select {
+	case db.writeLockC <- struct{}{}:
+		db.compWriteLocking = true
+	case err := <-db.compPerErrC:
+		return err
+	case _, _ = <-db.closeC:
+		return ErrClosed
+	}
+
+	// Set compaction read-only.
+	select {
+	case db.compErrSetC <- ErrReadOnly:
+	case perr := <-db.compPerErrC:
+		return perr
+	case _, _ = <-db.closeC:
+		return ErrClosed
+	}
+
+	return nil
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors.go

@@ -12,6 +12,7 @@ import (
 
 var (
 	ErrNotFound         = errors.ErrNotFound
+	ErrReadOnly         = errors.New("leveldb: read-only mode")
 	ErrSnapshotReleased = errors.New("leveldb: snapshot released")
 	ErrIterReleased     = errors.New("leveldb: iterator released")
 	ErrClosed           = errors.New("leveldb: closed")

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors/errors.go

@@ -52,12 +52,14 @@ func IsCorrupted(err error) bool {
 	switch err.(type) {
 	case *ErrCorrupted:
 		return true
+	case *storage.ErrCorrupted:
+		return true
 	}
 	return false
 }
 
 // ErrMissingFiles is the type that indicating a corruption due to missing
-// files.
+// files. ErrMissingFiles always wrapped with ErrCorrupted.
 type ErrMissingFiles struct {
 	Files []*storage.FileInfo
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go

@@ -206,6 +206,7 @@ func (p *DB) randHeight() (h int) {
 	return
 }
 
+// Must hold RW-lock if prev == true, as it use shared prevNode slice.
 func (p *DB) findGE(key []byte, prev bool) (int, bool) {
 	node := 0
 	h := p.maxHeight - 1
@@ -302,7 +303,7 @@ func (p *DB) Put(key []byte, value []byte) error {
 	node := len(p.nodeData)
 	p.nodeData = append(p.nodeData, kvOffset, len(key), len(value), h)
 	for i, n := range p.prevNode[:h] {
-		m := n + 4 + i
+		m := n + nNext + i
 		p.nodeData = append(p.nodeData, p.nodeData[m])
 		p.nodeData[m] = node
 	}
@@ -434,20 +435,22 @@ func (p *DB) Len() int {
 
 // Reset resets the DB to initial empty state. Allows reuse the buffer.
 func (p *DB) Reset() {
+	p.mu.Lock()
 	p.rnd = rand.New(rand.NewSource(0xdeadbeef))
 	p.maxHeight = 1
 	p.n = 0
 	p.kvSize = 0
 	p.kvData = p.kvData[:0]
-	p.nodeData = p.nodeData[:4+tMaxHeight]
+	p.nodeData = p.nodeData[:nNext+tMaxHeight]
 	p.nodeData[nKV] = 0
 	p.nodeData[nKey] = 0
 	p.nodeData[nVal] = 0
 	p.nodeData[nHeight] = tMaxHeight
 	for n := 0; n < tMaxHeight; n++ {
-		p.nodeData[4+n] = 0
+		p.nodeData[nNext+n] = 0
 		p.prevNode[n] = 0
 	}
+	p.mu.Unlock()
 }
 
 // New creates a new initalized in-memory key/value DB. The capacity

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go

@@ -34,10 +34,11 @@ var (
 	DefaultCompactionTotalSize           = 10 * MiB
 	DefaultCompactionTotalSizeMultiplier = 10.0
 	DefaultCompressionType               = SnappyCompression
-	DefaultOpenFilesCacher               = LRUCacher
-	DefaultOpenFilesCacheCapacity        = 500
+	DefaultIteratorSamplingRate          = 1 * MiB
 	DefaultMaxMemCompationLevel          = 2
 	DefaultNumLevel                      = 7
+	DefaultOpenFilesCacher               = LRUCacher
+	DefaultOpenFilesCacheCapacity        = 500
 	DefaultWriteBuffer                   = 4 * MiB
 	DefaultWriteL0PauseTrigger           = 12
 	DefaultWriteL0SlowdownTrigger        = 8
@@ -249,6 +250,11 @@ type Options struct {
 	// The default value (DefaultCompression) uses snappy compression.
 	Compression Compression
 
+	// DisableBufferPool allows disable use of util.BufferPool functionality.
+	//
+	// The default value is false.
+	DisableBufferPool bool
+
 	// DisableBlockCache allows disable use of cache.Cache functionality on
 	// 'sorted table' block.
 	//
@@ -288,6 +294,13 @@ type Options struct {
 	// The default value is nil.
 	Filter filter.Filter
 
+	// IteratorSamplingRate defines approximate gap (in bytes) between read
+	// sampling of an iterator. The samples will be used to determine when
+	// compaction should be triggered.
+	//
+	// The default is 1MiB.
+	IteratorSamplingRate int
+
 	// MaxMemCompationLevel defines maximum level a newly compacted 'memdb'
 	// will be pushed into if doesn't creates overlap. This should less than
 	// NumLevel. Use -1 for level-0.
@@ -295,6 +308,11 @@ type Options struct {
 	// The default is 2.
 	MaxMemCompationLevel int
 
+	// NoSync allows completely disable fsync.
+	//
+	// The default is false.
+	NoSync bool
+
 	// NumLevel defines number of database level. The level shouldn't changed
 	// between opens, or the database will panic.
 	//
@@ -313,6 +331,11 @@ type Options struct {
 	// The default value is 500.
 	OpenFilesCacheCapacity int
 
+	// If true then opens DB in read-only mode.
+	//
+	// The default value is false.
+	ReadOnly bool
+
 	// Strict defines the DB strict level.
 	Strict Strict
 
@@ -464,6 +487,20 @@ func (o *Options) GetCompression() Compression {
 	return o.Compression
 }
 
+func (o *Options) GetDisableBufferPool() bool {
+	if o == nil {
+		return false
+	}
+	return o.DisableBufferPool
+}
+
+func (o *Options) GetDisableBlockCache() bool {
+	if o == nil {
+		return false
+	}
+	return o.DisableBlockCache
+}
+
 func (o *Options) GetDisableCompactionBackoff() bool {
 	if o == nil {
 		return false
@@ -492,6 +529,13 @@ func (o *Options) GetFilter() filter.Filter {
 	return o.Filter
 }
 
+func (o *Options) GetIteratorSamplingRate() int {
+	if o == nil || o.IteratorSamplingRate <= 0 {
+		return DefaultIteratorSamplingRate
+	}
+	return o.IteratorSamplingRate
+}
+
 func (o *Options) GetMaxMemCompationLevel() int {
 	level := DefaultMaxMemCompationLevel
 	if o != nil {
@@ -507,6 +551,13 @@ func (o *Options) GetMaxMemCompationLevel() int {
 	return level
 }
 
+func (o *Options) GetNoSync() bool {
+	if o == nil {
+		return false
+	}
+	return o.NoSync
+}
+
 func (o *Options) GetNumLevel() int {
 	if o == nil || o.NumLevel <= 0 {
 		return DefaultNumLevel
@@ -533,6 +584,13 @@ func (o *Options) GetOpenFilesCacheCapacity() int {
 	return o.OpenFilesCacheCapacity
 }
 
+func (o *Options) GetReadOnly() bool {
+	if o == nil {
+		return false
+	}
+	return o.ReadOnly
+}
+
 func (o *Options) GetStrict(strict Strict) bool {
 	if o == nil || o.Strict == 0 {
 		return DefaultStrict&strict != 0

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go

@@ -11,10 +11,8 @@ import (
 	"io"
 	"os"
 	"sync"
-	"sync/atomic"
 
 	"github.com/syndtr/goleveldb/leveldb/errors"
-	"github.com/syndtr/goleveldb/leveldb/iterator"
 	"github.com/syndtr/goleveldb/leveldb/journal"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 	"github.com/syndtr/goleveldb/leveldb/storage"
@@ -127,11 +125,16 @@ func (s *session) recover() (err error) {
 		return
 	}
 	defer reader.Close()
-	strict := s.o.GetStrict(opt.StrictManifest)
-	jr := journal.NewReader(reader, dropper{s, m}, strict, true)
 
-	staging := s.stVersion.newStaging()
-	rec := &sessionRecord{numLevel: s.o.GetNumLevel()}
+	var (
+		// Options.
+		numLevel = s.o.GetNumLevel()
+		strict   = s.o.GetStrict(opt.StrictManifest)
+
+		jr      = journal.NewReader(reader, dropper{s, m}, strict, true)
+		rec     = &sessionRecord{}
+		staging = s.stVersion.newStaging()
+	)
 	for {
 		var r io.Reader
 		r, err = jr.Next()
@@ -143,7 +146,7 @@ func (s *session) recover() (err error) {
 			return errors.SetFile(err, m)
 		}
 
-		err = rec.decode(r)
+		err = rec.decode(r, numLevel)
 		if err == nil {
 			// save compact pointers
 			for _, r := range rec.compPtrs {
@@ -206,250 +209,3 @@ func (s *session) commit(r *sessionRecord) (err error) {
 
 	return
 }
-
-// Pick a compaction based on current state; need external synchronization.
-func (s *session) pickCompaction() *compaction {
-	v := s.version()
-
-	var level int
-	var t0 tFiles
-	if v.cScore >= 1 {
-		level = v.cLevel
-		cptr := s.stCompPtrs[level]
-		tables := v.tables[level]
-		for _, t := range tables {
-			if cptr == nil || s.icmp.Compare(t.imax, cptr) > 0 {
-				t0 = append(t0, t)
-				break
-			}
-		}
-		if len(t0) == 0 {
-			t0 = append(t0, tables[0])
-		}
-	} else {
-		if p := atomic.LoadPointer(&v.cSeek); p != nil {
-			ts := (*tSet)(p)
-			level = ts.level
-			t0 = append(t0, ts.table)
-		} else {
-			v.release()
-			return nil
-		}
-	}
-
-	return newCompaction(s, v, level, t0)
-}
-
-// Create compaction from given level and range; need external synchronization.
-func (s *session) getCompactionRange(level int, umin, umax []byte) *compaction {
-	v := s.version()
-
-	t0 := v.tables[level].getOverlaps(nil, s.icmp, umin, umax, level == 0)
-	if len(t0) == 0 {
-		v.release()
-		return nil
-	}
-
-	// Avoid compacting too much in one shot in case the range is large.
-	// But we cannot do this for level-0 since level-0 files can overlap
-	// and we must not pick one file and drop another older file if the
-	// two files overlap.
-	if level > 0 {
-		limit := uint64(v.s.o.GetCompactionSourceLimit(level))
-		total := uint64(0)
-		for i, t := range t0 {
-			total += t.size
-			if total >= limit {
-				s.logf("table@compaction limiting F·%d -> F·%d", len(t0), i+1)
-				t0 = t0[:i+1]
-				break
-			}
-		}
-	}
-
-	return newCompaction(s, v, level, t0)
-}
-
-func newCompaction(s *session, v *version, level int, t0 tFiles) *compaction {
-	c := &compaction{
-		s:             s,
-		v:             v,
-		level:         level,
-		tables:        [2]tFiles{t0, nil},
-		maxGPOverlaps: uint64(s.o.GetCompactionGPOverlaps(level)),
-		tPtrs:         make([]int, s.o.GetNumLevel()),
-	}
-	c.expand()
-	c.save()
-	return c
-}
-
-// compaction represent a compaction state.
-type compaction struct {
-	s *session
-	v *version
-
-	level         int
-	tables        [2]tFiles
-	maxGPOverlaps uint64
-
-	gp                tFiles
-	gpi               int
-	seenKey           bool
-	gpOverlappedBytes uint64
-	imin, imax        iKey
-	tPtrs             []int
-	released          bool
-
-	snapGPI               int
-	snapSeenKey           bool
-	snapGPOverlappedBytes uint64
-	snapTPtrs             []int
-}
-
-func (c *compaction) save() {
-	c.snapGPI = c.gpi
-	c.snapSeenKey = c.seenKey
-	c.snapGPOverlappedBytes = c.gpOverlappedBytes
-	c.snapTPtrs = append(c.snapTPtrs[:0], c.tPtrs...)
-}
-
-func (c *compaction) restore() {
-	c.gpi = c.snapGPI
-	c.seenKey = c.snapSeenKey
-	c.gpOverlappedBytes = c.snapGPOverlappedBytes
-	c.tPtrs = append(c.tPtrs[:0], c.snapTPtrs...)
-}
-
-func (c *compaction) release() {
-	if !c.released {
-		c.released = true
-		c.v.release()
-	}
-}
-
-// Expand compacted tables; need external synchronization.
-func (c *compaction) expand() {
-	limit := uint64(c.s.o.GetCompactionExpandLimit(c.level))
-	vt0, vt1 := c.v.tables[c.level], c.v.tables[c.level+1]
-
-	t0, t1 := c.tables[0], c.tables[1]
-	imin, imax := t0.getRange(c.s.icmp)
-	// We expand t0 here just incase ukey hop across tables.
-	t0 = vt0.getOverlaps(t0, c.s.icmp, imin.ukey(), imax.ukey(), c.level == 0)
-	if len(t0) != len(c.tables[0]) {
-		imin, imax = t0.getRange(c.s.icmp)
-	}
-	t1 = vt1.getOverlaps(t1, c.s.icmp, imin.ukey(), imax.ukey(), false)
-	// Get entire range covered by compaction.
-	amin, amax := append(t0, t1...).getRange(c.s.icmp)
-
-	// See if we can grow the number of inputs in "level" without
-	// changing the number of "level+1" files we pick up.
-	if len(t1) > 0 {
-		exp0 := vt0.getOverlaps(nil, c.s.icmp, amin.ukey(), amax.ukey(), c.level == 0)
-		if len(exp0) > len(t0) && t1.size()+exp0.size() < limit {
-			xmin, xmax := exp0.getRange(c.s.icmp)
-			exp1 := vt1.getOverlaps(nil, c.s.icmp, xmin.ukey(), xmax.ukey(), false)
-			if len(exp1) == len(t1) {
-				c.s.logf("table@compaction expanding L%d+L%d (F·%d S·%s)+(F·%d S·%s) -> (F·%d S·%s)+(F·%d S·%s)",
-					c.level, c.level+1, len(t0), shortenb(int(t0.size())), len(t1), shortenb(int(t1.size())),
-					len(exp0), shortenb(int(exp0.size())), len(exp1), shortenb(int(exp1.size())))
-				imin, imax = xmin, xmax
-				t0, t1 = exp0, exp1
-				amin, amax = append(t0, t1...).getRange(c.s.icmp)
-			}
-		}
-	}
-
-	// Compute the set of grandparent files that overlap this compaction
-	// (parent == level+1; grandparent == level+2)
-	if c.level+2 < c.s.o.GetNumLevel() {
-		c.gp = c.v.tables[c.level+2].getOverlaps(c.gp, c.s.icmp, amin.ukey(), amax.ukey(), false)
-	}
-
-	c.tables[0], c.tables[1] = t0, t1
-	c.imin, c.imax = imin, imax
-}
-
-// Check whether compaction is trivial.
-func (c *compaction) trivial() bool {
-	return len(c.tables[0]) == 1 && len(c.tables[1]) == 0 && c.gp.size() <= c.maxGPOverlaps
-}
-
-func (c *compaction) baseLevelForKey(ukey []byte) bool {
-	for level, tables := range c.v.tables[c.level+2:] {
-		for c.tPtrs[level] < len(tables) {
-			t := tables[c.tPtrs[level]]
-			if c.s.icmp.uCompare(ukey, t.imax.ukey()) <= 0 {
-				// We've advanced far enough.
-				if c.s.icmp.uCompare(ukey, t.imin.ukey()) >= 0 {
-					// Key falls in this file's range, so definitely not base level.
-					return false
-				}
-				break
-			}
-			c.tPtrs[level]++
-		}
-	}
-	return true
-}
-
-func (c *compaction) shouldStopBefore(ikey iKey) bool {
-	for ; c.gpi < len(c.gp); c.gpi++ {
-		gp := c.gp[c.gpi]
-		if c.s.icmp.Compare(ikey, gp.imax) <= 0 {
-			break
-		}
-		if c.seenKey {
-			c.gpOverlappedBytes += gp.size
-		}
-	}
-	c.seenKey = true
-
-	if c.gpOverlappedBytes > c.maxGPOverlaps {
-		// Too much overlap for current output; start new output.
-		c.gpOverlappedBytes = 0
-		return true
-	}
-	return false
-}
-
-// Creates an iterator.
-func (c *compaction) newIterator() iterator.Iterator {
-	// Creates iterator slice.
-	icap := len(c.tables)
-	if c.level == 0 {
-		// Special case for level-0
-		icap = len(c.tables[0]) + 1
-	}
-	its := make([]iterator.Iterator, 0, icap)
-
-	// Options.
-	ro := &opt.ReadOptions{
-		DontFillCache: true,
-		Strict:        opt.StrictOverride,
-	}
-	strict := c.s.o.GetStrict(opt.StrictCompaction)
-	if strict {
-		ro.Strict |= opt.StrictReader
-	}
-
-	for i, tables := range c.tables {
-		if len(tables) == 0 {
-			continue
-		}
-
-		// Level-0 is not sorted and may overlaps each other.
-		if c.level+i == 0 {
-			for _, t := range tables {
-				its = append(its, c.s.tops.newIterator(t, nil, ro))
-			}
-		} else {
-			it := iterator.NewIndexedIterator(tables.newIndexIterator(c.s.tops, c.s.icmp, nil, ro), strict)
-			its = append(its, it)
-		}
-	}
-
-	return iterator.NewMergedIterator(its, c.s.icmp, strict)
-}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_compaction.go

@@ -0,0 +1,287 @@
+// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
+// All rights reserved.
+//
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package leveldb
+
+import (
+	"sync/atomic"
+
+	"github.com/syndtr/goleveldb/leveldb/iterator"
+	"github.com/syndtr/goleveldb/leveldb/memdb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+)
+
+func (s *session) pickMemdbLevel(umin, umax []byte) int {
+	v := s.version()
+	defer v.release()
+	return v.pickMemdbLevel(umin, umax)
+}
+
+func (s *session) flushMemdb(rec *sessionRecord, mdb *memdb.DB, level int) (level_ int, err error) {
+	// Create sorted table.
+	iter := mdb.NewIterator(nil)
+	defer iter.Release()
+	t, n, err := s.tops.createFrom(iter)
+	if err != nil {
+		return level, err
+	}
+
+	// Pick level and add to record.
+	if level < 0 {
+		level = s.pickMemdbLevel(t.imin.ukey(), t.imax.ukey())
+	}
+	rec.addTableFile(level, t)
+
+	s.logf("memdb@flush created L%d@%d N·%d S·%s %q:%q", level, t.file.Num(), n, shortenb(int(t.size)), t.imin, t.imax)
+	return level, nil
+}
+
+// Pick a compaction based on current state; need external synchronization.
+func (s *session) pickCompaction() *compaction {
+	v := s.version()
+
+	var level int
+	var t0 tFiles
+	if v.cScore >= 1 {
+		level = v.cLevel
+		cptr := s.stCompPtrs[level]
+		tables := v.tables[level]
+		for _, t := range tables {
+			if cptr == nil || s.icmp.Compare(t.imax, cptr) > 0 {
+				t0 = append(t0, t)
+				break
+			}
+		}
+		if len(t0) == 0 {
+			t0 = append(t0, tables[0])
+		}
+	} else {
+		if p := atomic.LoadPointer(&v.cSeek); p != nil {
+			ts := (*tSet)(p)
+			level = ts.level
+			t0 = append(t0, ts.table)
+		} else {
+			v.release()
+			return nil
+		}
+	}
+
+	return newCompaction(s, v, level, t0)
+}
+
+// Create compaction from given level and range; need external synchronization.
+func (s *session) getCompactionRange(level int, umin, umax []byte) *compaction {
+	v := s.version()
+
+	t0 := v.tables[level].getOverlaps(nil, s.icmp, umin, umax, level == 0)
+	if len(t0) == 0 {
+		v.release()
+		return nil
+	}
+
+	// Avoid compacting too much in one shot in case the range is large.
+	// But we cannot do this for level-0 since level-0 files can overlap
+	// and we must not pick one file and drop another older file if the
+	// two files overlap.
+	if level > 0 {
+		limit := uint64(v.s.o.GetCompactionSourceLimit(level))
+		total := uint64(0)
+		for i, t := range t0 {
+			total += t.size
+			if total >= limit {
+				s.logf("table@compaction limiting F·%d -> F·%d", len(t0), i+1)
+				t0 = t0[:i+1]
+				break
+			}
+		}
+	}
+
+	return newCompaction(s, v, level, t0)
+}
+
+func newCompaction(s *session, v *version, level int, t0 tFiles) *compaction {
+	c := &compaction{
+		s:             s,
+		v:             v,
+		level:         level,
+		tables:        [2]tFiles{t0, nil},
+		maxGPOverlaps: uint64(s.o.GetCompactionGPOverlaps(level)),
+		tPtrs:         make([]int, s.o.GetNumLevel()),
+	}
+	c.expand()
+	c.save()
+	return c
+}
+
+// compaction represent a compaction state.
+type compaction struct {
+	s *session
+	v *version
+
+	level         int
+	tables        [2]tFiles
+	maxGPOverlaps uint64
+
+	gp                tFiles
+	gpi               int
+	seenKey           bool
+	gpOverlappedBytes uint64
+	imin, imax        iKey
+	tPtrs             []int
+	released          bool
+
+	snapGPI               int
+	snapSeenKey           bool
+	snapGPOverlappedBytes uint64
+	snapTPtrs             []int
+}
+
+func (c *compaction) save() {
+	c.snapGPI = c.gpi
+	c.snapSeenKey = c.seenKey
+	c.snapGPOverlappedBytes = c.gpOverlappedBytes
+	c.snapTPtrs = append(c.snapTPtrs[:0], c.tPtrs...)
+}
+
+func (c *compaction) restore() {
+	c.gpi = c.snapGPI
+	c.seenKey = c.snapSeenKey
+	c.gpOverlappedBytes = c.snapGPOverlappedBytes
+	c.tPtrs = append(c.tPtrs[:0], c.snapTPtrs...)
+}
+
+func (c *compaction) release() {
+	if !c.released {
+		c.released = true
+		c.v.release()
+	}
+}
+
+// Expand compacted tables; need external synchronization.
+func (c *compaction) expand() {
+	limit := uint64(c.s.o.GetCompactionExpandLimit(c.level))
+	vt0, vt1 := c.v.tables[c.level], c.v.tables[c.level+1]
+
+	t0, t1 := c.tables[0], c.tables[1]
+	imin, imax := t0.getRange(c.s.icmp)
+	// We expand t0 here just incase ukey hop across tables.
+	t0 = vt0.getOverlaps(t0, c.s.icmp, imin.ukey(), imax.ukey(), c.level == 0)
+	if len(t0) != len(c.tables[0]) {
+		imin, imax = t0.getRange(c.s.icmp)
+	}
+	t1 = vt1.getOverlaps(t1, c.s.icmp, imin.ukey(), imax.ukey(), false)
+	// Get entire range covered by compaction.
+	amin, amax := append(t0, t1...).getRange(c.s.icmp)
+
+	// See if we can grow the number of inputs in "level" without
+	// changing the number of "level+1" files we pick up.
+	if len(t1) > 0 {
+		exp0 := vt0.getOverlaps(nil, c.s.icmp, amin.ukey(), amax.ukey(), c.level == 0)
+		if len(exp0) > len(t0) && t1.size()+exp0.size() < limit {
+			xmin, xmax := exp0.getRange(c.s.icmp)
+			exp1 := vt1.getOverlaps(nil, c.s.icmp, xmin.ukey(), xmax.ukey(), false)
+			if len(exp1) == len(t1) {
+				c.s.logf("table@compaction expanding L%d+L%d (F·%d S·%s)+(F·%d S·%s) -> (F·%d S·%s)+(F·%d S·%s)",
+					c.level, c.level+1, len(t0), shortenb(int(t0.size())), len(t1), shortenb(int(t1.size())),
+					len(exp0), shortenb(int(exp0.size())), len(exp1), shortenb(int(exp1.size())))
+				imin, imax = xmin, xmax
+				t0, t1 = exp0, exp1
+				amin, amax = append(t0, t1...).getRange(c.s.icmp)
+			}
+		}
+	}
+
+	// Compute the set of grandparent files that overlap this compaction
+	// (parent == level+1; grandparent == level+2)
+	if c.level+2 < c.s.o.GetNumLevel() {
+		c.gp = c.v.tables[c.level+2].getOverlaps(c.gp, c.s.icmp, amin.ukey(), amax.ukey(), false)
+	}
+
+	c.tables[0], c.tables[1] = t0, t1
+	c.imin, c.imax = imin, imax
+}
+
+// Check whether compaction is trivial.
+func (c *compaction) trivial() bool {
+	return len(c.tables[0]) == 1 && len(c.tables[1]) == 0 && c.gp.size() <= c.maxGPOverlaps
+}
+
+func (c *compaction) baseLevelForKey(ukey []byte) bool {
+	for level, tables := range c.v.tables[c.level+2:] {
+		for c.tPtrs[level] < len(tables) {
+			t := tables[c.tPtrs[level]]
+			if c.s.icmp.uCompare(ukey, t.imax.ukey()) <= 0 {
+				// We've advanced far enough.
+				if c.s.icmp.uCompare(ukey, t.imin.ukey()) >= 0 {
+					// Key falls in this file's range, so definitely not base level.
+					return false
+				}
+				break
+			}
+			c.tPtrs[level]++
+		}
+	}
+	return true
+}
+
+func (c *compaction) shouldStopBefore(ikey iKey) bool {
+	for ; c.gpi < len(c.gp); c.gpi++ {
+		gp := c.gp[c.gpi]
+		if c.s.icmp.Compare(ikey, gp.imax) <= 0 {
+			break
+		}
+		if c.seenKey {
+			c.gpOverlappedBytes += gp.size
+		}
+	}
+	c.seenKey = true
+
+	if c.gpOverlappedBytes > c.maxGPOverlaps {
+		// Too much overlap for current output; start new output.
+		c.gpOverlappedBytes = 0
+		return true
+	}
+	return false
+}
+
+// Creates an iterator.
+func (c *compaction) newIterator() iterator.Iterator {
+	// Creates iterator slice.
+	icap := len(c.tables)
+	if c.level == 0 {
+		// Special case for level-0.
+		icap = len(c.tables[0]) + 1
+	}
+	its := make([]iterator.Iterator, 0, icap)
+
+	// Options.
+	ro := &opt.ReadOptions{
+		DontFillCache: true,
+		Strict:        opt.StrictOverride,
+	}
+	strict := c.s.o.GetStrict(opt.StrictCompaction)
+	if strict {
+		ro.Strict |= opt.StrictReader
+	}
+
+	for i, tables := range c.tables {
+		if len(tables) == 0 {
+			continue
+		}
+
+		// Level-0 is not sorted and may overlaps each other.
+		if c.level+i == 0 {
+			for _, t := range tables {
+				its = append(its, c.s.tops.newIterator(t, nil, ro))
+			}
+		} else {
+			it := iterator.NewIndexedIterator(tables.newIndexIterator(c.s.tops, c.s.icmp, nil, ro), strict)
+			its = append(its, it)
+		}
+	}
+
+	return iterator.NewMergedIterator(its, c.s.icmp, strict)
+}

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record.go

@@ -52,8 +52,6 @@ type dtRecord struct {
 }
 
 type sessionRecord struct {
-	numLevel int
-
 	hasRec         int
 	comparer       string
 	journalNum     uint64
@@ -230,7 +228,7 @@ func (p *sessionRecord) readBytes(field string, r byteReader) []byte {
 	return x
 }
 
-func (p *sessionRecord) readLevel(field string, r io.ByteReader) int {
+func (p *sessionRecord) readLevel(field string, r io.ByteReader, numLevel int) int {
 	if p.err != nil {
 		return 0
 	}
@@ -238,14 +236,14 @@ func (p *sessionRecord) readLevel(field string, r io.ByteReader) int {
 	if p.err != nil {
 		return 0
 	}
-	if x >= uint64(p.numLevel) {
+	if x >= uint64(numLevel) {
 		p.err = errors.NewErrCorrupted(nil, &ErrManifestCorrupted{field, "invalid level number"})
 		return 0
 	}
 	return int(x)
 }
 
-func (p *sessionRecord) decode(r io.Reader) error {
+func (p *sessionRecord) decode(r io.Reader, numLevel int) error {
 	br, ok := r.(byteReader)
 	if !ok {
 		br = bufio.NewReader(r)
@@ -286,13 +284,13 @@ func (p *sessionRecord) decode(r io.Reader) error {
 				p.setSeqNum(x)
 			}
 		case recCompPtr:
-			level := p.readLevel("comp-ptr.level", br)
+			level := p.readLevel("comp-ptr.level", br, numLevel)
 			ikey := p.readBytes("comp-ptr.ikey", br)
 			if p.err == nil {
 				p.addCompPtr(level, iKey(ikey))
 			}
 		case recAddTable:
-			level := p.readLevel("add-table.level", br)
+			level := p.readLevel("add-table.level", br, numLevel)
 			num := p.readUvarint("add-table.num", br)
 			size := p.readUvarint("add-table.size", br)
 			imin := p.readBytes("add-table.imin", br)
@@ -301,7 +299,7 @@ func (p *sessionRecord) decode(r io.Reader) error {
 				p.addTable(level, num, size, imin, imax)
 			}
 		case recDelTable:
-			level := p.readLevel("del-table.level", br)
+			level := p.readLevel("del-table.level", br, numLevel)
 			num := p.readUvarint("del-table.num", br)
 			if p.err == nil {
 				p.delTable(level, num)

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record_test.go

@@ -19,8 +19,8 @@ func decodeEncode(v *sessionRecord) (res bool, err error) {
 	if err != nil {
 		return
 	}
-	v2 := &sessionRecord{numLevel: opt.DefaultNumLevel}
-	err = v.decode(b)
+	v2 := &sessionRecord{}
+	err = v.decode(b, opt.DefaultNumLevel)
 	if err != nil {
 		return
 	}
@@ -34,7 +34,7 @@ func decodeEncode(v *sessionRecord) (res bool, err error) {
 
 func TestSessionRecord_EncodeDecode(t *testing.T) {
 	big := uint64(1) << 50
-	v := &sessionRecord{numLevel: opt.DefaultNumLevel}
+	v := &sessionRecord{}
 	i := uint64(0)
 	test := func() {
 		res, err := decodeEncode(v)

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go

@@ -182,7 +182,7 @@ func (s *session) newManifest(rec *sessionRecord, v *version) (err error) {
 		defer v.release()
 	}
 	if rec == nil {
-		rec = &sessionRecord{numLevel: s.o.GetNumLevel()}
+		rec = &sessionRecord{}
 	}
 	s.fillRecord(rec, true)
 	v.fillRecord(rec)
@@ -240,9 +240,11 @@ func (s *session) flushManifest(rec *sessionRecord) (err error) {
 	if err != nil {
 		return
 	}
-	err = s.manifestWriter.Sync()
-	if err != nil {
-		return
+	if !s.o.GetNoSync() {
+		err = s.manifestWriter.Sync()
+		if err != nil {
+			return
+		}
 	}
 	s.recordCommited(rec)
 	return

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go

@@ -243,7 +243,10 @@ func (fs *fileStorage) GetManifest() (f File, err error) {
 					rem = append(rem, fn)
 				}
 				if !pend1 || cerr == nil {
-					cerr = fmt.Errorf("leveldb/storage: corrupted or incomplete %s file", fn)
+					cerr = &ErrCorrupted{
+						File: fsParseName(filepath.Base(fn)),
+						Err:  errors.New("leveldb/storage: corrupted or incomplete manifest file"),
+					}
 				}
 			} else if f != nil && f1.Num() < f.Num() {
 				fs.log(fmt.Sprintf("skipping %s: obsolete", fn))
@@ -326,8 +329,7 @@ func (fs *fileStorage) Close() error {
 	runtime.SetFinalizer(fs, nil)
 
 	if fs.open > 0 {
-		fs.log(fmt.Sprintf("refuse to close, %d files still open", fs.open))
-		return fmt.Errorf("leveldb/storage: cannot close, %d files still open", fs.open)
+		fs.log(fmt.Sprintf("close: warning, %d files still open", fs.open))
 	}
 	fs.open = -1
 	e1 := fs.logw.Close()
@@ -505,30 +507,37 @@ func (f *file) path() string {
 	return filepath.Join(f.fs.path, f.name())
 }
 
-func (f *file) parse(name string) bool {
-	var num uint64
+func fsParseName(name string) *FileInfo {
+	fi := &FileInfo{}
 	var tail string
-	_, err := fmt.Sscanf(name, "%d.%s", &num, &tail)
+	_, err := fmt.Sscanf(name, "%d.%s", &fi.Num, &tail)
 	if err == nil {
 		switch tail {
 		case "log":
-			f.t = TypeJournal
+			fi.Type = TypeJournal
 		case "ldb", "sst":
-			f.t = TypeTable
+			fi.Type = TypeTable
 		case "tmp":
-			f.t = TypeTemp
+			fi.Type = TypeTemp
 		default:
-			return false
+			return nil
 		}
-		f.num = num
-		return true
+		return fi
 	}
-	n, _ := fmt.Sscanf(name, "MANIFEST-%d%s", &num, &tail)
+	n, _ := fmt.Sscanf(name, "MANIFEST-%d%s", &fi.Num, &tail)
 	if n == 1 {
-		f.t = TypeManifest
-		f.num = num
-		return true
+		fi.Type = TypeManifest
+		return fi
 	}
-
-	return false
+	return nil
+}
+
+func (f *file) parse(name string) bool {
+	fi := fsParseName(name)
+	if fi == nil {
+		return false
+	}
+	f.t = fi.Type
+	f.num = fi.Num
+	return true
 }

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_unix.go

@@ -50,13 +50,23 @@ func rename(oldpath, newpath string) error {
 	return os.Rename(oldpath, newpath)
 }
 
+func isErrInvalid(err error) bool {
+	if err == os.ErrInvalid {
+		return true
+	}
+	if syserr, ok := err.(*os.SyscallError); ok && syserr.Err == syscall.EINVAL {
+		return true
+	}
+	return false
+}
+
 func syncDir(name string) error {
 	f, err := os.Open(name)
 	if err != nil {
 		return err
 	}
 	defer f.Close()
-	if err := f.Sync(); err != nil {
+	if err := f.Sync(); err != nil && !isErrInvalid(err) {
 		return err
 	}
 	return nil

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/storage.go

@@ -46,6 +46,22 @@ var (
 	ErrClosed      = errors.New("leveldb/storage: closed")
 )
 
+// ErrCorrupted is the type that wraps errors that indicate corruption of
+// a file. Package storage has its own type instead of using
+// errors.ErrCorrupted to prevent circular import.
+type ErrCorrupted struct {
+	File *FileInfo
+	Err  error
+}
+
+func (e *ErrCorrupted) Error() string {
+	if e.File != nil {
+		return fmt.Sprintf("%v [file=%v]", e.Err, e.File)
+	} else {
+		return e.Err.Error()
+	}
+}
+
 // Syncer is the interface that wraps basic Sync method.
 type Syncer interface {
 	// Sync commits the current contents of the file to stable storage.

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage_test.go

@@ -42,6 +42,8 @@ type tsOp uint
 const (
 	tsOpOpen tsOp = iota
 	tsOpCreate
+	tsOpReplace
+	tsOpRemove
 	tsOpRead
 	tsOpReadAt
 	tsOpWrite
@@ -241,6 +243,10 @@ func (tf tsFile) Replace(newfile storage.File) (err error) {
 	if err != nil {
 		return
 	}
+	if tf.shouldErr(tsOpReplace) {
+		err = errors.New("leveldb.testStorage: emulated create error")
+		return
+	}
 	err = tf.File.Replace(newfile.(tsFile).File)
 	if err != nil {
 		ts.t.Errorf("E: cannot replace file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
@@ -258,6 +264,10 @@ func (tf tsFile) Remove() (err error) {
 	if err != nil {
 		return
 	}
+	if tf.shouldErr(tsOpRemove) {
+		err = errors.New("leveldb.testStorage: emulated create error")
+		return
+	}
 	err = tf.File.Remove()
 	if err != nil {
 		ts.t.Errorf("E: cannot remove file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go

@@ -287,6 +287,7 @@ func (x *tFilesSortByNum) Less(i, j int) bool {
 // Table operations.
 type tOps struct {
 	s      *session
+	noSync bool
 	cache  *cache.Cache
 	bcache *cache.Cache
 	bpool  *util.BufferPool
@@ -441,22 +442,27 @@ func newTableOps(s *session) *tOps {
 	var (
 		cacher cache.Cacher
 		bcache *cache.Cache
+		bpool  *util.BufferPool
 	)
 	if s.o.GetOpenFilesCacheCapacity() > 0 {
 		cacher = cache.NewLRU(s.o.GetOpenFilesCacheCapacity())
 	}
-	if !s.o.DisableBlockCache {
+	if !s.o.GetDisableBlockCache() {
 		var bcacher cache.Cacher
 		if s.o.GetBlockCacheCapacity() > 0 {
 			bcacher = cache.NewLRU(s.o.GetBlockCacheCapacity())
 		}
 		bcache = cache.NewCache(bcacher)
 	}
+	if !s.o.GetDisableBufferPool() {
+		bpool = util.NewBufferPool(s.o.GetBlockSize() + 5)
+	}
 	return &tOps{
 		s:      s,
+		noSync: s.o.GetNoSync(),
 		cache:  cache.NewCache(cacher),
 		bcache: bcache,
-		bpool:  util.NewBufferPool(s.o.GetBlockSize() + 5),
+		bpool:  bpool,
 	}
 }
 
@@ -501,9 +507,11 @@ func (w *tWriter) finish() (f *tFile, err error) {
 	if err != nil {
 		return
 	}
-	err = w.w.Sync()
-	if err != nil {
-		return
+	if !w.t.noSync {
+		err = w.w.Sync()
+		if err != nil {
+			return
+		}
 	}
 	f = newTableFile(w.file, uint64(w.tw.BytesLen()), iKey(w.first), iKey(w.last))
 	return

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/reader.go

@@ -14,7 +14,7 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/syndtr/gosnappy/snappy"
+	"github.com/golang/snappy"
 
 	"github.com/syndtr/goleveldb/leveldb/cache"
 	"github.com/syndtr/goleveldb/leveldb/comparer"

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/writer.go

@@ -12,7 +12,7 @@ import (
 	"fmt"
 	"io"
 
-	"github.com/syndtr/gosnappy/snappy"
+	"github.com/golang/snappy"
 
 	"github.com/syndtr/goleveldb/leveldb/comparer"
 	"github.com/syndtr/goleveldb/leveldb/filter"
@@ -167,11 +167,7 @@ func (w *Writer) writeBlock(buf *util.Buffer, compression opt.Compression) (bh b
 		if n := snappy.MaxEncodedLen(buf.Len()) + blockTrailerLen; len(w.compressionScratch) < n {
 			w.compressionScratch = make([]byte, n)
 		}
-		var compressed []byte
-		compressed, err = snappy.Encode(w.compressionScratch, buf.Bytes())
-		if err != nil {
-			return
-		}
+		compressed := snappy.Encode(w.compressionScratch, buf.Bytes())
 		n := len(compressed)
 		b = compressed[:n+blockTrailerLen]
 		b[n] = blockTypeSnappyCompression

Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/version.go

@@ -136,9 +136,8 @@ func (v *version) get(ikey iKey, ro *opt.ReadOptions, noValue bool) (value []byt
 		if !tseek {
 			if tset == nil {
 				tset = &tSet{level, t}
-			} else if tset.table.consumeSeek() <= 0 {
+			} else {
 				tseek = true
-				tcomp = atomic.CompareAndSwapPointer(&v.cSeek, nil, unsafe.Pointer(tset))
 			}
 		}
 
@@ -203,6 +202,28 @@ func (v *version) get(ikey iKey, ro *opt.ReadOptions, noValue bool) (value []byt
 		return true
 	})
 
+	if tseek && tset.table.consumeSeek() <= 0 {
+		tcomp = atomic.CompareAndSwapPointer(&v.cSeek, nil, unsafe.Pointer(tset))
+	}
+
+	return
+}
+
+func (v *version) sampleSeek(ikey iKey) (tcomp bool) {
+	var tset *tSet
+
+	v.walkOverlapping(ikey, func(level int, t *tFile) bool {
+		if tset == nil {
+			tset = &tSet{level, t}
+			return true
+		} else {
+			if tset.table.consumeSeek() <= 0 {
+				tcomp = atomic.CompareAndSwapPointer(&v.cSeek, nil, unsafe.Pointer(tset))
+			}
+			return false
+		}
+	}, nil)
+
 	return
 }
 
@@ -279,7 +300,7 @@ func (v *version) offsetOf(ikey iKey) (n uint64, err error) {
 	return
 }
 
-func (v *version) pickLevel(umin, umax []byte) (level int) {
+func (v *version) pickMemdbLevel(umin, umax []byte) (level int) {
 	if !v.tables[0].overlaps(v.s.icmp, umin, umax, true) {
 		var overlaps tFiles
 		maxLevel := v.s.o.GetMaxMemCompationLevel()

Godeps/_workspace/src/github.com/thejerf/suture/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2014 Barracuda Networks, Inc.
+Copyright (c) 2014-2015 Barracuda Networks, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Godeps/_workspace/src/github.com/thejerf/suture/suture.go

@@ -36,6 +36,13 @@ to your Supervisor. Supervisors are also services, so you can create a
 tree structure here, depending on the exact combination of restarts
 you want to create.
 
+As a special case, when adding Supervisors to Supervisors, the "sub"
+supervisor will have the "super" supervisor's Log function copied.
+This allows you to set one log function on the "top" supervisor, and
+have it propagate down to all the sub-supervisors. This also allows
+libraries or modules to provide Supervisors without having to commit
+their users to a particular logging method.
+
 Finally, as what is probably the last line of your main() function, call
 .Serve() on your top level supervisor. This will start all the services
 you've defined.
@@ -126,8 +133,10 @@ type Supervisor struct {
 	// If you ever come up with some need to get into these, submit a pull
 	// request to make them public and some smidge of justification, and
 	// I'll happily do it.
-	logBadStop func(Service)
-	logFailure func(service Service, currentFailures float64, failureThreshold float64, restarting bool, error interface{}, stacktrace []byte)
+	// But since I've now changed the signature on these once, I'm glad I
+	// didn't start with them public... :)
+	logBadStop func(*Supervisor, Service)
+	logFailure func(supervisor *Supervisor, service Service, currentFailures float64, failureThreshold float64, restarting bool, error interface{}, stacktrace []byte)
 	logBackoff func(*Supervisor, bool)
 
 	// avoid a dependency on github.com/thejerf/abtime by just implementing
@@ -233,10 +242,10 @@ func New(name string, spec Spec) (s *Supervisor) {
 	s.resumeTimer = make(chan time.Time)
 
 	// set up the default logging handlers
-	s.logBadStop = func(service Service) {
-		s.log(fmt.Sprintf("Service %s failed to terminate in a timely manner", serviceName(service)))
+	s.logBadStop = func(supervisor *Supervisor, service Service) {
+		s.log(fmt.Sprintf("%s: Service %s failed to terminate in a timely manner", serviceName(supervisor), serviceName(service)))
 	}
-	s.logFailure = func(service Service, failures float64, threshold float64, restarting bool, err interface{}, st []byte) {
+	s.logFailure = func(supervisor *Supervisor, service Service, failures float64, threshold float64, restarting bool, err interface{}, st []byte) {
 		var errString string
 
 		e, canError := err.(error)
@@ -246,7 +255,7 @@ func New(name string, spec Spec) (s *Supervisor) {
 			errString = fmt.Sprintf("%#v", err)
 		}
 
-		s.log(fmt.Sprintf("Failed service '%s' (%f failures of %f), restarting: %#v, error: %s, stacktrace: %s", serviceName(service), failures, threshold, restarting, errString, string(st)))
+		s.log(fmt.Sprintf("%s: Failed service '%s' (%f failures of %f), restarting: %#v, error: %s, stacktrace: %s", serviceName(supervisor), serviceName(service), failures, threshold, restarting, errString, string(st)))
 	}
 	s.logBackoff = func(s *Supervisor, entering bool) {
 		if entering {
@@ -346,12 +355,24 @@ will be started when the supervisor is.
 
 The returned ServiceID may be passed to the Remove method of the Supervisor
 to terminate the service.
+
+As a special behavior, if the service added is itself a supervisor, the
+supervisor being added will copy the Log function from the Supervisor it
+is being added to. This allows factoring out providing a Supervisor
+from its logging.
+
 */
 func (s *Supervisor) Add(service Service) ServiceToken {
 	if s == nil {
 		panic("can't add service to nil *suture.Supervisor")
 	}
 
+	if supervisor, isSupervisor := service.(*Supervisor); isSupervisor {
+		supervisor.logBadStop = s.logBadStop
+		supervisor.logFailure = s.logFailure
+		supervisor.logBackoff = s.logBackoff
+	}
+
 	if s.state == notRunning {
 		id := s.serviceCounter
 		s.serviceCounter++
@@ -492,12 +513,12 @@ func (s *Supervisor) handleFailedService(id serviceID, err interface{}, stacktra
 	if monitored {
 		if s.state == normal {
 			s.runService(failedService, id)
-			s.logFailure(failedService, s.failures, s.failureThreshold, true, err, stacktrace)
+			s.logFailure(s, failedService, s.failures, s.failureThreshold, true, err, stacktrace)
 		} else {
 			// FIXME: When restarting, check that the service still
 			// exists (it may have been stopped in the meantime)
 			s.restartQueue = append(s.restartQueue, id)
-			s.logFailure(failedService, s.failures, s.failureThreshold, false, err, stacktrace)
+			s.logFailure(s, failedService, s.failures, s.failureThreshold, false, err, stacktrace)
 		}
 	}
 }
@@ -536,7 +557,7 @@ func (s *Supervisor) removeService(id serviceID) {
 			case <-successChan:
 				// Life is good!
 			case <-failChan:
-				s.logBadStop(service)
+				s.logBadStop(s, service)
 			}
 		}()
 	}

Godeps/_workspace/src/github.com/thejerf/suture/suture_test.go

@@ -77,7 +77,7 @@ func TestFailures(t *testing.T) {
 		// to avoid deadlocks during shutdown, we have to not try to send
 		// things out on channels while we're shutting down (this undoes the
 		// logFailure overide about 25 lines down)
-		s.logFailure = func(Service, float64, float64, bool, interface{}, []byte) {}
+		s.logFailure = func(*Supervisor, Service, float64, float64, bool, interface{}, []byte) {}
 		s.Stop()
 	}()
 	s.sync()
@@ -102,7 +102,7 @@ func TestFailures(t *testing.T) {
 
 	failNotify := make(chan bool)
 	// use this to synchronize on here
-	s.logFailure = func(s Service, cf float64, ft float64, r bool, error interface{}, stacktrace []byte) {
+	s.logFailure = func(supervisor *Supervisor, s Service, cf float64, ft float64, r bool, error interface{}, stacktrace []byte) {
 		failNotify <- r
 	}
 
@@ -276,8 +276,8 @@ func TestDefaultLogging(t *testing.T) {
 
 	serviceName(&BarelyService{})
 
-	s.logBadStop(service)
-	s.logFailure(service, 1, 1, true, errors.New("test error"), []byte{})
+	s.logBadStop(s, service)
+	s.logFailure(s, service, 1, 1, true, errors.New("test error"), []byte{})
 
 	s.Stop()
 }
@@ -289,9 +289,17 @@ func TestNestedSupervisors(t *testing.T) {
 	super2 := NewSimple("Nested5")
 	service := NewService("Service5")
 
+	super2.logBadStop = func(*Supervisor, Service) {
+		panic("Failed to copy logBadStop")
+	}
+
 	super1.Add(super2)
 	super2.Add(service)
 
+	// test the functions got copied from super1; if this panics, it didn't
+	// get copied
+	super2.logBadStop(super2, service)
+
 	go super1.Serve()
 	super1.sync()
 
@@ -340,7 +348,7 @@ func TestStoppingStillWorksWithHungServices(t *testing.T) {
 		return resumeChan
 	}
 	failNotify := make(chan struct{})
-	s.logBadStop = func(s Service) {
+	s.logBadStop = func(supervisor *Supervisor, s Service) {
 		failNotify <- struct{}{}
 	}
 
@@ -438,7 +446,7 @@ func TestFailingSupervisors(t *testing.T) {
 	}
 	failNotify := make(chan string)
 	// use this to synchronize on here
-	s1.logFailure = func(s Service, cf float64, ft float64, r bool, error interface{}, stacktrace []byte) {
+	s1.logFailure = func(supervisor *Supervisor, s Service, cf float64, ft float64, r bool, error interface{}, stacktrace []byte) {
 		failNotify <- fmt.Sprintf("%s", s)
 	}
 

Godeps/_workspace/src/golang.org/x/text/unicode/norm/maketables.go

@@ -471,29 +471,22 @@ func computeNonStarterCounts() {
 		if exp := c.forms[FCompatibility].expandedDecomp; len(exp) > 0 {
 			runes = exp
 		}
+		// We consider runes that combine backwards to be non-starters for the
+		// purpose of Stream-Safe Text Processing.
 		for _, r := range runes {
-			if chars[r].ccc == 0 {
+			if cr := &chars[r]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward {
 				break
 			}
 			c.nLeadingNonStarters++
 		}
 		for i := len(runes) - 1; i >= 0; i-- {
-			if chars[runes[i]].ccc == 0 {
+			if cr := &chars[runes[i]]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward {
 				break
 			}
 			c.nTrailingNonStarters++
 		}
-
-		// We consider runes that combine backwards to be non-starters for the
-		// purpose of Stream-Safe Text Processing.
-		for _, f := range c.forms {
-			if c.ccc == 0 && f.combinesBackward {
-				if len(c.forms[FCompatibility].expandedDecomp) > 0 {
-					log.Fatalf("%U: CCC==0 modifier with an expansion is not supported.", i)
-				}
-				c.nTrailingNonStarters = 1
-				c.nLeadingNonStarters = 1
-			}
+		if c.nTrailingNonStarters > 3 {
+			log.Fatalf("%U: Decomposition with more than 3 (%d) trailing modifiers (%U)", i, c.nTrailingNonStarters, runes)
 		}
 
 		if isHangul(rune(i)) {
@@ -839,10 +832,16 @@ func verifyComputed() {
 				continue
 			}
 			if a, b := c.nLeadingNonStarters > 0, (c.ccc > 0 || f.combinesBackward); a != b {
-				// We accept these two runes to be treated differently (it only affects
-				// segment breaking in iteration, most likely on inproper use), but
+				// We accept these runes to be treated differently (it only affects
+				// segment breaking in iteration, most likely on improper use), but
 				// reconsider if more characters are added.
-				if i != 0xFF9E && i != 0xFF9F {
+				// U+FF9E HALFWIDTH KATAKANA VOICED SOUND MARK;Lm;0;L;<narrow> 3099;;;;N;;;;;
+				// U+FF9F HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK;Lm;0;L;<narrow> 309A;;;;N;;;;;
+				// U+3133 HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<compat> 11AA;;;;N;HANGUL LETTER GIYEOG SIOS;;;;
+				// U+318E HANGUL LETTER ARAEAE;Lo;0;L;<compat> 11A1;;;;N;HANGUL LETTER ALAE AE;;;;
+				// U+FFA3 HALFWIDTH HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<narrow> 3133;;;;N;HALFWIDTH HANGUL LETTER GIYEOG SIOS;;;;
+				// U+FFDC HALFWIDTH HANGUL LETTER I;Lo;0;L;<narrow> 3163;;;;N;;;;;
+				if i != 0xFF9E && i != 0xFF9F && !(0x3133 <= i && i <= 0x318E) && !(0xFFA3 <= i && i <= 0xFFDC) {
 					log.Fatalf("%U: nLead was %v; want %v", i, a, b)
 				}
 			}

Godeps/_workspace/src/golang.org/x/text/unicode/norm/normalize_test.go

@@ -113,7 +113,25 @@ var decomposeSegmentTests = []PositionTest{
 	{"\u00C0b", 2, "A\u0300"},
 	// long
 	{grave(31), 60, grave(30) + cgj},
+	{"a" + grave(31), 61, "a" + grave(30) + cgj},
+
+	// Stability tests: see http://www.unicode.org/review/pr-29.html.
+	// U+0300 COMBINING GRAVE ACCENT;Mn;230;NSM;;;;;N;NON-SPACING GRAVE;;;;
+	// U+0B47 ORIYA VOWEL SIGN E;Mc;0;L;;;;;N;;;;;
+	// U+0B3E ORIYA VOWEL SIGN AA;Mc;0;L;;;;;N;;;;;
+	// U+1100 HANGUL CHOSEONG KIYEOK;Lo;0;L;;;;;N;;;;;
+	// U+1161 HANGUL JUNGSEONG A;Lo;0;L;;;;;N;;;;;
+	{"\u0B47\u0300\u0B3E", 8, "\u0B47\u0300\u0B3E"},
+	{"\u1100\u0300\u1161", 8, "\u1100\u0300\u1161"},
+	{"\u0B47\u0B3E", 6, "\u0B47\u0B3E"},
+	{"\u1100\u1161", 6, "\u1100\u1161"},
+
+	// U+04DA MALAYALAM VOWEL SIGN O;Mc;0;L;0D46 0D3E;;;;N;;;;;
+	// Sequence of decomposing characters that are starters and modifiers.
+	{"\u0d4a" + strings.Repeat("\u0d3e", 31), 90, "\u0d46" + strings.Repeat("\u0d3e", 30) + cgj},
+
 	{grave(30), 60, grave(30)},
+	// U+FF9E is a starter, but decomposes to U+3099, which is not.
 	{grave(30) + "\uff9e", 60, grave(30) + cgj},
 	// ends with incomplete UTF-8 encoding
 	{"\xCC", 0, ""},
@@ -552,6 +570,44 @@ var appendTestsNFC = []AppendTest{
 		"a" + rep(0x0305, maxNonStarters+4) + "\u0316",
 		"a" + rep(0x0305, maxNonStarters) + cgj + "\u0316" + rep(0x305, 4),
 	},
+
+	{ // Combine across non-blocking non-starters.
+		// U+0327 COMBINING CEDILLA;Mn;202;NSM;;;;;N;NON-SPACING CEDILLA;;;;
+		// U+0325 COMBINING RING BELOW;Mn;220;NSM;;;;;N;NON-SPACING RING BELOW;;;;
+		"", "a\u0327\u0325", "\u1e01\u0327",
+	},
+
+	{ // Jamo V+T does not combine.
+		"",
+		"\u1161\u11a8",
+		"\u1161\u11a8",
+	},
+
+	// Stability tests: see http://www.unicode.org/review/pr-29.html.
+	{"", "\u0b47\u0300\u0b3e", "\u0b47\u0300\u0b3e"},
+	{"", "\u1100\u0300\u1161", "\u1100\u0300\u1161"},
+	{"", "\u0b47\u0b3e", "\u0b4b"},
+	{"", "\u1100\u1161", "\uac00"},
+
+	// U+04DA MALAYALAM VOWEL SIGN O;Mc;0;L;0D46 0D3E;;;;N;;;;;
+	{ // 0d4a starts a new segment.
+		"",
+		"\u0d4a" + strings.Repeat("\u0d3e", 15) + "\u0d4a" + strings.Repeat("\u0d3e", 15),
+		"\u0d4a" + strings.Repeat("\u0d3e", 15) + "\u0d4a" + strings.Repeat("\u0d3e", 15),
+	},
+
+	{ // Split combining characters.
+		// TODO: don't insert CGJ before starters.
+		"",
+		"\u0d46" + strings.Repeat("\u0d3e", 31),
+		"\u0d4a" + strings.Repeat("\u0d3e", 29) + cgj + "\u0d3e",
+	},
+
+	{ // Split combining characters.
+		"",
+		"\u0d4a" + strings.Repeat("\u0d3e", 30),
+		"\u0d4a" + strings.Repeat("\u0d3e", 29) + cgj + "\u0d3e",
+	},
 }
 
 var appendTestsNFD = []AppendTest{

NICKS

@@ -14,17 +14,24 @@ andrew-d	<andrew@du.nham.ca>
 asdil12		<dominik@heidler.eu>
 bencurthoys	<ben@bencurthoys.com>
 bigbear2nd	<bigbear2nd@gmail.com>
+brbecker	<brbecker@gmail.com>
 brendanlong	<self@brendanlong.com>
+brgmnn      <dan.arne.bergmann@gmail.com> <brgmnn@users.noreply.github.com>
 bsidhom		<bsidhom@gmail.com>
 calmh		<jakob@nym.se>
+canton7	<antony.male@gmail.com>
 cdata		<chris@scriptolo.gy>
+cdhowie	<me@chrishowie.com>
 ceh		<emil@hessman.se>
 cqcallaw	<enlightened.despot@gmail.com>
+dva	<denisva@gmail.com>
 dzarda		<dzardacz@gmail.com>
 facastagnini	<federico.castagnini@gmail.com>
 filoozoom	<philippe@schommers.be>
 frioux		<frew@afoolishmanifesto.com> <frioux@gmail.com>
+fti7	<frank@isemann.name>
 gillisig	<gilli@vx.is>
+hadogenes	<szafar@linux.pl>
 jarlebring	<jarlebring@gmail.com>
 jedie		<github.com@jensdiemer.de> <git@jensdiemer.de>
 jpjp		<jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
@@ -34,6 +41,7 @@ kozec		<kozec@kozec.com>
 krozycki	<rozycki.karol@gmail.com>
 marcindziadus	<dziadus.marcin@gmail.com>
 marclaporte <marc@marclaporte.com>
+mogwa1	<devriesb@gmail.com>
 moshen		<moshen.colin@gmail.com>
 mvdan		<mvdan@mvdan.cc>
 pascalj		<github@pascalj.com> <mail@pascal-jungblut.com>
@@ -42,9 +50,9 @@ philips		<brandon@ifup.org>
 piobpl		<piotrb10@gmail.com>
 pluby		<phill.luby@newredo.com>
 pyfisch		<pyfisch@gmail.com>
+qbit		<qbit@deftly.net>
 ralder		<ralder@yandex.ru>
 rumpelsepp	<stefan@sevenbyte.org>
-qbit		<qbit@deftly.net>
 sciurius	<jvromans@squirrel.nl>
 seehuhn		<voss@seehuhn.de>
 snnd		<dw@risu.io>
@@ -53,4 +61,5 @@ tnn2		<tnn@nygren.pp.se>
 tojrobinson	<tully@tojr.org>
 uok		<ueomkail@gmail.com> <uok@users.noreply.github.com>
 veeti		<veeti.paananen@rojekti.fi>
+wsgcsysadmin	<e.meitner@willystreet.coo>
 zukoo		<fxgsell@gmail.com>

README.md

@@ -1,11 +1,11 @@
-syncthing
+Syncthing
 =========
 
 [![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/syncthing.svg?style=flat-square)](http://build.syncthing.net/job/syncthing/lastBuild/)
 [![API Documentation](http://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](http://godoc.org/github.com/syncthing/syncthing)
 [![MPLv2 License](http://img.shields.io/badge/license-MPLv2-blue.svg?style=flat-square)](https://www.mozilla.org/MPL/2.0/)
 
-This is the `syncthing` project which pursues the following goals:
+This is the Syncthing project which pursues the following goals:
 
  1. Define a protocol for synchronization of a folder between a number of
     collaborating devices. This protocol should be well defined, unambiguous,
@@ -18,16 +18,16 @@ This is the `syncthing` project which pursues the following goals:
     alternative, compatible implementations of the protocol will arise.
 
 The two are evolving together; the protocol is not to be considered
-stable until syncthing 1.0 is released, at which point it is locked down
+stable until Syncthing 1.0 is released, at which point it is locked down
 for incompatible changes.
 
 Getting Started
 ---------------
 
 Take a look at the [getting started
-guide](https://github.com/syncthing/syncthing/wiki/Getting-Started).
+guide](http://docs.syncthing.net/intro/getting-started.html).
 
-There are a few examples for keeping syncthing running in the background
+There are a few examples for keeping Syncthing running in the background
 on your system in [the etc directory](https://github.com/syncthing/syncthing/blob/master/etc).
 
 There is an IRC channel, `#syncthing` on Freenode, for talking directly
@@ -37,7 +37,7 @@ Building
 --------
 
 Building Syncthing from source is easy, and there's a
-[guide](https://github.com/syncthing/syncthing/wiki/Building).
+[guide](http://docs.syncthing.net/dev/building.html).
 that describes it for both Unix and Windows systems.
 
 Signed Releases
@@ -51,9 +51,8 @@ available in the md5sum.txt.asc and sha1sum.txt.asc files.
 Documentation
 =============
 
-The [syncthing
-documentation](https://github.com/syncthing/syncthing/wiki/) is on the
-Github wiki.
+Please see the [Syncthing
+documentation site](http://docs.syncthing.net/).
 
 All code is licensed under the
 [MPLv2 License](https://github.com/syncthing/syncthing/blob/master/LICENSE).

build.go

@@ -36,6 +36,7 @@ var (
 	goos      string
 	noupgrade bool
 	version   string
+	goVersion float64
 	race      bool
 )
 
@@ -70,7 +71,7 @@ func main() {
 		log.Printf("Unknown goarch %q; proceed with caution!", goarch)
 	}
 
-	checkRequiredGoVersion()
+	goVersion, _ = checkRequiredGoVersion()
 
 	if flag.NArg() == 0 {
 		var tags []string
@@ -80,9 +81,9 @@ func main() {
 		install("./cmd/...", tags)
 
 		vet("./cmd/syncthing")
-		vet("./internal/...")
+		vet("./lib/...")
 		lint("./cmd/syncthing")
-		lint("./internal/...")
+		lint("./lib/...")
 		return
 	}
 
@@ -110,6 +111,9 @@ func main() {
 		case "test":
 			test("./...")
 
+		case "bench":
+			bench("./...")
+
 		case "assets":
 			assets()
 
@@ -131,16 +135,19 @@ func main() {
 		case "zip":
 			buildZip()
 
+		case "deb":
+			buildDeb()
+
 		case "clean":
 			clean()
 
 		case "vet":
 			vet("./cmd/syncthing")
-			vet("./internal/...")
+			vet("./lib/...")
 
 		case "lint":
 			lint("./cmd/syncthing")
-			lint("./internal/...")
+			lint("./lib/...")
 
 		default:
 			log.Fatalf("Unknown command %q", cmd)
@@ -148,7 +155,7 @@ func main() {
 	}
 }
 
-func checkRequiredGoVersion() {
+func checkRequiredGoVersion() (float64, bool) {
 	ver := run("go", "version")
 	re := regexp.MustCompile(`go version go(\d+\.\d+)`)
 	if m := re.FindSubmatch(ver); len(m) == 2 {
@@ -157,14 +164,16 @@ func checkRequiredGoVersion() {
 		f, err := strconv.ParseFloat(vs, 64)
 		if err != nil {
 			log.Printf("*** Couldn't parse Go version out of %q.\n*** This isn't known to work, proceed on your own risk.", vs)
-			return
+			return 0, false
 		}
 		if f < minGoVersion {
 			log.Fatalf("*** Go version %.01f is less than required %.01f.\n*** This is known not to work, not proceeding.", f, minGoVersion)
 		}
-	} else {
-		log.Printf("*** Unknown Go version %q.\n*** This isn't known to work, proceed on your own risk.", ver)
+		return f, true
 	}
+
+	log.Printf("*** Unknown Go version %q.\n*** This isn't known to work, proceed on your own risk.", ver)
+	return 0, false
 }
 
 func setup() {
@@ -172,6 +181,8 @@ func setup() {
 	runPrint("go", "get", "-v", "golang.org/x/tools/cmd/vet")
 	runPrint("go", "get", "-v", "golang.org/x/net/html")
 	runPrint("go", "get", "-v", "github.com/tools/godep")
+	runPrint("go", "get", "-v", "github.com/axw/gocov/gocov")
+	runPrint("go", "get", "-v", "github.com/AlekSi/gocov-xml")
 }
 
 func test(pkg string) {
@@ -179,6 +190,11 @@ func test(pkg string) {
 	runPrint("go", "test", "-short", "-timeout", "60s", pkg)
 }
 
+func bench(pkg string) {
+	setBuildEnv()
+	runPrint("go", "test", "-run", "NONE", "-bench", ".", pkg)
+}
+
 func install(pkg string, tags []string) {
 	os.Setenv("GOBIN", "./bin")
 	args := []string{"install", "-v", "-ldflags", ldflags()}
@@ -272,6 +288,97 @@ func buildZip() {
 	log.Println(filename)
 }
 
+func buildDeb() {
+	os.RemoveAll("deb")
+
+	// "goarch" here is set to whatever the Debian packages expect. We correct
+	// "it to what we actually know how to build and keep the Debian variant
+	// "name in "debarch".
+	debarch := goarch
+	switch goarch {
+	case "i386":
+		goarch = "386"
+	case "armel", "armhf":
+		goarch = "arm"
+	}
+
+	build("./cmd/syncthing", []string{"noupgrade"})
+
+	files := []archiveFile{
+		{src: "README.md", dst: "deb/usr/share/doc/syncthing/README.txt", perm: 0644},
+		{src: "LICENSE", dst: "deb/usr/share/doc/syncthing/LICENSE.txt", perm: 0644},
+		{src: "AUTHORS", dst: "deb/usr/share/doc/syncthing/AUTHORS.txt", perm: 0644},
+		{src: "syncthing", dst: "deb/usr/bin/syncthing", perm: 0755},
+		{src: "man/syncthing.1", dst: "deb/usr/share/man/man1/syncthing.1", perm: 0644},
+		{src: "man/syncthing-config.5", dst: "deb/usr/share/man/man5/syncthing-config.5", perm: 0644},
+		{src: "man/syncthing-stignore.5", dst: "deb/usr/share/man/man5/syncthing-stignore.5", perm: 0644},
+		{src: "man/syncthing-device-ids.7", dst: "deb/usr/share/man/man7/syncthing-device-ids.7", perm: 0644},
+		{src: "man/syncthing-event-api.7", dst: "deb/usr/share/man/man7/syncthing-event-api.7", perm: 0644},
+		{src: "man/syncthing-faq.7", dst: "deb/usr/share/man/man7/syncthing-faq.7", perm: 0644},
+		{src: "man/syncthing-networking.7", dst: "deb/usr/share/man/man7/syncthing-networking.7", perm: 0644},
+		{src: "man/syncthing-rest-api.7", dst: "deb/usr/share/man/man7/syncthing-rest-api.7", perm: 0644},
+		{src: "man/syncthing-security.7", dst: "deb/usr/share/man/man7/syncthing-security.7", perm: 0644},
+		{src: "man/syncthing-versioning.7", dst: "deb/usr/share/man/man7/syncthing-versioning.7", perm: 0644},
+		{src: "etc/linux-systemd/system/syncthing@.service", dst: "deb/lib/systemd/system/syncthing@.service", perm: 0644},
+		{src: "etc/linux-systemd/user/syncthing.service", dst: "deb/usr/lib/systemd/user/syncthing.service", perm: 0644},
+	}
+
+	for _, file := range listFiles("extra") {
+		files = append(files, archiveFile{src: file, dst: "deb/usr/share/doc/syncthing/" + filepath.Base(file), perm: 0644})
+	}
+
+	for _, af := range files {
+		if err := copyFile(af.src, af.dst, af.perm); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	control := `Package: syncthing
+Architecture: {{arch}}
+Depends: libc6
+Version: {{version}}
+Maintainer: Syncthing Release Management <release@syncthing.net>
+Description: Open Source Continuous File Synchronization
+	Syncthing does bidirectional synchronization of files between two or
+	more computers.
+`
+	changelog := `syncthing ({{version}}); urgency=medium
+
+  * Packaging of {{version}}.
+
+ -- Jakob Borg <jakob@nym.se>  {{date}}
+`
+
+	control = strings.Replace(control, "{{arch}}", debarch, -1)
+	control = strings.Replace(control, "{{version}}", version[1:], -1)
+	changelog = strings.Replace(changelog, "{{arch}}", debarch, -1)
+	changelog = strings.Replace(changelog, "{{version}}", version[1:], -1)
+	changelog = strings.Replace(changelog, "{{date}}", time.Now().Format(time.RFC1123), -1)
+
+	os.MkdirAll("deb/DEBIAN", 0755)
+	ioutil.WriteFile("deb/DEBIAN/control", []byte(control), 0644)
+	ioutil.WriteFile("deb/DEBIAN/compat", []byte("9\n"), 0644)
+	ioutil.WriteFile("deb/DEBIAN/changelog", []byte(changelog), 0644)
+
+}
+
+func copyFile(src, dst string, perm os.FileMode) error {
+	dstDir := filepath.Dir(dst)
+	os.MkdirAll(dstDir, 0755) // ignore error
+	srcFd, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer srcFd.Close()
+	dstFd, err := os.OpenFile(dst, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perm)
+	if err != nil {
+		return err
+	}
+	defer dstFd.Close()
+	_, err = io.Copy(dstFd, srcFd)
+	return err
+}
+
 func listFiles(dir string) []string {
 	var res []string
 	filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
@@ -300,16 +407,16 @@ func setBuildEnv() {
 
 func assets() {
 	setBuildEnv()
-	runPipe("internal/auto/gui.files.go", "go", "run", "cmd/genassets/main.go", "gui")
+	runPipe("lib/auto/gui.files.go", "go", "run", "script/genassets.go", "gui")
 }
 
 func xdr() {
-	runPrint("go", "generate", "./internal/discover", "./internal/db")
+	runPrint("go", "generate", "./lib/discover", "./lib/db")
 }
 
 func translate() {
 	os.Chdir("gui/assets/lang")
-	runPipe("lang-en-new.json", "go", "run", "../../../cmd/translate/main.go", "lang-en.json", "../../index.html")
+	runPipe("lang-en-new.json", "go", "run", "../../../script/translate.go", "lang-en.json", "../../")
 	os.Remove("lang-en.json")
 	err := os.Rename("lang-en-new.json", "lang-en.json")
 	if err != nil {
@@ -320,7 +427,7 @@ func translate() {
 
 func transifex() {
 	os.Chdir("gui/assets/lang")
-	runPrint("go", "run", "../../../cmd/transifexdl/main.go")
+	runPrint("go", "run", "../../../script/transifexdl.go")
 	os.Chdir("../../..")
 	assets()
 }
@@ -336,13 +443,18 @@ func clean() {
 }
 
 func ldflags() string {
-	var b bytes.Buffer
+	sep := ' '
+	if goVersion > 1.4 {
+		sep = '='
+	}
+
+	b := new(bytes.Buffer)
 	b.WriteString("-w")
-	b.WriteString(fmt.Sprintf(" -X main.Version %s", version))
-	b.WriteString(fmt.Sprintf(" -X main.BuildStamp %d", buildStamp()))
-	b.WriteString(fmt.Sprintf(" -X main.BuildUser %s", buildUser()))
-	b.WriteString(fmt.Sprintf(" -X main.BuildHost %s", buildHost()))
-	b.WriteString(fmt.Sprintf(" -X main.BuildEnv %s", buildEnvironment()))
+	fmt.Fprintf(b, " -X main.Version%c%s", sep, version)
+	fmt.Fprintf(b, " -X main.BuildStamp%c%d", sep, buildStamp())
+	fmt.Fprintf(b, " -X main.BuildUser%c%s", sep, buildUser())
+	fmt.Fprintf(b, " -X main.BuildHost%c%s", sep, buildHost())
+	fmt.Fprintf(b, " -X main.BuildEnv%c%s", sep, buildEnvironment())
 	return b.String()
 }
 
@@ -480,8 +592,9 @@ func runPipe(file, cmd string, args ...string) {
 }
 
 type archiveFile struct {
-	src string
-	dst string
+	src  string
+	dst  string
+	perm os.FileMode
 }
 
 func tarGz(out string, files []archiveFile) {
@@ -639,7 +752,9 @@ func vet(pkg string) {
 		if falseAlarmComposites.Match(line) || exitStatus.Match(line) {
 			continue
 		}
-		log.Printf("%s", line)
+		if len(line) > 0 {
+			log.Printf("%s", line)
+		}
 	}
 }
 
@@ -655,6 +770,8 @@ func lint(pkg string) {
 		if analCommentPolicy.Match(line) {
 			continue
 		}
-		log.Printf("%s", line)
+		if len(line) > 0 {
+			log.Printf("%s", line)
+		}
 	}
 }

build.sh

@@ -4,79 +4,97 @@ IFS=$'\n\t'
 
 STTRACE=${STTRACE:-}
 
+script() {
+	name="$1"
+	shift
+	go run "script/$name.go" "$@"
+}
+
+build() {
+	go run build.go "$@"
+}
+
 case "${1:-default}" in
 	default)
-		go run build.go
+		build
 		;;
 
 	clean)
-		go run build.go "$1"
-		;;
-
-	test)
-		ulimit -t 60 &>/dev/null || true
-		ulimit -d 512000 &>/dev/null || true
-		ulimit -m 512000 &>/dev/null || true
-
-		go run build.go "$1"
+		build "$@"
 		;;
 
 	tar)
-		go run build.go "$1"
+		build "$@"
 		;;
 
 	deps)
-		go run build.go "$1"
+		build "$@"
 		;;
 
 	assets)
-		go run build.go "$1"
+		build "$@"
 		;;
 
 	xdr)
-		go run build.go "$1"
+		build "$@"
 		;;
 
 	translate)
-		go run build.go "$1"
+		build "$@"
 		;;
 
-	transifex)
-		go run build.go "$1"
-		;;
-
-	noupgrade)
-		go run build.go -no-upgrade tar
-		;;
-
-	all)
-		go run build.go -goos darwin -goarch amd64 tar
-		go run build.go -goos darwin -goarch 386 tar
-
-		go run build.go -goos dragonfly -goarch 386 tar
-		go run build.go -goos dragonfly -goarch amd64 tar
-
-		go run build.go -goos freebsd -goarch 386 tar
-		go run build.go -goos freebsd -goarch amd64 tar
-
-		go run build.go -goos linux -goarch 386 tar
-		go run build.go -goos linux -goarch amd64 tar
-		go run build.go -goos linux -goarch arm tar
-
-		go run build.go -goos netbsd -goarch 386 tar
-		go run build.go -goos netbsd -goarch amd64 tar
-
-		go run build.go -goos openbsd -goarch 386 tar
-		go run build.go -goos openbsd -goarch amd64 tar
-
-		go run build.go -goos solaris -goarch amd64 tar
-
-		go run build.go -goos windows -goarch 386 zip
-		go run build.go -goos windows -goarch amd64 zip
+	deb)
+		build "$@"
 		;;
 
 	setup)
-		echo "Don't worry, just build."
+		build "$@"
+		;;
+
+	test)
+		ulimit -t 600 &>/dev/null || true
+		ulimit -d 512000 &>/dev/null || true
+		ulimit -m 512000 &>/dev/null || true
+		LOGGER_DISCARD=1 build test
+		;;
+
+	bench)
+		LOGGER_DISCARD=1 build bench | script benchfilter
+		;;
+
+	prerelease)
+		build transifex
+		git add -A gui/assets/ lib/auto/
+		pushd man ; ./refresh.sh ; popd
+		git add -A man
+		;;
+
+	noupgrade)
+		build -no-upgrade tar
+		;;
+
+	all)
+		build -goos darwin -goarch amd64 tar
+
+		build -goos dragonfly -goarch amd64 tar
+
+		build -goos freebsd -goarch 386 tar
+		build -goos freebsd -goarch amd64 tar
+
+		build -goos linux -goarch 386 tar
+		build -goos linux -goarch amd64 tar
+		build -goos linux -goarch arm tar
+
+		build -goos netbsd -goarch 386 tar
+		build -goos netbsd -goarch amd64 tar
+
+		build -goos openbsd -goarch 386 tar
+		build -goos openbsd -goarch amd64 tar
+
+		build -goos solaris -goarch amd64 tar
+
+		build -goos windows -goarch 386 zip
+		build -goos windows -goarch amd64 zip
 		;;
 
 	test-cov)
@@ -84,9 +102,6 @@ case "${1:-default}" in
 		ulimit -d 512000 &>/dev/null || true
 		ulimit -m 512000 &>/dev/null || true
 
-		go get github.com/axw/gocov/gocov
-		go get github.com/AlekSi/gocov-xml
-
 		echo "mode: set" > coverage.out
 		fail=0
 
@@ -112,22 +127,25 @@ case "${1:-default}" in
 		;;
 
 	docker-all)
+		img=${DOCKERIMG:-syncthing/build:latest}
 		docker run --rm -h syncthing-builder -u $(id -u) -t \
 			-v $(pwd):/go/src/github.com/syncthing/syncthing \
 			-w /go/src/github.com/syncthing/syncthing \
 			-e "STTRACE=$STTRACE" \
-			syncthing/build:latest \
+			"$img" \
 			sh -c './build.sh clean \
-				&& ./build.sh all \
-				&& STTRACE=all ./build.sh test-cov'
+				&& ./build.sh test-cov \
+				&& ./build.sh bench \
+				&& ./build.sh all'
 		;;
 
 	docker-test)
+		img=${DOCKERIMG:-syncthing/build:latest}
 		docker run --rm -h syncthing-builder -u $(id -u) -t \
 			-v $(pwd):/go/src/github.com/syncthing/syncthing \
 			-w /go/src/github.com/syncthing/syncthing \
 			-e "STTRACE=$STTRACE" \
-			syncthing/build:latest \
+			"$img" \
 			sh -euxc './build.sh clean \
 				&& go run build.go -race \
 				&& export GOPATH=$(pwd)/Godeps/_workspace:$GOPATH \
@@ -137,21 +155,23 @@ case "${1:-default}" in
 		;;
 
 	docker-lint)
+		img=${DOCKERIMG:-syncthing/build:latest}
 		docker run --rm -h syncthing-builder -u $(id -u) -t \
 			-v $(pwd):/go/src/github.com/syncthing/syncthing \
 			-w /go/src/github.com/syncthing/syncthing \
 			-e "STTRACE=$STTRACE" \
-			syncthing/build:latest \
+			"$img" \
 			sh -euxc 'go run build.go lint'
 		;;
 
 
 	docker-vet)
+		img=${DOCKERIMG:-syncthing/build:latest}
 		docker run --rm -h syncthing-builder -u $(id -u) -t \
 			-v $(pwd):/go/src/github.com/syncthing/syncthing \
 			-w /go/src/github.com/syncthing/syncthing \
 			-e "STTRACE=$STTRACE" \
-			syncthing/build:latest \
+			"$img" \
 			sh -euxc 'go run build.go vet'
 		;;
 

check-contrib.sh

@@ -1,52 +0,0 @@
-#!/bin/bash
-
-missing-authors() {
-	for email in $(git log --format=%ae HEAD | sort | uniq) ; do
-		grep -q "$email" AUTHORS || echo $email
-	done
-}
-
-no-docs-typos() {
-	# Commits that are known to not change code
-	grep -v 63bd0136fb40a91efaa279cb4b4159d82e8e6904 |\
-	grep -v 4e2feb6fbc791bb8a2daf0ab8efb10775d66343e |\
-	grep -v f2459ef3319b2f060dbcdacd0c35a1788a94b8bd |\
-	grep -v b61f418bf2d1f7d5a9d7088a20a2a448e5e66801 |\
-	grep -v f0621207e3953711f9ab86d99724f1d0faac45b1 |\
-	grep -v f1120d7aa936c0658429edef0037792520b46334 |\
-	grep -v a9339d0627fff439879d157c75077f02c9fac61b |\
-	grep -v 254c63763a3ad42fd82259f1767db526cff94a14 |\
-	grep -v 4b76ec40c07078beaa2c5e250ed7d9bd6276a718 |\
-	grep -v ffc39dfbcb34eacc3ea12327a02b6e7741a2c207 |\
-	grep -v 32a76901a91ff0f663db6f0830e0aedec946e4d0 |\
-	grep -v af3288043a49bcc28f8ae3060852a09de552fe5f
-}
-
-print-missing-authors() {
-	for email in $(missing-authors) ; do
-		git log --author="$email" --format="%H %ae %s" | no-docs-typos
-	done
-}
-
-print-missing-copyright() {
-	find . -name \*.go | xargs egrep -L 'Copyright \(C\)|automatically generated' | grep -v Godeps | grep -v internal/auto/
-}
-
-authors=$(print-missing-authors)
-if [[ ! -z $authors ]] ; then
-	echo '***'
-	echo Author emails not in AUTHORS:
-	echo $authors
-	echo '***'
-	exit 1
-fi
-
-copy=$(print-missing-copyright)
-if [[ ! -z $copy ]] ; then
-	echo ***
-	echo Files missing copyright notice:
-	echo $copy
-	echo ***
-	exit 1
-fi
-

cmd/stcompdirs/main.go

@@ -16,7 +16,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/syncthing/syncthing/internal/symlinks"
+	"github.com/syncthing/syncthing/lib/symlinks"
 )
 
 func main() {

cmd/stfileinfo/main.go

@@ -13,7 +13,7 @@ import (
 	"path/filepath"
 
 	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/internal/scanner"
+	"github.com/syncthing/syncthing/lib/scanner"
 )
 
 func main() {

cmd/stfinddevice/main.go

@@ -12,7 +12,7 @@ import (
 	"os"
 
 	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/internal/discover"
+	"github.com/syncthing/syncthing/lib/discover"
 )
 
 func main() {

cmd/stindex/main.go

@@ -14,7 +14,7 @@ import (
 	"os"
 
 	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/internal/db"
+	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 )
@@ -65,13 +65,13 @@ func main() {
 			fmt.Printf("[block] F:%q H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
 
 		case db.KeyTypeDeviceStatistic:
-			fmt.Printf("[dstat]\n  %x\n  %x", it.Key(), it.Value())
+			fmt.Printf("[dstat]\n  %x\n  %x\n", it.Key(), it.Value())
 
 		case db.KeyTypeFolderStatistic:
-			fmt.Printf("[fstat]\n  %x\n  %x", it.Key(), it.Value())
+			fmt.Printf("[fstat]\n  %x\n  %x\n", it.Key(), it.Value())
 
 		default:
-			fmt.Printf("[???]\n  %x\n  %x", it.Key(), it.Value())
+			fmt.Printf("[???]\n  %x\n  %x\n", it.Key(), it.Value())
 		}
 	}
 }

cmd/stwatchfile/main.go

@@ -0,0 +1,98 @@
+// Copyright (C) 2015 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package main
+
+import (
+	"bytes"
+	"crypto/md5"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"time"
+)
+
+func getmd5(filePath string) ([]byte, error) {
+	var result []byte
+	file, err := os.Open(filePath)
+	if err != nil {
+		return result, err
+	}
+	defer file.Close()
+
+	hash := md5.New()
+	if _, err := io.Copy(hash, file); err != nil {
+		return result, err
+	}
+
+	return hash.Sum(result), nil
+}
+
+func main() {
+	period := flag.Duration("period", 200*time.Millisecond, "Sleep period between checks")
+	flag.Parse()
+
+	file := flag.Arg(0)
+
+	if file == "" {
+		fmt.Println("Expects a path as an argument")
+		return
+	}
+
+	exists := true
+	size := int64(0)
+	mtime := time.Time{}
+	hash := []byte{}
+
+	for {
+		time.Sleep(*period)
+
+		newExists := true
+		fi, err := os.Stat(file)
+		if err != nil && os.IsNotExist(err) {
+			newExists = false
+		} else if err != nil {
+			fmt.Println("stat:", err)
+			return
+		}
+
+		if newExists != exists {
+			exists = newExists
+			if !newExists {
+				fmt.Println(file, "does not exist")
+			} else {
+				fmt.Println(file, "appeared")
+			}
+		}
+
+		if !exists {
+			size = 0
+			mtime = time.Time{}
+			hash = []byte{}
+			continue
+		}
+
+		if fi.IsDir() {
+			fmt.Println(file, "is directory")
+			return
+		}
+		newSize := fi.Size()
+		newMtime := fi.ModTime()
+
+		newHash, err := getmd5(file)
+		if err != nil {
+			fmt.Println("getmd5:", err)
+		}
+
+		if newSize != size || newMtime != mtime || !bytes.Equal(newHash, hash) {
+			fmt.Println(file, "Size:", newSize, "Mtime:", newMtime, "Hash:", fmt.Sprintf("%x", newHash))
+			hash = newHash
+			size = newSize
+			mtime = newMtime
+		}
+	}
+}

cmd/syncthing/audit.go

@@ -10,7 +10,7 @@ import (
 	"encoding/json"
 	"io"
 
-	"github.com/syncthing/syncthing/internal/events"
+	"github.com/syncthing/syncthing/lib/events"
 )
 
 // The auditSvc subscribes to events and writes these in JSON format, one

cmd/syncthing/auditsvc_test.go

@@ -12,7 +12,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/syncthing/syncthing/internal/events"
+	"github.com/syncthing/syncthing/lib/events"
 )
 
 func TestAuditService(t *testing.T) {

cmd/syncthing/connections.go

@@ -15,9 +15,9 @@ import (
 	"time"
 
 	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/internal/config"
-	"github.com/syncthing/syncthing/internal/events"
-	"github.com/syncthing/syncthing/internal/model"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/model"
 	"github.com/thejerf/suture"
 )
 
@@ -177,10 +177,6 @@ next:
 				if debugNet {
 					l.Debugf("cipher suite: %04X in lan: %t", conn.ConnectionState().CipherSuite, !limit)
 				}
-				events.Default.Log(events.DeviceConnected, map[string]string{
-					"id":   remoteID.String(),
-					"addr": conn.RemoteAddr().String(),
-				})
 
 				s.model.AddConnection(conn, protoConn)
 				continue next
@@ -353,3 +349,24 @@ func (s *connectionSvc) shouldLimit(addr net.Addr) bool {
 	}
 	return !tcpaddr.IP.IsLoopback()
 }
+
+func (s *connectionSvc) VerifyConfiguration(from, to config.Configuration) error {
+	return nil
+}
+
+func (s *connectionSvc) CommitConfiguration(from, to config.Configuration) bool {
+	// We require a restart if a device as been removed.
+
+	newDevices := make(map[protocol.DeviceID]bool, len(to.Devices))
+	for _, dev := range to.Devices {
+		newDevices[dev.DeviceID] = true
+	}
+
+	for _, dev := range from.Devices {
+		if !newDevices[dev.DeviceID] {
+			return false
+		}
+	}
+
+	return true
+}

cmd/syncthing/debug.go

@@ -12,6 +12,7 @@ import (
 )
 
 var (
-	debugNet  = strings.Contains(os.Getenv("STTRACE"), "net") || os.Getenv("STTRACE") == "all"
-	debugHTTP = strings.Contains(os.Getenv("STTRACE"), "http") || os.Getenv("STTRACE") == "all"
+	debugNet    = strings.Contains(os.Getenv("STTRACE"), "net") || os.Getenv("STTRACE") == "all"
+	debugHTTP   = strings.Contains(os.Getenv("STTRACE"), "http") || os.Getenv("STTRACE") == "all"
+	debugSuture = strings.Contains(os.Getenv("STTRACE"), "suture") || os.Getenv("STTRACE") == "all"
 )

cmd/syncthing/gui.go

@@ -26,15 +26,15 @@ import (
 
 	"github.com/calmh/logger"
 	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/internal/auto"
-	"github.com/syncthing/syncthing/internal/config"
-	"github.com/syncthing/syncthing/internal/db"
-	"github.com/syncthing/syncthing/internal/discover"
-	"github.com/syncthing/syncthing/internal/events"
-	"github.com/syncthing/syncthing/internal/model"
-	"github.com/syncthing/syncthing/internal/osutil"
-	"github.com/syncthing/syncthing/internal/sync"
-	"github.com/syncthing/syncthing/internal/upgrade"
+	"github.com/syncthing/syncthing/lib/auto"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/discover"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/model"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syncthing/syncthing/lib/upgrade"
 	"github.com/vitrun/qart/qr"
 	"golang.org/x/crypto/bcrypt"
 )
@@ -49,31 +49,36 @@ var (
 	guiErrors    = []guiError{}
 	guiErrorsMut = sync.NewMutex()
 	startTime    = time.Now()
-	eventSub     *events.BufferedSubscription
 )
 
 type apiSvc struct {
-	cfg      config.GUIConfiguration
-	assetDir string
-	model    *model.Model
-	fss      *folderSummarySvc
-	listener net.Listener
+	id              protocol.DeviceID
+	cfg             config.GUIConfiguration
+	assetDir        string
+	model           *model.Model
+	listener        net.Listener
+	fss             *folderSummarySvc
+	stop            chan struct{}
+	systemConfigMut sync.Mutex
+	eventSub        *events.BufferedSubscription
 }
 
-func newAPISvc(cfg config.GUIConfiguration, assetDir string, m *model.Model) (*apiSvc, error) {
+func newAPISvc(id protocol.DeviceID, cfg config.GUIConfiguration, assetDir string, m *model.Model, eventSub *events.BufferedSubscription) (*apiSvc, error) {
 	svc := &apiSvc{
-		cfg:      cfg,
-		assetDir: assetDir,
-		model:    m,
-		fss:      newFolderSummarySvc(m),
+		id:              id,
+		cfg:             cfg,
+		assetDir:        assetDir,
+		model:           m,
+		systemConfigMut: sync.NewMutex(),
+		eventSub:        eventSub,
 	}
 
 	var err error
-	svc.listener, err = svc.getListener()
+	svc.listener, err = svc.getListener(cfg)
 	return svc, err
 }
 
-func (s *apiSvc) getListener() (net.Listener, error) {
+func (s *apiSvc) getListener(cfg config.GUIConfiguration) (net.Listener, error) {
 	cert, err := tls.LoadX509KeyPair(locations[locHTTPSCertFile], locations[locHTTPSKeyFile])
 	if err != nil {
 		l.Infoln("Loading HTTPS certificate:", err)
@@ -110,7 +115,7 @@ func (s *apiSvc) getListener() (net.Listener, error) {
 		},
 	}
 
-	rawListener, err := net.Listen("tcp", s.cfg.Address)
+	rawListener, err := net.Listen("tcp", cfg.Address)
 	if err != nil {
 		return nil, err
 	}
@@ -120,10 +125,9 @@ func (s *apiSvc) getListener() (net.Listener, error) {
 }
 
 func (s *apiSvc) Serve() {
+	s.stop = make(chan struct{})
+
 	l.AddHandler(logger.LevelWarn, s.showGuiError)
-	sub := events.Default.Subscribe(events.AllEvents)
-	eventSub = events.NewBufferedSubscription(sub, 1000)
-	defer events.Default.Unsubscribe(sub)
 
 	// The GET handlers
 	getRestMux := http.NewServeMux()
@@ -186,14 +190,14 @@ func (s *apiSvc) Serve() {
 
 	// Wrap everything in CSRF protection. The /rest prefix should be
 	// protected, other requests will grant cookies.
-	handler := csrfMiddleware("/rest", s.cfg.APIKey, mux)
+	handler := csrfMiddleware(s.id.String()[:5], "/rest", s.cfg.APIKey, mux)
 
-	// Add our version as a header to responses
-	handler = withVersionMiddleware(handler)
+	// Add our version and ID as a header to responses
+	handler = withDetailsMiddleware(s.id, handler)
 
 	// Wrap everything in basic auth, if user/password is set.
 	if len(s.cfg.User) > 0 && len(s.cfg.Password) > 0 {
-		handler = basicAuthAndSessionMiddleware(s.cfg, handler)
+		handler = basicAuthAndSessionMiddleware("sessionid-"+s.id.String()[:5], s.cfg, handler)
 	}
 
 	// Redirect to HTTPS if we are supposed to
@@ -210,15 +214,63 @@ func (s *apiSvc) Serve() {
 		ReadTimeout: 10 * time.Second,
 	}
 
+	s.fss = newFolderSummarySvc(s.model)
+	defer s.fss.Stop()
 	s.fss.ServeBackground()
 
+	l.Infoln("API listening on", s.listener.Addr())
 	err := srv.Serve(s.listener)
-	l.Warnln("API:", err)
+
+	// The return could be due to an intentional close. Wait for the stop
+	// signal before returning. IF there is no stop signal within a second, we
+	// assume it was unintentional and log the error before retrying.
+	select {
+	case <-s.stop:
+	case <-time.After(time.Second):
+		l.Warnln("API:", err)
+	}
 }
 
 func (s *apiSvc) Stop() {
+	close(s.stop)
 	s.listener.Close()
-	s.fss.Stop()
+}
+
+func (s *apiSvc) String() string {
+	return fmt.Sprintf("apiSvc@%p", s)
+}
+
+func (s *apiSvc) VerifyConfiguration(from, to config.Configuration) error {
+	return nil
+}
+
+func (s *apiSvc) CommitConfiguration(from, to config.Configuration) bool {
+	if to.GUI == from.GUI {
+		return true
+	}
+
+	// Order here is important. We must close the listener to stop Serve(). We
+	// must create a new listener before Serve() starts again. We can't create
+	// a new listener on the same port before the previous listener is closed.
+	// To assist in this little dance the Serve() method will wait for a
+	// signal on the stop channel after the listener has closed.
+
+	s.listener.Close()
+
+	var err error
+	s.listener, err = s.getListener(to.GUI)
+	if err != nil {
+		// Ideally this should be a verification error, but we check it by
+		// creating a new listener which requires shutting down the previous
+		// one first, which is too destructive for the VerifyConfiguration
+		// method.
+		return false
+	}
+	s.cfg = to.GUI
+
+	close(s.stop)
+
+	return true
 }
 
 func getPostHandler(get, post http.Handler) http.Handler {
@@ -284,9 +336,10 @@ func noCacheMiddleware(h http.Handler) http.Handler {
 	})
 }
 
-func withVersionMiddleware(h http.Handler) http.Handler {
+func withDetailsMiddleware(id protocol.DeviceID, h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("X-Syncthing-Version", Version)
+		w.Header().Set("X-Syncthing-ID", id.String())
 		h.ServeHTTP(w, r)
 	})
 }
@@ -302,6 +355,7 @@ func (s *apiSvc) getSystemVersion(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json; charset=utf-8")
 	json.NewEncoder(w).Encode(map[string]string{
 		"version":     Version,
+		"codename":    Codename,
 		"longVersion": LongVersion,
 		"os":          runtime.GOOS,
 		"arch":        runtime.GOARCH,
@@ -375,7 +429,10 @@ func folderSummary(m *model.Model, folder string) map[string]interface{} {
 		res["error"] = err.Error()
 	}
 
-	res["version"] = m.CurrentLocalVersion(folder) + m.RemoteLocalVersion(folder)
+	lv, _ := m.CurrentLocalVersion(folder)
+	rv, _ := m.RemoteLocalVersion(folder)
+
+	res["version"] = lv + rv
 
 	ignorePatterns, _, _ := m.GetIgnores(folder)
 	res["ignorePatterns"] = false
@@ -464,49 +521,46 @@ func (s *apiSvc) getSystemConfig(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *apiSvc) postSystemConfig(w http.ResponseWriter, r *http.Request) {
-	var newCfg config.Configuration
-	err := json.NewDecoder(r.Body).Decode(&newCfg)
+	s.systemConfigMut.Lock()
+	defer s.systemConfigMut.Unlock()
+
+	var to config.Configuration
+	err := json.NewDecoder(r.Body).Decode(&to)
 	if err != nil {
 		l.Warnln("decoding posted config:", err)
 		http.Error(w, err.Error(), 500)
 		return
 	}
 
-	if newCfg.GUI.Password != cfg.GUI().Password {
-		if newCfg.GUI.Password != "" {
-			hash, err := bcrypt.GenerateFromPassword([]byte(newCfg.GUI.Password), 0)
+	if to.GUI.Password != cfg.GUI().Password {
+		if to.GUI.Password != "" {
+			hash, err := bcrypt.GenerateFromPassword([]byte(to.GUI.Password), 0)
 			if err != nil {
 				l.Warnln("bcrypting password:", err)
 				http.Error(w, err.Error(), 500)
 				return
 			}
 
-			newCfg.GUI.Password = string(hash)
+			to.GUI.Password = string(hash)
 		}
 	}
 
-	// Start or stop usage reporting as appropriate
+	// Fixup usage reporting settings
 
-	if curAcc := cfg.Options().URAccepted; newCfg.Options.URAccepted > curAcc {
+	if curAcc := cfg.Options().URAccepted; to.Options.URAccepted > curAcc {
 		// UR was enabled
-		newCfg.Options.URAccepted = usageReportVersion
-		newCfg.Options.URUniqueID = randomString(8)
-		err := sendUsageReport(s.model)
-		if err != nil {
-			l.Infoln("Usage report:", err)
-		}
-		go usageReportingLoop(s.model)
-	} else if newCfg.Options.URAccepted < curAcc {
+		to.Options.URAccepted = usageReportVersion
+		to.Options.URUniqueID = randomString(8)
+	} else if to.Options.URAccepted < curAcc {
 		// UR was disabled
-		newCfg.Options.URAccepted = -1
-		newCfg.Options.URUniqueID = ""
-		stopUsageReporting()
+		to.Options.URAccepted = -1
+		to.Options.URUniqueID = ""
 	}
 
 	// Activate and save
 
-	configInSync = !config.ChangeRequiresRestart(cfg.Raw(), newCfg)
-	cfg.Replace(newCfg)
+	resp := cfg.Replace(to)
+	configInSync = !resp.RequiresRestart
 	cfg.Save()
 }
 
@@ -523,21 +577,26 @@ func (s *apiSvc) postSystemRestart(w http.ResponseWriter, r *http.Request) {
 func (s *apiSvc) postSystemReset(w http.ResponseWriter, r *http.Request) {
 	var qs = r.URL.Query()
 	folder := qs.Get("folder")
-	var err error
-	if len(folder) == 0 {
-		err = resetDB()
-	} else {
-		err = s.model.ResetFolder(folder)
-	}
-	if err != nil {
-		http.Error(w, err.Error(), 500)
-		return
+
+	if len(folder) > 0 {
+		if _, ok := cfg.Folders()[folder]; !ok {
+			http.Error(w, "Invalid folder ID", 500)
+			return
+		}
 	}
+
 	if len(folder) == 0 {
+		// Reset all folders.
+		for folder := range cfg.Folders() {
+			s.model.ResetFolder(folder)
+		}
 		s.flushResponse(`{"ok": "resetting database"}`, w)
 	} else {
-		s.flushResponse(`{"ok": "resetting folder " + folder}`, w)
+		// Reset a specific folder, assuming it's supposed to exist.
+		s.model.ResetFolder(folder)
+		s.flushResponse(`{"ok": "resetting folder `+folder+`"}`, w)
 	}
+
 	go restart()
 }
 
@@ -694,7 +753,7 @@ func (s *apiSvc) getEvents(w http.ResponseWriter, r *http.Request) {
 	f := w.(http.Flusher)
 	f.Flush()
 
-	evs := eventSub.Since(since, nil)
+	evs := s.eventSub.Since(since, nil)
 	if 0 < limit && limit < len(evs) {
 		evs = evs[len(evs)-limit:]
 	}
@@ -897,6 +956,11 @@ func (s embeddedStatic) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if r.Header.Get("If-Modified-Since") == auto.AssetsBuildDate {
+		w.WriteHeader(http.StatusNotModified)
+		return
+	}
+
 	mtype := s.mimeTypeForFile(file)
 	if len(mtype) != 0 {
 		w.Header().Set("Content-Type", mtype)
@@ -912,6 +976,7 @@ func (s embeddedStatic) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(bs)))
 	w.Header().Set("Last-Modified", auto.AssetsBuildDate)
+	w.Header().Set("Cache-Control", "public")
 
 	w.Write(bs)
 }

cmd/syncthing/gui_auth.go

@@ -14,8 +14,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/syncthing/syncthing/internal/config"
-	"github.com/syncthing/syncthing/internal/sync"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/sync"
 	"golang.org/x/crypto/bcrypt"
 )
 
@@ -24,14 +24,15 @@ var (
 	sessionsMut = sync.NewMutex()
 )
 
-func basicAuthAndSessionMiddleware(cfg config.GUIConfiguration, next http.Handler) http.Handler {
+func basicAuthAndSessionMiddleware(cookieName string, cfg config.GUIConfiguration, next http.Handler) http.Handler {
+
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if cfg.APIKey != "" && r.Header.Get("X-API-Key") == cfg.APIKey {
 			next.ServeHTTP(w, r)
 			return
 		}
 
-		cookie, err := r.Cookie("sessionid")
+		cookie, err := r.Cookie(cookieName)
 		if err == nil && cookie != nil {
 			sessionsMut.Lock()
 			_, ok := sessions[cookie.Value]
@@ -86,7 +87,7 @@ func basicAuthAndSessionMiddleware(cfg config.GUIConfiguration, next http.Handle
 		sessions[sessionid] = true
 		sessionsMut.Unlock()
 		http.SetCookie(w, &http.Cookie{
-			Name:   "sessionid",
+			Name:   cookieName,
 			Value:  sessionid,
 			MaxAge: 0,
 		})

cmd/syncthing/gui_csrf.go

@@ -12,10 +12,9 @@ import (
 	"net/http"
 	"os"
 	"strings"
-	"time"
 
-	"github.com/syncthing/syncthing/internal/osutil"
-	"github.com/syncthing/syncthing/internal/sync"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/sync"
 )
 
 var csrfTokens []string
@@ -24,7 +23,7 @@ var csrfMut = sync.NewMutex()
 // Check for CSRF token on /rest/ URLs. If a correct one is not given, reject
 // the request with 403. For / and /index.html, set a new CSRF cookie if none
 // is currently set.
-func csrfMiddleware(prefix, apiKey string, next http.Handler) http.Handler {
+func csrfMiddleware(unique, prefix, apiKey string, next http.Handler) http.Handler {
 	loadCsrfTokens()
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// Allow requests carrying a valid API key
@@ -35,10 +34,10 @@ func csrfMiddleware(prefix, apiKey string, next http.Handler) http.Handler {
 
 		// Allow requests for the front page, and set a CSRF cookie if there isn't already a valid one.
 		if !strings.HasPrefix(r.URL.Path, prefix) {
-			cookie, err := r.Cookie("CSRF-Token")
+			cookie, err := r.Cookie("CSRF-Token-" + unique)
 			if err != nil || !validCsrfToken(cookie.Value) {
 				cookie = &http.Cookie{
-					Name:  "CSRF-Token",
+					Name:  "CSRF-Token-" + unique,
 					Value: newCsrfToken(),
 				}
 				http.SetCookie(w, cookie)
@@ -54,7 +53,7 @@ func csrfMiddleware(prefix, apiKey string, next http.Handler) http.Handler {
 		}
 
 		// Verify the CSRF token
-		token := r.Header.Get("X-CSRF-Token")
+		token := r.Header.Get("X-CSRF-Token-" + unique)
 		if !validCsrfToken(token) {
 			http.Error(w, "CSRF Error", 403)
 			return
@@ -91,28 +90,20 @@ func newCsrfToken() string {
 }
 
 func saveCsrfTokens() {
-	name := locations[locCsrfTokens]
-	tmp := fmt.Sprintf("%s.tmp.%d", name, time.Now().UnixNano())
+	// We're ignoring errors in here. It's not super critical and there's
+	// nothing relevant we can do about them anyway...
 
-	f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
+	name := locations[locCsrfTokens]
+	f, err := osutil.CreateAtomic(name, 0600)
 	if err != nil {
 		return
 	}
-	defer os.Remove(tmp)
 
 	for _, t := range csrfTokens {
-		_, err := fmt.Fprintln(f, t)
-		if err != nil {
-			return
-		}
+		fmt.Fprintln(f, t)
 	}
 
-	err = f.Close()
-	if err != nil {
-		return
-	}
-
-	osutil.Rename(tmp, name)
+	f.Close()
 }
 
 func loadCsrfTokens() {

cmd/syncthing/locations.go

@@ -13,7 +13,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/syncthing/syncthing/internal/osutil"
+	"github.com/syncthing/syncthing/lib/osutil"
 )
 
 type locationEnum string
@@ -31,6 +31,7 @@ const (
 	locCsrfTokens                 = "csrfTokens"
 	locPanicLog                   = "panicLog"
 	locAuditLog                   = "auditLog"
+	locGUIAssets                  = "GUIAssets"
 	locDefFolder                  = "defFolder"
 )
 
@@ -52,6 +53,7 @@ var locations = map[locationEnum]string{
 	locCsrfTokens:    "${config}/csrftokens.txt",
 	locPanicLog:      "${config}/panic-${timestamp}.log",
 	locAuditLog:      "${config}/audit-${timestamp}.log",
+	locGUIAssets:     "${config}/gui",
 	locDefFolder:     "${home}/Sync",
 }
 

cmd/syncthing/main.go

@@ -28,14 +28,14 @@ import (
 	"github.com/calmh/logger"
 	"github.com/juju/ratelimit"
 	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/internal/config"
-	"github.com/syncthing/syncthing/internal/db"
-	"github.com/syncthing/syncthing/internal/discover"
-	"github.com/syncthing/syncthing/internal/events"
-	"github.com/syncthing/syncthing/internal/model"
-	"github.com/syncthing/syncthing/internal/osutil"
-	"github.com/syncthing/syncthing/internal/symlinks"
-	"github.com/syncthing/syncthing/internal/upgrade"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/discover"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/model"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/symlinks"
+	"github.com/syncthing/syncthing/lib/upgrade"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -45,6 +45,7 @@ import (
 
 var (
 	Version     = "unknown-dev"
+	Codename    = "Aluminium Ant"
 	BuildEnv    = "default"
 	BuildStamp  = "0"
 	BuildDate   time.Time
@@ -93,7 +94,7 @@ func init() {
 	BuildDate = time.Unix(int64(stamp), 0)
 
 	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf("syncthing %s (%s %s-%s %s) %s@%s %s", Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildEnv, BuildUser, BuildHost, date)
+	LongVersion = fmt.Sprintf(`syncthing %s "%s" (%s %s-%s %s) %s@%s %s`, Version, Codename, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildEnv, BuildUser, BuildHost, date)
 
 	if os.Getenv("STTRACE") != "" {
 		logFlags = log.Ltime | log.Ldate | log.Lmicroseconds | log.Lshortfile
@@ -155,6 +156,7 @@ are mostly useful for developers. Use with care.
                  - "model"    (the model package)
                  - "scanner"  (the scanner package)
                  - "stats"    (the stats package)
+                 - "suture"   (the suture package; service management)
                  - "upnp"     (the upnp package)
                  - "xdr"      (the xdr package)
                  - "all"      (all of the above)
@@ -251,6 +253,10 @@ func main() {
 		l.Fatalln(err)
 	}
 
+	if guiAssets == "" {
+		guiAssets = locations[locGUIAssets]
+	}
+
 	if runtime.GOOS == "windows" {
 		if logFile == "" {
 			// Use the default log file location
@@ -279,7 +285,7 @@ func main() {
 			l.Fatalln(dir, "is not a directory")
 		}
 		if err != nil && os.IsNotExist(err) {
-			err = os.MkdirAll(dir, 0700)
+			err = osutil.MkdirAll(dir, 0700)
 			if err != nil {
 				l.Fatalln("generate:", err)
 			}
@@ -420,11 +426,12 @@ func upgradeViaRest() error {
 
 func syncthingMain() {
 	// Create a main service manager. We'll add things to this as we go along.
-	// We want any logging it does to go through our log system, with INFO
-	// severity.
+	// We want any logging it does to go through our log system.
 	mainSvc := suture.New("main", suture.Spec{
 		Log: func(line string) {
-			l.Infoln(line)
+			if debugSuture {
+				l.Debugln(line)
+			}
 		},
 	})
 	mainSvc.ServeBackground()
@@ -441,11 +448,17 @@ func syncthingMain() {
 		mainSvc.Add(newVerboseSvc())
 	}
 
+	// Event subscription for the API; must start early to catch the early events.
+	apiSub := events.NewBufferedSubscription(events.Default.Subscribe(events.AllEvents), 1000)
+
 	if len(os.Getenv("GOMAXPROCS")) == 0 {
 		runtime.GOMAXPROCS(runtime.NumCPU())
 	}
 
-	events.Default.Log(events.Starting, map[string]string{"home": baseDirs["config"]})
+	// Attempt to increase the limit on number of open files to the maximum
+	// allowed, in case we have many peers. We don't really care enough to
+	// report the error if there is one.
+	osutil.MaximizeOpenFileLimit()
 
 	// Ensure that that we have a certificate and key.
 	cert, err := tls.LoadX509KeyPair(locations[locCertFile], locations[locKeyFile])
@@ -466,6 +479,13 @@ func syncthingMain() {
 	l.Infoln(LongVersion)
 	l.Infoln("My ID:", myID)
 
+	// Emit the Starting event, now that we know who we are.
+
+	events.Default.Log(events.Starting, map[string]string{
+		"home": baseDirs["config"],
+		"myID": myID.String(),
+	})
+
 	// Prepare to be able to save configuration
 
 	cfgFile := locations[locConfigFile]
@@ -551,6 +571,9 @@ func syncthingMain() {
 		symlinks.Supported = false
 	}
 
+	protocol.PingTimeout = time.Duration(opts.PingTimeoutS) * time.Second
+	protocol.PingIdleTime = time.Duration(opts.PingIdleTimeS) * time.Second
+
 	if opts.MaxSendKbps > 0 {
 		writeRateLimit = ratelimit.NewBucketWithRate(float64(1000*opts.MaxSendKbps), int64(5*1000*opts.MaxSendKbps))
 	}
@@ -569,9 +592,19 @@ func syncthingMain() {
 
 	dbFile := locations[locDatabase]
 	ldb, err := leveldb.OpenFile(dbFile, dbOpts())
-	if err != nil && errors.IsCorrupted(err) {
+	if leveldbIsCorrupted(err) {
 		ldb, err = leveldb.RecoverFile(dbFile, dbOpts())
 	}
+	if leveldbIsCorrupted(err) {
+		// The database is corrupted, and we've tried to recover it but it
+		// didn't work. At this point there isn't much to do beyond dropping
+		// the database and reindexing...
+		l.Infoln("Database corruption detected, unable to recover. Reinitializing...")
+		if err := resetDB(); err != nil {
+			l.Fatalln("Remove database:", err)
+		}
+		ldb, err = leveldb.OpenFile(dbFile, dbOpts())
+	}
 	if err != nil {
 		l.Fatalln("Cannot open database:", err, "- Is another copy of Syncthing already running?")
 	}
@@ -586,6 +619,7 @@ func syncthingMain() {
 	}
 
 	m := model.NewModel(cfg, myID, myName, "syncthing", Version, ldb)
+	cfg.Subscribe(m)
 
 	if t := os.Getenv("STDEADLOCKTIMEOUT"); len(t) > 0 {
 		it, err := strconv.Atoi(t)
@@ -596,10 +630,6 @@ func syncthingMain() {
 		m.StartDeadlockDetector(20 * 60 * time.Second)
 	}
 
-	// GUI
-
-	setupGUI(mainSvc, cfg, m)
-
 	// Clear out old indexes for other devices. Otherwise we'll start up and
 	// start needing a bunch of files which are nowhere to be found. This
 	// needs to be changed when we correctly do persistent indexes.
@@ -611,8 +641,21 @@ func syncthingMain() {
 			}
 			m.Index(device, folderCfg.ID, nil, 0, nil)
 		}
+		// Routine to pull blocks from other devices to synchronize the local
+		// folder. Does not run when we are in read only (publish only) mode.
+		if folderCfg.ReadOnly {
+			m.StartFolderRO(folderCfg.ID)
+		} else {
+			m.StartFolderRW(folderCfg.ID)
+		}
 	}
 
+	mainSvc.Add(m)
+
+	// GUI
+
+	setupGUI(mainSvc, cfg, m, apiSub)
+
 	// The default port we announce, possibly modified by setupUPnP next.
 
 	addr, err := net.ResolveTCPAddr("tcp", opts.ListenAddress[0])
@@ -634,27 +677,15 @@ func syncthingMain() {
 	}
 
 	connectionSvc := newConnectionSvc(cfg, myID, m, tlsCfg)
+	cfg.Subscribe(connectionSvc)
 	mainSvc.Add(connectionSvc)
 
-	for _, folder := range cfg.Folders() {
-		// Routine to pull blocks from other devices to synchronize the local
-		// folder. Does not run when we are in read only (publish only) mode.
-		if folder.ReadOnly {
-			l.Okf("Ready to synchronize %s (read only; no external updates accepted)", folder.ID)
-			m.StartFolderRO(folder.ID)
-		} else {
-			l.Okf("Ready to synchronize %s (read-write)", folder.ID)
-			m.StartFolderRW(folder.ID)
-		}
-	}
-
 	if cpuProfile {
 		f, err := os.Create(fmt.Sprintf("cpu-%d.pprof", os.Getpid()))
 		if err != nil {
 			log.Fatal(err)
 		}
 		pprof.StartCPUProfile(f)
-		defer pprof.StopCPUProfile()
 	}
 
 	for _, device := range cfg.Devices() {
@@ -677,16 +708,13 @@ func syncthingMain() {
 			cfg.SetOptions(opts)
 			cfg.Save()
 		}
-		go usageReportingLoop(m)
-		go func() {
-			time.Sleep(10 * time.Minute)
-			err := sendUsageReport(m)
-			if err != nil {
-				l.Infoln("Usage report:", err)
-			}
-		}()
 	}
 
+	// The usageReportingManager registers itself to listen to configuration
+	// changes, and there's nothing more we need to tell it from the outside.
+	// Hence we don't keep the returned pointer.
+	newUsageReportingManager(m, cfg)
+
 	if opts.RestartOnWakeup {
 		go standbyMonitor()
 	}
@@ -701,7 +729,9 @@ func syncthingMain() {
 		}
 	}
 
-	events.Default.Log(events.StartupComplete, nil)
+	events.Default.Log(events.StartupComplete, map[string]string{
+		"myID": myID.String(),
+	})
 	go generatePingEvents()
 
 	cleanConfigDirectory()
@@ -711,29 +741,37 @@ func syncthingMain() {
 	mainSvc.Stop()
 
 	l.Okln("Exiting")
+
+	if cpuProfile {
+		pprof.StopCPUProfile()
+	}
+
 	os.Exit(code)
 }
 
 func dbOpts() *opt.Options {
-	// Calculate a sutiable database block cache capacity. We start at the
-	// default of 8 MiB and use larger values for machines with more memory.
-	// In reality, the database will use twice the amount we calculate here,
-	// as it also has two write buffers each sized at half the block cache.
+	// Calculate a suitable database block cache capacity.
 
+	// Default is 8 MiB.
 	blockCacheCapacity := 8 << 20
-	if bytes, err := memorySize(); err == nil {
-		if bytes > 74<<30 {
-			// At 74 GiB of RAM, we hit a 256 MiB block cache (per the
-			// calculations below). There's probably no point in growing the
-			// cache beyond this point.
-			blockCacheCapacity = 256 << 20
-		} else if bytes > 8<<30 {
-			// Slowly grow from 128 MiB at 8 GiB of RAM up to 256 MiB for a
-			// ~74 GiB RAM machine
-			blockCacheCapacity = int(bytes/512) + 128 - 16
-		} else if bytes > 512<<20 {
-			// Grow from 8 MiB at start to 128 MiB of cache at 8 GiB of RAM.
-			blockCacheCapacity = int(bytes / 64)
+	// Increase block cache up to this maximum:
+	const maxCapacity = 64 << 20
+	// ... which we reach when the box has this much RAM:
+	const maxAtRAM = 8 << 30
+
+	if v := cfg.Options().DatabaseBlockCacheMiB; v != 0 {
+		// Use the value from the config, if it's set.
+		blockCacheCapacity = v << 20
+	} else if bytes, err := memorySize(); err == nil {
+		// We start at the default of 8 MiB and use larger values for machines
+		// with more memory.
+
+		if bytes > maxAtRAM {
+			// Cap the cache at maxCapacity when we reach maxAtRam amount of memory
+			blockCacheCapacity = maxCapacity
+		} else if bytes > maxAtRAM/maxCapacity*int64(blockCacheCapacity) {
+			// Grow from the default to maxCapacity at maxAtRam amount of memory
+			blockCacheCapacity = int(bytes * maxCapacity / maxAtRAM)
 		}
 		l.Infoln("Database block cache capacity", blockCacheCapacity/1024, "KiB")
 	}
@@ -741,7 +779,7 @@ func dbOpts() *opt.Options {
 	return &opt.Options{
 		OpenFilesCacheCapacity: 100,
 		BlockCacheCapacity:     blockCacheCapacity,
-		WriteBuffer:            blockCacheCapacity / 2,
+		WriteBuffer:            4 << 20,
 	}
 }
 
@@ -762,7 +800,7 @@ func startAuditing(mainSvc *suture.Supervisor) {
 	l.Infoln("Audit log in", auditFile)
 }
 
-func setupGUI(mainSvc *suture.Supervisor, cfg *config.Wrapper, m *model.Model) {
+func setupGUI(mainSvc *suture.Supervisor, cfg *config.Wrapper, m *model.Model, apiSub *events.BufferedSubscription) {
 	opts := cfg.Options()
 	guiCfg := overrideGUIConfig(cfg.GUI(), guiAddress, guiAuthentication, guiAPIKey)
 
@@ -791,10 +829,11 @@ func setupGUI(mainSvc *suture.Supervisor, cfg *config.Wrapper, m *model.Model) {
 
 			urlShow := fmt.Sprintf("%s://%s/", proto, net.JoinHostPort(hostShow, strconv.Itoa(addr.Port)))
 			l.Infoln("Starting web GUI on", urlShow)
-			api, err := newAPISvc(guiCfg, guiAssets, m)
+			api, err := newAPISvc(myID, guiCfg, guiAssets, m, apiSub)
 			if err != nil {
 				l.Fatalln("Cannot start GUI:", err)
 			}
+			cfg.Subscribe(api)
 			mainSvc.Add(api)
 
 			if opts.StartBrowser && !noBrowser && !stRestarting {
@@ -814,6 +853,7 @@ func defaultConfig(myName string) config.Configuration {
 			ID:              "default",
 			RawPath:         locations[locDefFolder],
 			RescanIntervalS: 60,
+			MinDiskFreePct:  1,
 			Devices:         []config.FolderDeviceConfiguration{{DeviceID: myID}},
 		},
 	}
@@ -880,7 +920,7 @@ func discovery(extPort int) *discover.Discoverer {
 func ensureDir(dir string, mode int) {
 	fi, err := os.Stat(dir)
 	if os.IsNotExist(err) {
-		err := os.MkdirAll(dir, 0700)
+		err := osutil.MkdirAll(dir, 0700)
 		if err != nil {
 			l.Fatalln(err)
 		}
@@ -1076,3 +1116,19 @@ func checkShortIDs(cfg *config.Wrapper) error {
 	}
 	return nil
 }
+
+// A "better" version of leveldb's errors.IsCorrupted.
+func leveldbIsCorrupted(err error) bool {
+	switch {
+	case err == nil:
+		return false
+
+	case errors.IsCorrupted(err):
+		return true
+
+	case strings.Contains(err.Error(), "corrupted"):
+		return true
+	}
+
+	return false
+}

cmd/syncthing/main_test.go

@@ -11,9 +11,9 @@ import (
 	"testing"
 
 	"github.com/syncthing/protocol"
-	"github.com/syncthing/syncthing/internal/config"
-	"github.com/syncthing/syncthing/internal/db"
-	"github.com/syncthing/syncthing/internal/model"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/db"
+	"github.com/syncthing/syncthing/lib/model"
 
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/storage"

cmd/syncthing/monitor.go

@@ -17,8 +17,8 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/syncthing/syncthing/internal/osutil"
-	"github.com/syncthing/syncthing/internal/sync"
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/sync"
 )
 
 var (
@@ -146,9 +146,8 @@ func monitorMain() {
 						// binary as part of the upgrade process.
 						l.Infoln("Restarting monitor...")
 						os.Setenv("STNORESTART", "")
-						err := exec.Command(args[0], args[1:]...).Start()
-						if err != nil {
-							l.Warnln("restart:", err)
+						if err = restartMonitor(args); err != nil {
+							l.Warnln("Restart:", err)
 						}
 						return
 					}
@@ -182,7 +181,8 @@ func copyStderr(stderr io.Reader, dst io.Writer) {
 				}
 
 				l.Warnf("Panic detected, writing to \"%s\"", panicFd.Name())
-				l.Warnln("Please create an issue at https://github.com/syncthing/syncthing/issues/ with the panic log attached")
+				l.Warnln("Please check for existing issues with similar panic message at https://github.com/syncthing/syncthing/issues/")
+				l.Warnln("If no issue with similar panic message exists, please create a new issue with the panic log attached")
 
 				stdoutMut.Lock()
 				for _, line := range stdoutFirstLines {
@@ -226,3 +226,39 @@ func copyStdout(stdout io.Reader, dst io.Writer) {
 		dst.Write([]byte(line))
 	}
 }
+
+func restartMonitor(args []string) error {
+	if runtime.GOOS != "windows" {
+		// syscall.Exec is the cleanest way to restart on Unixes as it
+		// replaces the current process with the new one, keeping the pid and
+		// controlling terminal and so on
+		return restartMonitorUnix(args)
+	}
+
+	// but it isn't supported on Windows, so there we start a normal
+	// exec.Command and return.
+	return restartMonitorWindows(args)
+}
+
+func restartMonitorUnix(args []string) error {
+	if !strings.ContainsRune(args[0], os.PathSeparator) {
+		// The path to the binary doesn't contain a slash, so it should be
+		// found in $PATH.
+		binary, err := exec.LookPath(args[0])
+		if err != nil {
+			return err
+		}
+		args[0] = binary
+	}
+
+	return syscall.Exec(args[0], args, os.Environ())
+}
+
+func restartMonitorWindows(args []string) error {
+	cmd := exec.Command(args[0], args[1:]...)
+	// Retain the standard streams
+	cmd.Stderr = os.Stderr
+	cmd.Stdout = os.Stdout
+	cmd.Stdin = os.Stdin
+	return cmd.Start()
+}

cmd/syncthing/random_test.go

@@ -7,6 +7,7 @@
 package main
 
 import (
+	"runtime"
 	"sync"
 	"testing"
 )
@@ -14,10 +15,13 @@ import (
 var predictableRandomTest sync.Once
 
 func TestPredictableRandom(t *testing.T) {
+	if runtime.GOARCH != "amd64" {
+		t.Skip("Test only for 64 bit platforms; but if it works there, it should work on 32 bit")
+	}
 	predictableRandomTest.Do(func() {
 		// predictable random sequence is predictable
-		e := 3440579354231278675
-		if v := predictableRandom.Int(); v != e {
+		e := int64(3440579354231278675)
+		if v := int64(predictableRandom.Int()); v != e {
 			t.Errorf("Unexpected random value %d != %d", v, e)
 		}
 	})

cmd/syncthing/summarysvc.go

@@ -9,9 +9,9 @@ package main
 import (
 	"time"
 
-	"github.com/syncthing/syncthing/internal/events"
-	"github.com/syncthing/syncthing/internal/model"
-	"github.com/syncthing/syncthing/internal/sync"
+	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/model"
+	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/thejerf/suture"
 )
 

cmd/syncthing/tls.go

@@ -102,7 +102,9 @@ func (l *DowngradingListener) Accept() (net.Conn, error) {
 	}
 
 	br := bufio.NewReader(conn)
+	conn.SetReadDeadline(time.Now().Add(1 * time.Second))
 	bs, err := br.Peek(1)
+	conn.SetReadDeadline(time.Time{})
 	if err != nil {
 		// We hit a read error here, but the Accept() call succeeded so we must not return an error.
 		// We return the connection as is and let whoever tries to use it deal with the error.

cmd/syncthing/upnpsvc.go

@@ -10,8 +10,8 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/syncthing/syncthing/internal/config"
-	"github.com/syncthing/syncthing/internal/upnp"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/upnp"
 )
 
 // The UPnP service runs a loop for discovery of IGDs (Internet Gateway
@@ -79,8 +79,10 @@ func (s *upnpSvc) tryIGDs(igds []upnp.IGD, prevExtPort int) int {
 			// External port changed; refresh the discovery announcement.
 			// TODO: Don't reach out to some magic global here?
 			l.Infof("New UPnP port mapping: external port %d to local port %d.", extPort, s.localPort)
-			discoverer.StopGlobal()
-			discoverer.StartGlobal(s.cfg.Options().GlobalAnnServers, uint16(extPort))
+			if s.cfg.Options().GlobalAnnEnabled {
+				discoverer.StopGlobal()
+				discoverer.StartGlobal(s.cfg.Options().GlobalAnnServers, uint16(extPort))
+			}
 		}
 		if debugNet {
 			l.Debugf("Created/updated UPnP port mapping for external port %d on device %s.", extPort, igd.FriendlyIdentifier())

cmd/syncthing/usage_report.go

@@ -11,12 +11,15 @@ import (
 	"crypto/rand"
 	"crypto/sha256"
 	"encoding/json"
+	"fmt"
 	"net"
 	"net/http"
 	"runtime"
 	"time"
 
-	"github.com/syncthing/syncthing/internal/model"
+	"github.com/syncthing/syncthing/lib/config"
+	"github.com/syncthing/syncthing/lib/model"
+	"github.com/thejerf/suture"
 )
 
 // Current version number of the usage report, for acceptance purposes. If
@@ -24,8 +27,54 @@ import (
 // are prompted for acceptance of the new report.
 const usageReportVersion = 1
 
-var stopUsageReportingCh = make(chan struct{})
+type usageReportingManager struct {
+	model *model.Model
+	sup   *suture.Supervisor
+}
 
+func newUsageReportingManager(m *model.Model, cfg *config.Wrapper) *usageReportingManager {
+	mgr := &usageReportingManager{
+		model: m,
+	}
+
+	// Start UR if it's enabled.
+	mgr.CommitConfiguration(config.Configuration{}, cfg.Raw())
+
+	// Listen to future config changes so that we can start and stop as
+	// appropriate.
+	cfg.Subscribe(mgr)
+
+	return mgr
+}
+
+func (m *usageReportingManager) VerifyConfiguration(from, to config.Configuration) error {
+	return nil
+}
+
+func (m *usageReportingManager) CommitConfiguration(from, to config.Configuration) bool {
+	if to.Options.URAccepted >= usageReportVersion && m.sup == nil {
+		// Usage reporting was turned on; lets start it.
+		svc := &usageReportingService{
+			model: m.model,
+		}
+		m.sup = suture.NewSimple("usageReporting")
+		m.sup.Add(svc)
+		m.sup.ServeBackground()
+	} else if to.Options.URAccepted < usageReportVersion && m.sup != nil {
+		// Usage reporting was turned off
+		m.sup.Stop()
+		m.sup = nil
+	}
+
+	return true
+}
+
+func (m *usageReportingManager) String() string {
+	return fmt.Sprintf("usageReportingManager@%p", m)
+}
+
+// reportData returns the data to be sent in a usage report. It's used in
+// various places, so not part of the usageReportingSvc object.
 func reportData(m *model.Model) map[string]interface{} {
 	res := make(map[string]interface{})
 	res["uniqueID"] = cfg.Options().URUniqueID
@@ -75,8 +124,13 @@ func reportData(m *model.Model) map[string]interface{} {
 	return res
 }
 
-func sendUsageReport(m *model.Model) error {
-	d := reportData(m)
+type usageReportingService struct {
+	model *model.Model
+	stop  chan struct{}
+}
+
+func (s *usageReportingService) sendUsageReport() error {
+	d := reportData(s.model)
 	var b bytes.Buffer
 	json.NewEncoder(&b).Encode(d)
 
@@ -94,32 +148,32 @@ func sendUsageReport(m *model.Model) error {
 	return err
 }
 
-func usageReportingLoop(m *model.Model) {
+func (s *usageReportingService) Serve() {
+	s.stop = make(chan struct{})
+
 	l.Infoln("Starting usage reporting")
-	t := time.NewTicker(86400 * time.Second)
-loop:
+	defer l.Infoln("Stopping usage reporting")
+
+	t := time.NewTimer(10 * time.Minute) // time to initial report at start
 	for {
 		select {
-		case <-stopUsageReportingCh:
-			break loop
+		case <-s.stop:
+			return
 		case <-t.C:
-			err := sendUsageReport(m)
+			err := s.sendUsageReport()
 			if err != nil {
 				l.Infoln("Usage report:", err)
 			}
+			t.Reset(24 * time.Hour) // next report tomorrow
 		}
 	}
-	l.Infoln("Stopping usage reporting")
 }
 
-func stopUsageReporting() {
-	select {
-	case stopUsageReportingCh <- struct{}{}:
-	default:
-	}
+func (s *usageReportingService) Stop() {
+	close(s.stop)
 }
 
-// Returns CPU performance as a measure of single threaded SHA-256 MiB/s
+// cpuBench returns CPU performance as a measure of single threaded SHA-256 MiB/s
 func cpuBench() float64 {
 	chunkSize := 100 * 1 << 10
 	h := sha256.New()