lib/scanner: Correctly scan symlinks (fixes #3445)

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3446

.gitignore

@@ -1,7 +1,7 @@
 /syncthing
-/discosrv
+/stdiscosrv
 syncthing.exe
-discosrv.exe
+stdiscosrv.exe
 *.tar.gz
 *.zip
 *.asc

README.md

@@ -27,6 +27,11 @@ There are a few examples for keeping Syncthing running in the background
 on your system in [the etc directory][3]. There are also several [GUI
 implementations][11] for Windows, Mac and Linux.
 
+## Vote on features/bugs
+
+We'd like to encourage you to [vote][12] on issues that matter to you.
+This helps the team understand what are the biggest pain points for our users, and could potentially influence what is being worked on next.
+
 ## Getting in Touch
 
 The first and best point of contact is the [Forum][8]. There is also an IRC
@@ -66,3 +71,4 @@ All code is licensed under the [MPLv2 License][7].
 [9]: https://kiwiirc.com/client/irc.freenode.net/#syncthing
 [10]: https://github.com/syncthing/syncthing/issues
 [11]: http://docs.syncthing.net/users/contrib.html#gui-wrappers
+[12]: https://www.bountysource.com/teams/syncthing/issues

build.go

@@ -96,38 +96,55 @@ var targets = map[string]target{
 			{src: "etc/linux-systemd/user/syncthing.service", dst: "deb/usr/lib/systemd/user/syncthing.service", perm: 0644},
 		},
 	},
-	"discosrv": {
-		name:       "discosrv",
-		buildPkg:   "./cmd/discosrv",
-		binaryName: "discosrv", // .exe will be added automatically for Windows builds
+	"stdiscosrv": {
+		name:       "stdiscosrv",
+		buildPkg:   "./cmd/stdiscosrv",
+		binaryName: "stdiscosrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
-			{src: "cmd/discosrv/README.md", dst: "README.txt", perm: 0644},
-			{src: "cmd/discosrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
+			{src: "cmd/stdiscosrv/README.md", dst: "README.txt", perm: 0644},
+			{src: "cmd/stdiscosrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
 		debianFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
-			{src: "cmd/discosrv/README.md", dst: "deb/usr/share/doc/discosrv/README.txt", perm: 0644},
-			{src: "cmd/discosrv/LICENSE", dst: "deb/usr/share/doc/discosrv/LICENSE.txt", perm: 0644},
-			{src: "AUTHORS", dst: "deb/usr/share/doc/discosrv/AUTHORS.txt", perm: 0644},
+			{src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/stdiscosrv/README.txt", perm: 0644},
+			{src: "cmd/stdiscosrv/LICENSE", dst: "deb/usr/share/doc/stdiscosrv/LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "deb/usr/share/doc/stdiscosrv/AUTHORS.txt", perm: 0644},
 		},
 		tags: []string{"purego"},
 	},
-	"relaysrv": {
-		name:       "relaysrv",
-		buildPkg:   "./cmd/relaysrv",
-		binaryName: "relaysrv", // .exe will be added automatically for Windows builds
+	"strelaysrv": {
+		name:       "strelaysrv",
+		buildPkg:   "./cmd/strelaysrv",
+		binaryName: "strelaysrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
-			{src: "cmd/relaysrv/README.md", dst: "README.txt", perm: 0644},
-			{src: "cmd/relaysrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
+			{src: "cmd/strelaysrv/README.md", dst: "README.txt", perm: 0644},
+			{src: "cmd/strelaysrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
 		debianFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
-			{src: "cmd/relaysrv/README.md", dst: "deb/usr/share/doc/relaysrv/README.txt", perm: 0644},
-			{src: "cmd/relaysrv/LICENSE", dst: "deb/usr/share/doc/relaysrv/LICENSE.txt", perm: 0644},
+			{src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/strelaysrv/README.txt", perm: 0644},
+			{src: "cmd/strelaysrv/LICENSE", dst: "deb/usr/share/doc/strelaysrv/LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "deb/usr/share/doc/strelaysrv/AUTHORS.txt", perm: 0644},
+		},
+	},
+	"strelaypoolsrv": {
+		name:       "strelaypoolsrv",
+		buildPkg:   "./cmd/strelaypoolsrv",
+		binaryName: "strelaypoolsrv", // .exe will be added automatically for Windows builds
+		archiveFiles: []archiveFile{
+			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
+			{src: "cmd/strelaypoolsrv/README.md", dst: "README.txt", perm: 0644},
+			{src: "cmd/strelaypoolsrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
+		},
+		debianFiles: []archiveFile{
+			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
+			{src: "cmd/strelaypoolsrv/README.md", dst: "deb/usr/share/doc/relaysrv/README.txt", perm: 0644},
+			{src: "cmd/strelaypoolsrv/LICENSE", dst: "deb/usr/share/doc/relaysrv/LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "deb/usr/share/doc/relaysrv/AUTHORS.txt", perm: 0644},
 		},
 	},
@@ -238,8 +255,8 @@ func runCommand(cmd string, target target) {
 	case "assets":
 		rebuildAssets()
 
-	case "xdr":
-		xdr()
+	case "proto":
+		proto()
 
 	case "translate":
 		translate()
@@ -271,9 +288,12 @@ func runCommand(cmd string, target target) {
 	case "metalint":
 		if isGometalinterInstalled() {
 			dirs := []string{".", "./cmd/...", "./lib/..."}
-			gometalinter("deadcode", dirs, "test/util.go")
-			gometalinter("structcheck", dirs)
-			gometalinter("varcheck", dirs)
+			ok := gometalinter("deadcode", dirs, "test/util.go")
+			ok = gometalinter("structcheck", dirs) && ok
+			ok = gometalinter("varcheck", dirs) && ok
+			if !ok {
+				os.Exit(1)
+			}
 		}
 
 	default:
@@ -543,16 +563,17 @@ func listFiles(dir string) []string {
 
 func rebuildAssets() {
 	runPipe("lib/auto/gui.files.go", "go", "run", "script/genassets.go", "gui")
+	runPipe("cmd/strelaypoolsrv/auto/gui.go", "go", "run", "script/genassets.go", "cmd/strelaypoolsrv/gui")
 }
 
 func lazyRebuildAssets() {
-	if shouldRebuildAssets() {
+	if shouldRebuildAssets("lib/auto/gui.files.go", "gui") || shouldRebuildAssets("cmd/strelaypoolsrv/auto/gui.go", "cmd/strelaypoolsrv/auto/gui") {
 		rebuildAssets()
 	}
 }
 
-func shouldRebuildAssets() bool {
-	info, err := os.Stat("lib/auto/gui.files.go")
+func shouldRebuildAssets(target, srcdir string) bool {
+	info, err := os.Stat(target)
 	if err != nil {
 		// If the file doesn't exist, we must rebuild it
 		return true
@@ -562,7 +583,7 @@ func shouldRebuildAssets() bool {
 	// so we should rebuild it.
 	currentBuild := info.ModTime()
 	assetsAreNewer := false
-	filepath.Walk("gui", func(path string, info os.FileInfo, err error) error {
+	filepath.Walk(srcdir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -576,8 +597,8 @@ func shouldRebuildAssets() bool {
 	return assetsAreNewer
 }
 
-func xdr() {
-	runPrint("go", "generate", "./lib/discover", "./lib/db", "./lib/protocol", "./lib/relay/protocol")
+func proto() {
+	runPrint("go", "generate", "./lib/...")
 }
 
 func translate() {
@@ -956,13 +977,17 @@ func lint(pkg string) {
 	}
 
 	analCommentPolicy := regexp.MustCompile(`exported (function|method|const|type|var) [^\s]+ should have comment`)
-	for _, line := range bytes.Split(bs, []byte("\n")) {
-		if analCommentPolicy.Match(line) {
+	for _, line := range strings.Split(string(bs), "\n") {
+		if line == "" {
 			continue
 		}
-		if len(line) > 0 {
-			log.Printf("%s", line)
+		if analCommentPolicy.MatchString(line) {
+			continue
 		}
+		if strings.Contains(line, ".pb.go:") {
+			continue
+		}
+		log.Println(line)
 	}
 }
 
@@ -1003,7 +1028,7 @@ func isGometalinterInstalled() bool {
 	return true
 }
 
-func gometalinter(linter string, dirs []string, excludes ...string) {
+func gometalinter(linter string, dirs []string, excludes ...string) bool {
 	params := []string{"--disable-all"}
 	params = append(params, fmt.Sprintf("--deadline=%ds", 60))
 	params = append(params, "--enable="+linter)
@@ -1016,12 +1041,19 @@ func gometalinter(linter string, dirs []string, excludes ...string) {
 		params = append(params, dir)
 	}
 
-	bs, err := runError("gometalinter", params...)
+	bs, _ := runError("gometalinter", params...)
 
-	if len(bs) > 0 {
-		log.Printf("%s", bs)
-	}
-	if err != nil {
-		log.Printf("%v", err)
+	nerr := 0
+	for _, line := range strings.Split(string(bs), "\n") {
+		if line == "" {
+			continue
+		}
+		if strings.Contains(line, ".pb.go:") {
+			continue
+		}
+		log.Println(line)
+		nerr++
 	}
+
+	return nerr == 0
 }

build.sh

@@ -131,55 +131,6 @@ case "${1:-default}" in
 		go2xunit -output tests.xml -fail < tests.out
 		;;
 
-	docker-all)
-		img=${DOCKERIMG:-syncthing/build:latest}
-		docker run --rm -h syncthing-builder -u $(id -u) -t \
-			-v $(pwd):/go/src/github.com/syncthing/syncthing \
-			-w /go/src/github.com/syncthing/syncthing \
-			-e "STTRACE=$STTRACE" \
-			"$img" \
-			sh -c './build.sh clean \
-				&& ./build.sh test-cov \
-				&& ./build.sh bench \
-				&& ./build.sh all'
-		;;
-
-	docker-test)
-		img=${DOCKERIMG:-syncthing/build:latest}
-		docker run --rm -h syncthing-builder -u $(id -u) -t \
-			-v $(pwd):/go/src/github.com/syncthing/syncthing \
-			-w /go/src/github.com/syncthing/syncthing \
-			-e "STTRACE=$STTRACE" \
-			"$img" \
-			sh -euxc './build.sh clean \
-				&& go run build.go -race \
-				&& export GOPATH=$(pwd)/Godeps/_workspace:$GOPATH \
-				&& cd test \
-				&& go test -tags integration -v -timeout 90m -short \
-				&& git clean -fxd .'
-		;;
-
-	docker-lint)
-		img=${DOCKERIMG:-syncthing/build:latest}
-		docker run --rm -h syncthing-builder -u $(id -u) -t \
-			-v $(pwd):/go/src/github.com/syncthing/syncthing \
-			-w /go/src/github.com/syncthing/syncthing \
-			-e "STTRACE=$STTRACE" \
-			"$img" \
-			sh -euxc 'go run build.go lint'
-		;;
-
-
-	docker-vet)
-		img=${DOCKERIMG:-syncthing/build:latest}
-		docker run --rm -h syncthing-builder -u $(id -u) -t \
-			-v $(pwd):/go/src/github.com/syncthing/syncthing \
-			-w /go/src/github.com/syncthing/syncthing \
-			-e "STTRACE=$STTRACE" \
-			"$img" \
-			sh -euxc 'go run build.go vet'
-		;;
-
 	*)
 		echo "Unknown build command $1"
 		;;

cmd/stdisco/main.go

@@ -9,6 +9,7 @@ package main
 import (
 	"bytes"
 	"crypto/rand"
+	"encoding/binary"
 	"flag"
 	"log"
 	"strings"
@@ -66,24 +67,25 @@ func recv(bc beacon.Interface) {
 	seen := make(map[string]bool)
 	for {
 		data, src := bc.Recv()
-		var ann discover.Announce
-		ann.UnmarshalXDR(data)
+		if m := binary.BigEndian.Uint32(data); m != discover.Magic {
+			log.Printf("Incorrect magic %x in announcement from %v", m, src)
+			continue
+		}
 
-		if bytes.Equal(ann.This.ID, myID) {
+		var ann discover.Announce
+		ann.Unmarshal(data[4:])
+
+		if bytes.Equal(ann.ID, myID) {
 			// This is one of our own fake packets, don't print it.
 			continue
 		}
 
 		// Print announcement details for the first packet from a given
 		// device ID and source address, or if -all was given.
-		key := string(ann.This.ID) + src.String()
+		key := string(ann.ID) + src.String()
 		if all || !seen[key] {
 			log.Printf("Announcement from %v\n", src)
-			log.Printf(" %v at %s\n", protocol.DeviceIDFromBytes(ann.This.ID), strings.Join(addrStrs(ann.This), ", "))
-
-			for _, dev := range ann.Extra {
-				log.Printf(" %v at %s\n", protocol.DeviceIDFromBytes(dev.ID), strings.Join(addrStrs(dev), ", "))
-			}
+			log.Printf(" %v at %s\n", protocol.DeviceIDFromBytes(ann.ID), strings.Join(ann.Addresses, ", "))
 			seen[key] = true
 		}
 	}
@@ -92,15 +94,10 @@ func recv(bc beacon.Interface) {
 // sends fake discovery announcements once every second
 func send(bc beacon.Interface) {
 	ann := discover.Announce{
-		Magic: discover.AnnouncementMagic,
-		This: discover.Device{
-			ID: myID,
-			Addresses: []discover.Address{
-				{URL: "tcp://fake.example.com:12345"},
-			},
-		},
+		ID:        myID,
+		Addresses: []string{"tcp://fake.example.com:12345"},
 	}
-	bs, _ := ann.MarshalXDR()
+	bs, _ := ann.Marshal()
 
 	for {
 		bc.Send(bs)
@@ -108,15 +105,6 @@ func send(bc beacon.Interface) {
 	}
 }
 
-// returns the list of address URLs
-func addrStrs(dev discover.Device) []string {
-	ss := make([]string, len(dev.Addresses))
-	for i, addr := range dev.Addresses {
-		ss[i] = addr.URL
-	}
-	return ss
-}
-
 // returns a random but recognizable device ID
 func randomDeviceID() []byte {
 	var id [32]byte

cmd/stdiscosrv/README.md

@@ -1,12 +1,12 @@
-discosrv
-========
+stdiscosrv
+==========
 
-[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/discosrv.svg?style=flat-square)](http://build.syncthing.net/job/discosrv/lastBuild/)
+[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/stdiscosrv.svg?style=flat-square)](http://build.syncthing.net/job/stdiscosrv/lastBuild/)
 
 This is the global discovery server for the `syncthing` project.
 
-To get it, run `go get github.com/syncthing/discosrv` or download the
-[latest build](http://build.syncthing.net/job/discosrv/lastSuccessfulBuild/artifact/)
+To get it, run `go get github.com/syncthing/stdiscosrv` or download the
+[latest build](http://build.syncthing.net/job/stdiscosrv/lastSuccessfulBuild/artifact/)
 from the build server.
 
 Usage
@@ -19,15 +19,15 @@ By default it will use in-memory `ql` backend. If you wish to persist the
 information on disk between restarts in `ql`, specify a file DSN:
 
 ```bash
-$ discosrv -db-dsn="file:///var/run/discosrv.db"
+$ stdiscosrv -db-dsn="file:///var/run/stdiscosrv.db"
 ```
 
 For `postgres`, you will need to create a database and a user with permissions
-to create tables in it, then start the discosrv as follows:
+to create tables in it, then start the stdiscosrv as follows:
 
 ```bash
-$ export DISCOSRV_DB_DSN="postgres://user:password@localhost/databasename"
-$ discosrv -db-backend="postgres"
+$ export STDISCOSRV_DB_DSN="postgres://user:password@localhost/databasename"
+$ stdiscosrv -db-backend="postgres"
 ```
 
 You can pass the DSN as command line option, but the value what you pass in will
@@ -37,4 +37,4 @@ to other users.
 In all cases, the appropriate tables and indexes will be created at first
 startup. If it doesn't exit with an error, you're fine.
 
-See `discosrv -help` for other options.
+See `stdiscosrv -help` for other options.

cmd/stdiscosrv/main.go

@@ -38,7 +38,7 @@ func init() {
 	BuildDate = time.Unix(int64(stamp), 0)
 
 	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`discosrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
+	LongVersion = fmt.Sprintf(`stdiscosrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
 }
 
 var (
@@ -48,7 +48,7 @@ var (
 	globalStats stats
 	statsFile   string
 	backend     = "ql"
-	dsn         = getEnvDefault("DISCOSRV_DB_DSN", "memory://discosrv")
+	dsn         = getEnvDefault("STDISCOSRV_DB_DSN", "memory://stdiscosrv")
 	certFile    = "cert.pem"
 	keyFile     = "key.pem"
 	debug       = false

cmd/stdiscosrv/psql.go

@@ -28,7 +28,7 @@ func postgresSetup(db *sql.DB) error {
 	}
 
 	row := db.QueryRow(`SELECT 'DevicesDeviceIDIndex'::regclass`)
-	if err := row.Scan(nil); err != nil {
+	if err = row.Scan(nil); err != nil {
 		_, err = db.Exec(`CREATE INDEX DevicesDeviceIDIndex ON Devices (DeviceID)`)
 	}
 	if err != nil {
@@ -36,7 +36,7 @@ func postgresSetup(db *sql.DB) error {
 	}
 
 	row = db.QueryRow(`SELECT 'DevicesSeenIndex'::regclass`)
-	if err := row.Scan(nil); err != nil {
+	if err = row.Scan(nil); err != nil {
 		_, err = db.Exec(`CREATE INDEX DevicesSeenIndex ON Devices (Seen)`)
 	}
 	if err != nil {
@@ -53,7 +53,7 @@ func postgresSetup(db *sql.DB) error {
 	}
 
 	row = db.QueryRow(`SELECT 'AddressesDeviceIDSeenIndex'::regclass`)
-	if err := row.Scan(nil); err != nil {
+	if err = row.Scan(nil); err != nil {
 		_, err = db.Exec(`CREATE INDEX AddressesDeviceIDSeenIndex ON Addresses (DeviceID, Seen)`)
 	}
 	if err != nil {
@@ -61,7 +61,7 @@ func postgresSetup(db *sql.DB) error {
 	}
 
 	row = db.QueryRow(`SELECT 'AddressesDeviceIDAddressIndex'::regclass`)
-	if err := row.Scan(nil); err != nil {
+	if err = row.Scan(nil); err != nil {
 		_, err = db.Exec(`CREATE INDEX AddressesDeviceIDAddressIndex ON Addresses (DeviceID, Address)`)
 	}
 	if err != nil {

cmd/stindex/dump.go

@@ -13,50 +13,61 @@ import (
 
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syndtr/goleveldb/leveldb"
 )
 
-func dump(ldb *leveldb.DB) {
+func dump(ldb *db.Instance) {
 	it := ldb.NewIterator(nil, nil)
-	var dev protocol.DeviceID
 	for it.Next() {
 		key := it.Key()
 		switch key[0] {
 		case db.KeyTypeDevice:
-			folder := nulString(key[1 : 1+64])
-			devBytes := key[1+64 : 1+64+32]
-			name := nulString(key[1+64+32:])
-			copy(dev[:], devBytes)
-			fmt.Printf("[device] F:%q N:%q D:%v\n", folder, name, dev)
+			folder := binary.BigEndian.Uint32(key[1:])
+			device := binary.BigEndian.Uint32(key[1+4:])
+			name := nulString(key[1+4+4:])
+			fmt.Printf("[device] F:%d D:%d N:%q", folder, device, name)
 
 			var f protocol.FileInfo
-			err := f.UnmarshalXDR(it.Value())
+			err := f.Unmarshal(it.Value())
 			if err != nil {
 				log.Fatal(err)
 			}
-			fmt.Printf("  N:%q\n  F:%#o\n  M:%d\n  V:%v\n  S:%d\n  B:%d\n", f.Name, f.Flags, f.Modified, f.Version, f.Size(), len(f.Blocks))
+			fmt.Printf(" V:%v\n", f)
 
 		case db.KeyTypeGlobal:
-			folder := nulString(key[1 : 1+64])
-			name := nulString(key[1+64:])
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
 			var flv db.VersionList
-			flv.UnmarshalXDR(it.Value())
-			fmt.Printf("[global] F:%q N:%q V: %s\n", folder, name, flv)
+			flv.Unmarshal(it.Value())
+			fmt.Printf("[global] F:%d N:%q V:%s\n", folder, name, flv)
 
 		case db.KeyTypeBlock:
-			folder := nulString(key[1 : 1+64])
-			hash := key[1+64 : 1+64+32]
-			name := nulString(key[1+64+32:])
-			fmt.Printf("[block] F:%q H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
+			folder := binary.BigEndian.Uint32(key[1:])
+			hash := key[1+4 : 1+4+32]
+			name := nulString(key[1+4+32:])
+			fmt.Printf("[block] F:%d H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
 
 		case db.KeyTypeDeviceStatistic:
-			fmt.Printf("[dstat]\n  %x\n  %x\n", it.Key(), it.Value())
+			fmt.Printf("[dstat] K:%x V:%x\n", it.Key(), it.Value())
 
 		case db.KeyTypeFolderStatistic:
-			fmt.Printf("[fstat]\n  %x\n  %x\n", it.Key(), it.Value())
+			fmt.Printf("[fstat] K:%x V:%x\n", it.Key(), it.Value())
 
 		case db.KeyTypeVirtualMtime:
-			fmt.Printf("[mtime]\n  %x\n  %x\n", it.Key(), it.Value())
+			fmt.Printf("[mtime] K:%x V:%x\n", it.Key(), it.Value())
+
+		case db.KeyTypeFolderIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			fmt.Printf("[folderidx] K:%d V:%q\n", key, it.Value())
+
+		case db.KeyTypeDeviceIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			val := it.Value()
+			if len(val) == 0 {
+				fmt.Printf("[deviceidx] K:%d V:<nil>\n", key)
+			} else {
+				dev := protocol.DeviceIDFromBytes(val)
+				fmt.Printf("[deviceidx] K:%d V:%s\n", key, dev)
+			}
 
 		default:
 			fmt.Printf("[???]\n  %x\n  %x\n", it.Key(), it.Value())

cmd/stindex/dumpsize.go

@@ -8,11 +8,10 @@ package main
 
 import (
 	"container/heap"
+	"encoding/binary"
 	"fmt"
 
 	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syndtr/goleveldb/leveldb"
 )
 
 type SizedElement struct {
@@ -38,33 +37,31 @@ func (h *ElementHeap) Pop() interface{} {
 	return x
 }
 
-func dumpsize(ldb *leveldb.DB) {
+func dumpsize(ldb *db.Instance) {
 	h := &ElementHeap{}
 	heap.Init(h)
 
 	it := ldb.NewIterator(nil, nil)
-	var dev protocol.DeviceID
 	var ele SizedElement
 	for it.Next() {
 		key := it.Key()
 		switch key[0] {
 		case db.KeyTypeDevice:
-			folder := nulString(key[1 : 1+64])
-			devBytes := key[1+64 : 1+64+32]
-			name := nulString(key[1+64+32:])
-			copy(dev[:], devBytes)
-			ele.key = fmt.Sprintf("DEVICE:%s:%s:%s", dev, folder, name)
+			folder := binary.BigEndian.Uint32(key[1:])
+			device := binary.BigEndian.Uint32(key[1+4:])
+			name := nulString(key[1+4+4:])
+			ele.key = fmt.Sprintf("DEVICE:%d:%d:%s", folder, device, name)
 
 		case db.KeyTypeGlobal:
-			folder := nulString(key[1 : 1+64])
-			name := nulString(key[1+64:])
-			ele.key = fmt.Sprintf("GLOBAL:%s:%s", folder, name)
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			ele.key = fmt.Sprintf("GLOBAL:%d:%s", folder, name)
 
 		case db.KeyTypeBlock:
-			folder := nulString(key[1 : 1+64])
-			hash := key[1+64 : 1+64+32]
-			name := nulString(key[1+64+32:])
-			ele.key = fmt.Sprintf("BLOCK:%s:%x:%s", folder, hash, name)
+			folder := binary.BigEndian.Uint32(key[1:])
+			hash := key[1+4 : 1+4+32]
+			name := nulString(key[1+4+32:])
+			ele.key = fmt.Sprintf("BLOCK:%d:%x:%s", folder, hash, name)
 
 		case db.KeyTypeDeviceStatistic:
 			ele.key = fmt.Sprintf("DEVICESTATS:%s", key[1:])
@@ -75,6 +72,14 @@ func dumpsize(ldb *leveldb.DB) {
 		case db.KeyTypeVirtualMtime:
 			ele.key = fmt.Sprintf("MTIME:%s", key[1:])
 
+		case db.KeyTypeFolderIdx:
+			id := binary.BigEndian.Uint32(key[1:])
+			ele.key = fmt.Sprintf("FOLDERIDX:%d", id)
+
+		case db.KeyTypeDeviceIdx:
+			id := binary.BigEndian.Uint32(key[1:])
+			ele.key = fmt.Sprintf("DEVICEIDX:%d", id)
+
 		default:
 			ele.key = fmt.Sprintf("UNKNOWN:%x", key)
 		}

cmd/stindex/main.go

@@ -13,8 +13,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syncthing/syncthing/lib/db"
 )
 
 func main() {
@@ -28,16 +27,12 @@ func main() {
 
 	path := flag.Arg(0)
 	if path == "" {
-		path = filepath.Join(defaultConfigDir(), "index-v0.11.0.db")
+		path = filepath.Join(defaultConfigDir(), "index-v0.14.0.db")
 	}
 
 	fmt.Println("Path:", path)
 
-	ldb, err := leveldb.OpenFile(path, &opt.Options{
-		ErrorIfMissing:         true,
-		Strict:                 opt.StrictAll,
-		OpenFilesCacheCapacity: 100,
-	})
+	ldb, err := db.Open(path)
 	if err != nil {
 		log.Fatal(err)
 	}

cmd/strelaypoolsrv/README.md

@@ -0,0 +1,15 @@
+# relaypoolsrv
+
+[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/relaypoolsrv.svg?style=flat-square)](http://build.syncthing.net/job/relaypoolsrv/lastBuild/)
+
+This is the relay pool server for the `syncthing` project, which allows community hosted [relaysrv](https://github.com/syncthing/relaysrv)'s to join the public pool.
+
+Servers that join the pool are then advertised to users of `syncthing` as potential connection points for those who are unable to connect directly due to NAT or firewall issues.
+
+There is very little reason why you'd want to run this yourself, as `relaypoolsrv` is just used for announcement and lookup of public relay servers. If you are looking to setup a private or a public relay, please check the documentation for [relaysrv](https://github.com/syncthing/relaysrv), which also explains how to join the default public pool.
+
+If you still want to run it, you can run `go get github.com/syncthing/relaypoolsrv` download it or download the
+[latest build](http://build.syncthing.net/job/relaypoolsrv/lastSuccessfulBuild/artifact/)
+from the build server.
+
+See `relaypoolsrv -help` for configuration options.

cmd/strelaypoolsrv/auto/.gitignore

@@ -0,0 +1 @@
+gui.go

cmd/strelaypoolsrv/gui/index.html

@@ -0,0 +1,395 @@
+<!DOCTYPE html>
+
+<html lang="en" ng-app="syncthing" ng-controller="relayDataController">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta name="description" content="">
+    <meta name="author" content="">
+
+    <title>Relay stats</title>
+    <link href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet">
+    <link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/font-awesome/4.6.1/css/font-awesome.min.css">
+
+    <style>
+      #map {
+        height: 600px;
+      }
+      .ng-cloak {
+        display: none;
+      }
+      table {
+        font-size: 11px !important;
+        width: 100%;
+        border: 1px;
+
+      }
+      td {
+        padding: 0px !important;
+      }
+      tfoot td {
+        font-weight: bold;
+      }
+    </style>
+  </head>
+
+  <body class="ng-cloak">
+    <div class="container">
+      <h1>Relay Pool Data</h2>
+      <div ng-if="relays === undefined" class="text-center">
+        <img src="//cdnjs.cloudflare.com/ajax/libs/galleriffic/2.0.1/css/loader.gif"/>
+        <p>Please wait while we gather data</p>
+      </div>
+      <div>
+        <div ng-show="relays !== undefined" class="ng-hide">
+          <p>
+            Currently {{ relays.length }} relays online ({{ totals.goMaxProcs }} cores in total).
+          </p>
+        </div>
+        <div id="map"></div> <!-- Can't hide the map, otherwise it freaks out -->
+        <p>The circle size represents how much bytes the relay transfered relative to other relays</p>
+      </div>
+      <div>
+        <table class="table table-striped table-condensed table">
+          <thead>
+            <tr>
+              <th rowspan="2">Address</td>
+              <th rowspan="2">
+                <a ng-click="sortType = 'status.numActiveSessions || -1'; sortReverse = !sortReverse">
+                  Sessions
+                  <span ng-show="sortType == 'status.numActiveSessions || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.numActiveSessions || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+              <th rowspan="2">
+                <a ng-click="sortType = 'status.numConnections || -1'; sortReverse = !sortReverse">
+                  Connections
+                  <span ng-show="sortType == 'status.numConnections || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.numConnections || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+              <th rowspan="2">
+                <a ng-click="sortType = 'status.bytesProxied || -1'; sortReverse = !sortReverse">
+                  Data relayed
+                  <span ng-show="sortType == 'status.bytesProxied || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.bytesProxied || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+              <th colspan="6" class="text-center">Transfer rate in the last period</th>
+              <th rowspan="2">
+                <a ng-click="sortType = 'status.uptimeSeconds || -1'; sortReverse = !sortReverse">
+                  Uptime hours
+                  <span ng-show="sortType == 'status.uptimeSeconds || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.uptimeSeconds || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+              <th rowspan="2">
+                <a ng-click="sortType = 'status.options[\'provided-by\'] || \'\''; sortReverse = !sortReverse">
+                  Provided by
+                  <span ng-show="sortType == 'status.options[\'provided-by\'] || \'\'' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.options[\'provided-by\'] || \'\'' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+            </tr>
+            <tr>
+              <th>
+                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[0] || -1'; sortReverse = !sortReverse">
+                  10s
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[0] || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[0] || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+              <th>
+                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[1] || -1'; sortReverse = !sortReverse">
+                  1m
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[1] || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[1] || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+              <th>
+                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[2] || -1'; sortReverse = !sortReverse">
+                  5m
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[2] || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[2] || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+              <th>
+                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[3] || -1'; sortReverse = !sortReverse">
+                  15m
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[3] || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[3] || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+              <th>
+                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[4] || -1'; sortReverse = !sortReverse">
+                  30m
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[4] || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[4] || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+              <th>
+                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[5] || -1'; sortReverse = !sortReverse">
+                  60m
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[5] || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[5] || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+            </tr>
+          </thead>
+          <tbody>
+            <tr ng-repeat="relay in relays | orderBy:sortType:sortReverse ">
+              <td>{{ relay.address }}</td>
+              <td ng-if="relay.status === undefined" colspan="11" class="text-center">Looking up...</td>
+              <td ng-if-start="relay.status !== undefined">{{ relay.status.numActiveSessions }}</td>
+              <td>{{ relay.status.numConnections }}</td>
+              <td>{{ relay.status.bytesProxied | bytes }}</td>
+              <td>{{ relay.status.kbps10s1m5m15m30m60m[0] * 128 | bytes }}/s</td>
+              <td>{{ relay.status.kbps10s1m5m15m30m60m[1] * 128 | bytes }}/s</td>
+              <td>{{ relay.status.kbps10s1m5m15m30m60m[2] * 128 | bytes }}/s</td>
+              <td>{{ relay.status.kbps10s1m5m15m30m60m[3] * 128 | bytes }}/s</td>
+              <td>{{ relay.status.kbps10s1m5m15m30m60m[4] * 128 | bytes }}/s</td>
+              <td>{{ relay.status.kbps10s1m5m15m30m60m[5] * 128 | bytes }}/s</td>
+              <td ng-if="relay.status.uptimeSeconds != undefined">{{ relay.status.uptimeSeconds/60/60 | number:0 }}</td>
+              <td ng-if="relay.status.uptimeSeconds == undefined"></td>
+              <td title="{{ relay.status.options['provided-by'] || '' }}" ng-if-end>
+                {{ relay.status.options['provided-by'] || '' | limitTo:50 }}
+                <span ng-if="(relay.status.options['provided-by'] || '').length > 50">&hellip;
+              </td>
+            </tr>
+          </tbody>
+          <tfoot>
+            <tr>
+              <td>Totals</td>
+              <td>{{ totals.numActiveSessions }}</td>
+              <td>{{ totals.numConnections }}</td>
+              <td>{{ totals.bytesProxied | bytes }}</td>
+              <td>{{ totals.kbps10s1m5m15m30m60m[0] * 128 | bytes }}/s</td>
+              <td>{{ totals.kbps10s1m5m15m30m60m[1] * 128 | bytes }}/s</td>
+              <td>{{ totals.kbps10s1m5m15m30m60m[2] * 128 | bytes }}/s</td>
+              <td>{{ totals.kbps10s1m5m15m30m60m[3] * 128 | bytes }}/s</td>
+              <td>{{ totals.kbps10s1m5m15m30m60m[4] * 128 | bytes }}/s</td>
+              <td>{{ totals.kbps10s1m5m15m30m60m[5] * 128 | bytes }}/s</td>
+              <td>{{ totals.uptimeSeconds/60/60 | number:0 }} hours</td>
+              <td>{{ relays.length }} relays</td>
+            </tr>
+          </tfoor>
+        </table>
+      </div>
+      <hr>
+      <p>
+        This product includes GeoLite2 data created by MaxMind, available from
+        <a href="http://www.maxmind.com">http://www.maxmind.com</a>.
+      </p>
+    </div>
+
+
+    <script src="//code.jquery.com/jquery-2.1.4.min.js"></script>
+    <script src="//cdnjs.cloudflare.com/ajax/libs/angular.js/1.4.7/angular.min.js"></script>
+    <script src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
+    <script src="//maps.googleapis.com/maps/api/js"></script>
+  </body>
+
+  <script>
+  angular.module('syncthing', [
+  ])
+  .config(function($httpProvider) {
+    $httpProvider.defaults.timeout = 5000;
+  })
+  .filter('bytes', function() {
+    return function(bytes, precision) {
+      if (isNaN(parseFloat(bytes)) || !isFinite(bytes)) return '-';
+      if (typeof precision === 'undefined') precision = 1;
+
+      var units = ['bytes', 'kB', 'MB', 'GB', 'TB', 'PB'],
+          number = Math.floor(Math.log(bytes) / Math.log(1024));
+
+      var value = (bytes / Math.pow(1000, Math.floor(number)));
+      if (!isFinite(value)) {
+        value = 0;
+        precision = 0;
+      }
+      if (!isFinite(number)) {
+        units = 'bytes';
+      } else {
+        units = units[number];
+      }
+      return value.toFixed(precision) + ' ' + units;
+    }
+  })
+  .controller('relayDataController', ['$scope', '$rootScope', '$http', '$q', '$compile', '$timeout', function($scope, $rootScope, $http, $q, $compile, $timeout) {
+    $scope.totals = {
+      bytesProxied: 0,
+      goMaxProcs: 0,
+      kbps10s1m5m15m30m60m: [0, 0, 0, 0, 0, 0],
+      numActiveSessions: 0,
+      numConnections: 0,
+      numPendingSessionKeys: 0,
+      numProxies: 0,
+      uptimeSeconds: 0,
+    };
+    $scope.map = new google.maps.Map(document.getElementById('map'), {
+      zoom: 1,
+      mapTypeId: google.maps.MapTypeId.ROADMAP
+    });
+    $scope.mapBounds = new google.maps.LatLngBounds();
+    $scope.tooltipTemplate = $('#infoTemplate').html();
+    $scope.usedLocations = {};
+    $scope.sortType = 'status.numActiveSessions || -1';
+    $scope.sortReverse = true;
+
+    $http.get("/endpoint").then(function(response) {
+      $scope.relays = response.data.relays;
+      var promises = [];
+      angular.forEach($scope.relays, function(relay) {
+
+        relay.uri = constructURI(relay.url);
+        relay.address = relay.url.split('/')[2];
+
+        addMarkerToMap(relay);
+
+        promises.push(getRelayStatus(relay));
+      });
+
+      // Can only add circles once we know the totals for transfers, which means
+      // we need to resolve all statuses.
+      $q.all(promises).then(function() {
+        angular.forEach($scope.relays, function(relay) {
+          if (relay.status) {
+            addCircleToMap(relay);
+          }
+        });
+      });
+
+      $scope.map.fitBounds($scope.mapBounds);
+      if ($scope.relays.length == 1) {
+        $scope.map.setZoom(13);
+      }
+    });
+
+    function addMarkerToMap(relay) {
+        var loc = relay.location.latitude + "," + relay.location.longitude;
+
+        // Deal with overlapping markers
+        while (loc in $scope.usedLocations) {
+          var locParts = loc.split(',');
+          locParts = [parseFloat(locParts[0]), parseFloat(locParts[1])];
+          locParts[Math.round(Math.random())] += 0.5 * (Math.random() >= 0.5 ? 1 : -1);
+          loc = locParts.join(',');
+        }
+
+        $scope.usedLocations[loc] = true;
+
+        var locParts = loc.split(',');
+
+        relay.marker = new google.maps.Marker({
+          map: $scope.map,
+          position: new google.maps.LatLng(locParts[0], locParts[1]),
+          title: relay.url,
+        });
+
+        var scope = $rootScope.$new(true);
+        scope.relay = relay;
+
+        relay.marker.info = new google.maps.InfoWindow({
+          content: $compile($scope.tooltipTemplate)(scope)[0],
+        });
+
+        relay.marker.addListener('mouseover', function() {
+          relay.marker.info.open($scope.map, relay.marker);
+        });
+
+        relay.marker.addListener('mouseout', function() {
+          relay.marker.info.close();
+        });
+
+        $scope.mapBounds.extend(relay.marker.position);
+    }
+
+    function addCircleToMap(relay) {
+      relay.marker.circle = new google.maps.Circle({
+        strokeColor: '#FF0000',
+        strokeOpacity: 0.8,
+        strokeWeight: 2,
+        fillColor: '#FF0000',
+        fillOpacity: 0.35,
+        map: $scope.map,
+        center: relay.marker.position,
+        radius: ((relay.status.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000
+      });
+    }
+
+    function getRelayStatus(relay) {
+      // Normal timeout doesn't deal with relays which accept the TCP connection
+      // but don't respond (some firewalls do that), so deal with it this way.
+      var timeoutRequest = $q.defer();
+      var resolveStatus = $q.defer();
+
+      $http.get("http://" + relay.uri.hostname + (relay.uri.args.statusAddr || ":22070") + "/status", { timeout: timeoutRequest.promise }).then(function (response) {
+        relay.status = response.data;
+        resolveStatus.resolve();
+        angular.forEach($scope.totals, function(value, key) {
+          if (typeof $scope.totals[key] == 'number') {
+            $scope.totals[key] += response.data[key];
+          } else if (typeof $scope.totals[key] == 'object' && $scope.totals[key] instanceof Array) {
+            angular.forEach($scope.totals[key], function(value, index) {
+              $scope.totals[key][index] += response.data[key][index];
+            });
+          }
+        });
+      }, function() {
+        relay.status = null;
+        resolveStatus.resolve();
+      });
+
+      $timeout(function() {
+        timeoutRequest.resolve();
+      }, 5000);
+
+      return resolveStatus.promise;
+    }
+
+    function constructURI(url) {
+      var uri = document.createElement('a');
+
+      // HAX, otherwise doesn't work
+      uri.href = url.replace('relay://', 'http://');
+
+      // Convert query string to object
+      uri.args = {};
+      angular.forEach(uri.search.replace(/^\?/, '').split('&'), function(query) {
+          var split = query.split('=');
+          uri.args[split[0]] = split[1];
+      });
+
+      return uri;
+    }
+  }]);
+  </script>
+
+  <script type="text/template" id="infoTemplate">
+    <div>
+      <p><b>{{ relay.uri.hostname }}</b> <span ng-if="relay.status.options['provided-by']">provided by <u>{{ relay.status.options['provided-by'] }}</u></span></p>
+      <div ng-if="relay.status">
+        <span ng-if="relay.status.startTime">Start time: {{ relay.status.startTime | date:"medium" }}</br></span>
+        <span ng-if="relay.status.bytesProxied != undefined">Proxied: {{ relay.status.bytesProxied | bytes }}</br></span>
+        <span ng-if="relay.status.numActiveSessions != undefined">Sessions: {{ relay.status.numActiveSessions }}</br></span>
+        <span ng-if="relay.status.numConnections != undefined">Clients: {{ relay.status.numConnections }}</br></span>
+        <span ng-if="relay.status.options.pools">Pools: {{ relay.status.options.pools.join(', ') }}</br></span>
+        <span ng-if="relay.status.options['global-rate'] != undefined">
+          <span ng-if="relay.status.options['global-rate'] > 0">Global rate limit: {{ relay.status.options['global-rate'] | bytes }}/s</span>
+          <span ng-if="relay.status.options['global-rate'] == 0">Global rate limit: unlimited</span>
+          </br>
+        </span>
+        <span ng-if="relay.status.options['per-session-rate'] != undefined">
+          <span ng-if="relay.status.options['per-session-rate'] > 0">Session rate limit: {{ relay.status.options['per-session-rate'] | bytes }}/s</span>
+          <span ng-if="relay.status.options['per-session-rate'] == 0">Session rate limit: unlimited</span>
+          </br>
+        </span>
+      </div>
+      <div ng-if="!relay.status">
+        Data unavailable.
+      <div>
+    </div>
+  </script>
+</html>

cmd/strelaypoolsrv/main.go

@@ -0,0 +1,536 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+//go:generate go run genassets.go gui auto/gui.go
+
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"crypto/tls"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"mime"
+	"net"
+	"net/http"
+	"net/url"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/golang/groupcache/lru"
+	"github.com/juju/ratelimit"
+
+	"github.com/oschwald/geoip2-golang"
+
+	"github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto"
+	"github.com/syncthing/syncthing/lib/relay/client"
+	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syncthing/syncthing/lib/tlsutil"
+)
+
+type location struct {
+	Latitude  float64 `json:"latitude"`
+	Longitude float64 `json:"longitude"`
+}
+
+type relay struct {
+	URL      string   `json:"url"`
+	Location location `json:"location"`
+	uri      *url.URL
+}
+
+func (r relay) String() string {
+	return r.URL
+}
+
+type request struct {
+	relay  relay
+	uri    *url.URL
+	result chan result
+}
+
+type result struct {
+	err      error
+	eviction time.Duration
+}
+
+var (
+	testCert       tls.Certificate
+	listen         = ":80"
+	dir            string
+	evictionTime   = time.Hour
+	debug          bool
+	getLRUSize           = 10 << 10
+	getLimitBurst  int64 = 10
+	getLimitAvg          = 1
+	postLRUSize          = 1 << 10
+	postLimitBurst int64 = 2
+	postLimitAvg         = 1
+	getLimit       time.Duration
+	postLimit      time.Duration
+	permRelaysFile string
+	ipHeader       string
+	geoipPath      string
+
+	getMut      = sync.NewRWMutex()
+	getLRUCache *lru.Cache
+
+	postMut      = sync.NewRWMutex()
+	postLRUCache *lru.Cache
+
+	requests = make(chan request, 10)
+
+	mut             = sync.NewRWMutex()
+	knownRelays     = make([]relay, 0)
+	permanentRelays = make([]relay, 0)
+	evictionTimers  = make(map[string]*time.Timer)
+)
+
+func main() {
+	flag.StringVar(&listen, "listen", listen, "Listen address")
+	flag.StringVar(&dir, "keys", dir, "Directory where http-cert.pem and http-key.pem is stored for TLS listening")
+	flag.BoolVar(&debug, "debug", debug, "Enable debug output")
+	flag.DurationVar(&evictionTime, "eviction", evictionTime, "After how long the relay is evicted")
+	flag.IntVar(&getLRUSize, "get-limit-cache", getLRUSize, "Get request limiter cache size")
+	flag.IntVar(&getLimitAvg, "get-limit-avg", 2, "Allowed average get request rate, per 10 s")
+	flag.Int64Var(&getLimitBurst, "get-limit-burst", getLimitBurst, "Allowed burst get requests")
+	flag.IntVar(&postLRUSize, "post-limit-cache", postLRUSize, "Post request limiter cache size")
+	flag.IntVar(&postLimitAvg, "post-limit-avg", 2, "Allowed average post request rate, per minute")
+	flag.Int64Var(&postLimitBurst, "post-limit-burst", postLimitBurst, "Allowed burst post requests")
+	flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays")
+	flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.")
+	flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
+
+	flag.Parse()
+
+	getLimit = 10 * time.Second / time.Duration(getLimitAvg)
+	postLimit = time.Minute / time.Duration(postLimitAvg)
+
+	getLRUCache = lru.New(getLRUSize)
+	postLRUCache = lru.New(postLRUSize)
+
+	var listener net.Listener
+	var err error
+
+	if permRelaysFile != "" {
+		loadPermanentRelays(permRelaysFile)
+	}
+
+	testCert = createTestCertificate()
+
+	go requestProcessor()
+
+	if dir != "" {
+		if debug {
+			log.Println("Starting TLS listener on", listen)
+		}
+		certFile, keyFile := filepath.Join(dir, "http-cert.pem"), filepath.Join(dir, "http-key.pem")
+		var cert tls.Certificate
+		cert, err = tls.LoadX509KeyPair(certFile, keyFile)
+		if err != nil {
+			log.Fatalln("Failed to load HTTP X509 key pair:", err)
+		}
+
+		tlsCfg := &tls.Config{
+			Certificates: []tls.Certificate{cert},
+			MinVersion:   tls.VersionTLS10, // No SSLv3
+			CipherSuites: []uint16{
+				// No RC4
+				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+				tls.TLS_RSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_RSA_WITH_AES_256_CBC_SHA,
+				tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,
+				tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA,
+			},
+		}
+
+		listener, err = tls.Listen("tcp", listen, tlsCfg)
+	} else {
+		if debug {
+			log.Println("Starting plain listener on", listen)
+		}
+		listener, err = net.Listen("tcp", listen)
+	}
+
+	if err != nil {
+		log.Fatalln("listen:", err)
+	}
+
+	handler := http.NewServeMux()
+	handler.HandleFunc("/", handleAssets)
+	handler.HandleFunc("/endpoint", handleRequest)
+
+	srv := http.Server{
+		Handler:     handler,
+		ReadTimeout: 10 * time.Second,
+	}
+
+	err = srv.Serve(listener)
+	if err != nil {
+		log.Fatalln("serve:", err)
+	}
+}
+
+func handleAssets(w http.ResponseWriter, r *http.Request) {
+	assets := auto.Assets()
+	path := r.URL.Path[1:]
+	if path == "" {
+		path = "index.html"
+	}
+
+	bs, ok := assets[path]
+	if !ok {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+
+	mtype := mimeTypeForFile(path)
+	if len(mtype) != 0 {
+		w.Header().Set("Content-Type", mtype)
+	}
+
+	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
+		w.Header().Set("Content-Encoding", "gzip")
+	} else {
+		// ungzip if browser not send gzip accepted header
+		var gr *gzip.Reader
+		gr, _ = gzip.NewReader(bytes.NewReader(bs))
+		bs, _ = ioutil.ReadAll(gr)
+		gr.Close()
+	}
+	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(bs)))
+
+	w.Write(bs)
+}
+
+func mimeTypeForFile(file string) string {
+	// We use a built in table of the common types since the system
+	// TypeByExtension might be unreliable. But if we don't know, we delegate
+	// to the system.
+	ext := filepath.Ext(file)
+	switch ext {
+	case ".htm", ".html":
+		return "text/html"
+	case ".css":
+		return "text/css"
+	case ".js":
+		return "application/javascript"
+	case ".json":
+		return "application/json"
+	case ".png":
+		return "image/png"
+	case ".ttf":
+		return "application/x-font-ttf"
+	case ".woff":
+		return "application/x-font-woff"
+	case ".svg":
+		return "image/svg+xml"
+	default:
+		return mime.TypeByExtension(ext)
+	}
+}
+
+func handleRequest(w http.ResponseWriter, r *http.Request) {
+	if ipHeader != "" {
+		r.RemoteAddr = r.Header.Get(ipHeader)
+	}
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	switch r.Method {
+	case "GET":
+		if limit(r.RemoteAddr, getLRUCache, getMut, getLimit, int64(getLimitBurst)) {
+			w.WriteHeader(429)
+			return
+		}
+		handleGetRequest(w, r)
+	case "POST":
+		if limit(r.RemoteAddr, postLRUCache, postMut, postLimit, int64(postLimitBurst)) {
+			w.WriteHeader(429)
+			return
+		}
+		handlePostRequest(w, r)
+	default:
+		if debug {
+			log.Println("Unhandled HTTP method", r.Method)
+		}
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func handleGetRequest(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	mut.RLock()
+	relays := append(permanentRelays, knownRelays...)
+	mut.RUnlock()
+
+	// Shuffle
+	for i := range relays {
+		j := rand.Intn(i + 1)
+		relays[i], relays[j] = relays[j], relays[i]
+	}
+
+	json.NewEncoder(w).Encode(map[string][]relay{
+		"relays": relays,
+	})
+}
+
+func handlePostRequest(w http.ResponseWriter, r *http.Request) {
+	var newRelay relay
+	err := json.NewDecoder(r.Body).Decode(&newRelay)
+	r.Body.Close()
+
+	if err != nil {
+		if debug {
+			log.Println("Failed to parse payload")
+		}
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	uri, err := url.Parse(newRelay.URL)
+	if err != nil {
+		if debug {
+			log.Println("Failed to parse URI", newRelay.URL)
+		}
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	host, port, err := net.SplitHostPort(uri.Host)
+	if err != nil {
+		if debug {
+			log.Println("Failed to split URI", newRelay.URL)
+		}
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	// Get the IP address of the client
+	rhost, _, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		if debug {
+			log.Println("Failed to split remote address", r.RemoteAddr)
+		}
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	// The client did not provide an IP address, use the IP address of the client.
+	if host == "" {
+		uri.Host = net.JoinHostPort(rhost, port)
+		newRelay.URL = uri.String()
+	} else if host != rhost {
+		if debug {
+			log.Println("IP address advertised does not match client IP address", r.RemoteAddr, uri)
+		}
+		http.Error(w, "IP address does not match client IP", http.StatusUnauthorized)
+		return
+	}
+	newRelay.uri = uri
+	newRelay.Location = getLocation(uri.Host)
+
+	for _, current := range permanentRelays {
+		if current.uri.Host == newRelay.uri.Host {
+			if debug {
+				log.Println("Asked to add a relay", newRelay, "which exists in permanent list")
+			}
+			http.Error(w, "Invalid request", 500)
+			return
+		}
+	}
+
+	reschan := make(chan result)
+
+	select {
+	case requests <- request{newRelay, uri, reschan}:
+		result := <-reschan
+		if result.err != nil {
+			http.Error(w, result.err.Error(), 500)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		json.NewEncoder(w).Encode(map[string]time.Duration{
+			"evictionIn": result.eviction,
+		})
+
+	default:
+		if debug {
+			log.Println("Dropping request")
+		}
+		w.WriteHeader(429)
+	}
+}
+
+func requestProcessor() {
+	for request := range requests {
+		if debug {
+			log.Println("Request for", request.relay)
+		}
+		if !client.TestRelay(request.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3) {
+			if debug {
+				log.Println("Test for relay", request.relay, "failed")
+			}
+			request.result <- result{fmt.Errorf("test failed"), 0}
+			continue
+		}
+
+		mut.Lock()
+		timer, ok := evictionTimers[request.relay.uri.Host]
+		if ok {
+			if debug {
+				log.Println("Stopping existing timer for", request.relay)
+			}
+			timer.Stop()
+		}
+
+		for i, current := range knownRelays {
+			if current.uri.Host == request.relay.uri.Host {
+				if debug {
+					log.Println("Relay", request.relay, "already exists")
+				}
+
+				// Evict the old entry anyway, as configuration might have changed.
+				last := len(knownRelays) - 1
+				knownRelays[i] = knownRelays[last]
+				knownRelays = knownRelays[:last]
+
+				goto found
+			}
+		}
+
+		if debug {
+			log.Println("Adding new relay", request.relay)
+		}
+
+	found:
+
+		knownRelays = append(knownRelays, request.relay)
+
+		evictionTimers[request.relay.uri.Host] = time.AfterFunc(evictionTime, evict(request.relay))
+		mut.Unlock()
+		request.result <- result{nil, evictionTime}
+	}
+
+}
+
+func evict(relay relay) func() {
+	return func() {
+		mut.Lock()
+		defer mut.Unlock()
+		if debug {
+			log.Println("Evicting", relay)
+		}
+		for i, current := range knownRelays {
+			if current.uri.Host == relay.uri.Host {
+				if debug {
+					log.Println("Evicted", relay)
+				}
+				last := len(knownRelays) - 1
+				knownRelays[i] = knownRelays[last]
+				knownRelays = knownRelays[:last]
+			}
+		}
+		delete(evictionTimers, relay.uri.Host)
+	}
+}
+
+func limit(addr string, cache *lru.Cache, lock sync.RWMutex, rate time.Duration, burst int64) bool {
+	host, _, err := net.SplitHostPort(addr)
+	if err != nil {
+		return false
+	}
+
+	lock.RLock()
+	bkt, ok := cache.Get(host)
+	lock.RUnlock()
+	if ok {
+		bkt := bkt.(*ratelimit.Bucket)
+		if bkt.TakeAvailable(1) != 1 {
+			// Rate limit
+			return true
+		}
+	} else {
+		lock.Lock()
+		cache.Add(host, ratelimit.NewBucket(rate, burst))
+		lock.Unlock()
+	}
+	return false
+}
+
+func loadPermanentRelays(file string) {
+	content, err := ioutil.ReadFile(file)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, line := range strings.Split(string(content), "\n") {
+		if len(line) == 0 {
+			continue
+		}
+
+		uri, err := url.Parse(line)
+		if err != nil {
+			if debug {
+				log.Println("Skipping permanent relay", line, "due to parse error", err)
+			}
+			continue
+
+		}
+
+		permanentRelays = append(permanentRelays, relay{
+			URL:      line,
+			Location: getLocation(uri.Host),
+			uri:      uri,
+		})
+		if debug {
+			log.Println("Adding permanent relay", line)
+		}
+	}
+}
+
+func createTestCertificate() tls.Certificate {
+	tmpDir, err := ioutil.TempDir("", "relaypoolsrv")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	certFile, keyFile := filepath.Join(tmpDir, "cert.pem"), filepath.Join(tmpDir, "key.pem")
+	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 3072)
+	if err != nil {
+		log.Fatalln("Failed to create test X509 key pair:", err)
+	}
+
+	return cert
+}
+
+func getLocation(host string) location {
+	db, err := geoip2.Open(geoipPath)
+	if err != nil {
+		return location{}
+	}
+	defer db.Close()
+
+	addr, err := net.ResolveTCPAddr("tcp", host)
+	if err != nil {
+		return location{}
+	}
+
+	city, err := db.City(addr.IP)
+	if err != nil {
+		return location{}
+	}
+
+	return location{
+		Latitude:  city.Location.Latitude,
+		Longitude: city.Location.Longitude,
+	}
+}

cmd/strelaysrv/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 The Syncthing Project
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

cmd/strelaysrv/README.md

@@ -1,12 +1,12 @@
-relaysrv
-========
+strelaysrv
+==========
 
-[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/relaysrv.svg?style=flat-square)](http://build.syncthing.net/job/relaysrv/lastBuild/)
+[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/strelaysrv.svg?style=flat-square)](http://build.syncthing.net/job/strelaysrv/lastBuild/)
 
 This is the relay server for the `syncthing` project.
 
-To get it, run `go get github.com/syncthing/relaysrv` or download the
-[latest build](http://build.syncthing.net/job/relaysrv/lastSuccessfulBuild/artifact/)
+To get it, run `go get github.com/syncthing/strelaysrv` or download the
+[latest build](http://build.syncthing.net/job/strelaysrv/lastSuccessfulBuild/artifact/)
 from the build server.
 
 :exclamation:Warnings:exclamation: - Read or regret
@@ -16,13 +16,13 @@ By default, all relay servers will join the default public relay pool, which mea
 
 If you wish to disable this behaviour, please specify `-pools=""` argument.
 
-Please note that `relaysrv` is only usable by `syncthing` **version v0.12 and onwards**.
+Please note that `strelaysrv` is only usable by `syncthing` **version v0.12 and onwards**.
 
-To run `relaysrv` you need to have port 22067 available to the internet, which means you might need to allow it through your firewall if you **have a public IP, or setup a port-forwarding** (22067 to 22067) if you are behind a router.
+To run `strelaysrv` you need to have port 22067 available to the internet, which means you might need to allow it through your firewall if you **have a public IP, or setup a port-forwarding** (22067 to 22067) if you are behind a router.
 
-Furthermore, **by default relaysrv will also expose a /status HTTP endpoint on port 22070**, which is used by the pool servers to peek at metrics of the relaysrv, such as what are the current transfer rates, how many clients are connected, etc, etc. If you wish this information to be available, similarlly you might want to allow it through your firewall, or port-forward it (22070 to 22070) on your NAT device.
+Furthermore, **by default strelaysrv will also expose a /status HTTP endpoint on port 22070**, which is used by the pool servers to peek at metrics of the strelaysrv, such as what are the current transfer rates, how many clients are connected, etc, etc. If you wish this information to be available, similarlly you might want to allow it through your firewall, or port-forward it (22070 to 22070) on your NAT device.
 
-This is **not mandatory** for the relaysrv to function, and is used only to gather metrics and present them in the overview page of the pool server, displaying stats about the specific relay.
+This is **not mandatory** for the strelaysrv to function, and is used only to gather metrics and present them in the overview page of the pool server, displaying stats about the specific relay.
 
 At the point of writing the endpoint output looks as follows:
 
@@ -62,31 +62,31 @@ At the point of writing the endpoint output looks as follows:
 }
 ```
 
-If you wish to disable the /status endpoint, provide `-status-srv=""` as one of the arguments when starting the relaysrv.
+If you wish to disable the /status endpoint, provide `-status-srv=""` as one of the arguments when starting the strelaysrv.
 
 Running for public use
 ----
 Make sure you have a public IP with port 22067 open, or make sure you have port-forwarding (22067 to 22067) if you are behind a router.
 
-Run the `relaysrv` with no arguments (or `-debug` if you want more output), and that should be enough for the server to join the public relay pool.
+Run the `strelaysrv` with no arguments (or `-debug` if you want more output), and that should be enough for the server to join the public relay pool.
 You should see a message saying:
 ```
 2015/09/21 22:45:46 pool.go:60: Joined https://relays.syncthing.net/endpoint rejoining in 48m0s
 ```
 
-See `relaysrv -help` for other options, such as rate limits, timeout intervals, etc.
+See `strelaysrv -help` for other options, such as rate limits, timeout intervals, etc.
 
 Running for private use
 -----
 
-Once you've started the `relaysrv`, it will generate a key pair and print an URI:
+Once you've started the `strelaysrv`, it will generate a key pair and print an URI:
 ```bash
 relay://:22067/?id=EZQOIDM-6DDD4ZI-DJ65NSM-4OQWRAT-EIKSMJO-OZ552BO-WQZEGYY-STS5RQM&pingInterval=1m0s&networkTimeout=2m0s&sessionLimitBps=0&globalLimitBps=0&statusAddr=:22070
 ```
 
 This URI contains partial address of the relay server, as well as it's options which in the future may be taken into account when choosing the best suitable relay out of multiple available.
 
-Because `-listen` option was not used, the `relaysrv` does not know it's external IP, therefore you should replace the host part of the URI with your public IP address on which the `relaysrv` will be available:
+Because `-listen` option was not used, the `strelaysrv` does not know it's external IP, therefore you should replace the host part of the URI with your public IP address on which the `strelaysrv` will be available:
 
 ```bash
 relay://123.123.123.123:22067/?id=EZQOIDM-6DDD4ZI-DJ65NSM-4OQWRAT-EIKSMJO-OZ552BO-WQZEGYY-STS5RQM&pingInterval=1m0s&networkTimeout=2m0s&sessionLimitBps=0&globalLimitBps=0&statusAddr=:22070
@@ -100,7 +100,7 @@ relay://123.123.123.123:22067
 
 This URI can then be used in `syncthing` as one of the relay servers.
 
-See `relaysrv -help` for other options, such as rate limits, timeout intervals, etc.
+See `strelaysrv -help` for other options, such as rate limits, timeout intervals, etc.
 
 Other items available in this repo
 ----

cmd/strelaysrv/etc/linux-systemd/strelaysrv.service

@@ -3,10 +3,10 @@ Description=Syncthing relay server
 After=network.target
 
 [Service]
-User=syncthing-relaysrv
-Group=syncthing-relaysrv
-ExecStart=/usr/bin/relaysrv
-WorkingDirectory=/var/lib/syncthing-relaysrv
+User=strelaysrv
+Group=strelaysrv
+ExecStart=/usr/bin/strelaysrv
+WorkingDirectory=/var/lib/strelaysrv
 
 PrivateTmp=true
 ProtectSystem=full

cmd/strelaysrv/main.go

@@ -42,7 +42,7 @@ func init() {
 	BuildDate = time.Unix(int64(stamp), 0)
 
 	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`relaysrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
+	LongVersion = fmt.Sprintf(`strelaysrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
 }
 
 var (
@@ -121,7 +121,7 @@ func main() {
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "relaysrv", 3072)
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv", 3072)
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}

cmd/syncthing/gui.go

@@ -41,8 +41,7 @@ import (
 )
 
 var (
-	configInSync = true
-	startTime    = time.Now()
+	startTime = time.Now()
 )
 
 type apiService struct {
@@ -100,12 +99,13 @@ type configIntf interface {
 	GUI() config.GUIConfiguration
 	Raw() config.Configuration
 	Options() config.OptionsConfiguration
-	Replace(cfg config.Configuration) config.CommitResponse
+	Replace(cfg config.Configuration) error
 	Subscribe(c config.Committer)
 	Folders() map[string]config.FolderConfiguration
 	Devices() map[protocol.DeviceID]config.DeviceConfiguration
 	Save() error
 	ListenAddresses() []string
+	RequiresRestart() bool
 }
 
 type connectionsIntf interface {
@@ -722,13 +722,19 @@ func (s *apiService) postSystemConfig(w http.ResponseWriter, r *http.Request) {
 
 	// Activate and save
 
-	resp := s.cfg.Replace(to)
-	configInSync = !resp.RequiresRestart
-	s.cfg.Save()
+	if err := s.cfg.Replace(to); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if err := s.cfg.Save(); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
 }
 
 func (s *apiService) getSystemConfigInsync(w http.ResponseWriter, r *http.Request) {
-	sendJSON(w, map[string]bool{"configInSync": configInSync})
+	sendJSON(w, map[string]bool{"configInSync": !s.cfg.RequiresRestart()})
 }
 
 func (s *apiService) postSystemRestart(w http.ResponseWriter, r *http.Request) {
@@ -1173,13 +1179,17 @@ type jsonFileInfo protocol.FileInfo
 
 func (f jsonFileInfo) MarshalJSON() ([]byte, error) {
 	return json.Marshal(map[string]interface{}{
-		"name":         f.Name,
-		"size":         protocol.FileInfo(f).Size(),
-		"flags":        fmt.Sprintf("%#o", f.Flags),
-		"modified":     time.Unix(f.Modified, 0),
-		"localVersion": f.LocalVersion,
-		"numBlocks":    len(f.Blocks),
-		"version":      jsonVersionVector(f.Version),
+		"name":          f.Name,
+		"type":          f.Type,
+		"size":          f.Size,
+		"permissions":   fmt.Sprintf("%#o", f.Permissions),
+		"deleted":       f.Deleted,
+		"invalid":       f.Invalid,
+		"noPermissions": f.NoPermissions,
+		"modified":      time.Unix(f.Modified, 0),
+		"localVersion":  f.LocalVersion,
+		"numBlocks":     len(f.Blocks),
+		"version":       jsonVersionVector(f.Version),
 	})
 }
 
@@ -1187,20 +1197,23 @@ type jsonDBFileInfo db.FileInfoTruncated
 
 func (f jsonDBFileInfo) MarshalJSON() ([]byte, error) {
 	return json.Marshal(map[string]interface{}{
-		"name":         f.Name,
-		"size":         db.FileInfoTruncated(f).Size(),
-		"flags":        fmt.Sprintf("%#o", f.Flags),
-		"modified":     time.Unix(f.Modified, 0),
-		"localVersion": f.LocalVersion,
-		"version":      jsonVersionVector(f.Version),
+		"name":          f.Name,
+		"type":          f.Type,
+		"size":          f.Size,
+		"permissions":   fmt.Sprintf("%#o", f.Permissions),
+		"deleted":       f.Deleted,
+		"invalid":       f.Invalid,
+		"noPermissions": f.NoPermissions,
+		"modified":      time.Unix(f.Modified, 0),
+		"localVersion":  f.LocalVersion,
 	})
 }
 
 type jsonVersionVector protocol.Vector
 
 func (v jsonVersionVector) MarshalJSON() ([]byte, error) {
-	res := make([]string, len(v))
-	for i, c := range v {
+	res := make([]string, len(v.Counters))
+	for i, c := range v.Counters {
 		res[i] = fmt.Sprintf("%v:%d", c.ID, c.Value)
 	}
 	return json.Marshal(res)

cmd/syncthing/locations.go

@@ -48,7 +48,7 @@ var locations = map[locationEnum]string{
 	locKeyFile:       "${config}/key.pem",
 	locHTTPSCertFile: "${config}/https-cert.pem",
 	locHTTPSKeyFile:  "${config}/https-key.pem",
-	locDatabase:      "${config}/index-v0.13.0.db",
+	locDatabase:      "${config}/index-v0.14.0.db",
 	locLogFile:       "${config}/syncthing.log", // -logfile on Windows
 	locCsrfTokens:    "${config}/csrftokens.txt",
 	locPanicLog:      "${config}/panic-${timestamp}.log",

cmd/syncthing/main.go

@@ -51,7 +51,7 @@ import (
 
 var (
 	Version           = "unknown-dev"
-	Codename          = "Copper Cockroach"
+	Codename          = "Dysprosium Dragonfly"
 	BuildStamp        = "0"
 	BuildDate         time.Time
 	BuildHost         = "unknown"
@@ -539,8 +539,9 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 	errors := logger.NewRecorder(l, logger.LevelWarn, maxSystemErrors, 0)
 	systemLog := logger.NewRecorder(l, logger.LevelDebug, maxSystemLog, initialSystemLog)
 
-	// Event subscription for the API; must start early to catch the early events.  The LocalChangeDetected
-	// event might overwhelm the event reciever in some situations so we will not subscribe to it here.
+	// Event subscription for the API; must start early to catch the early
+	// events. The LocalChangeDetected event might overwhelm the event
+	// receiver in some situations so we will not subscribe to it here.
 	apiSub := events.NewBufferedSubscription(events.Default.Subscribe(events.AllEvents&^events.LocalChangeDetected), 1000)
 
 	if len(os.Getenv("GOMAXPROCS")) == 0 {
@@ -681,17 +682,9 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 		}
 	}
 
-	// Clear out old indexes for other devices. Otherwise we'll start up and
-	// start needing a bunch of files which are nowhere to be found. This
-	// needs to be changed when we correctly do persistent indexes.
+	// Add and start folders
 	for _, folderCfg := range cfg.Folders() {
 		m.AddFolder(folderCfg)
-		for _, device := range folderCfg.DeviceIDs() {
-			if device == myID {
-				continue
-			}
-			m.Index(device, folderCfg.ID, nil, 0, nil)
-		}
 		m.StartFolder(folderCfg.ID)
 	}
 

cmd/syncthing/mocked_config_test.go

@@ -31,8 +31,8 @@ func (c *mockedConfig) Options() config.OptionsConfiguration {
 	return config.OptionsConfiguration{}
 }
 
-func (c *mockedConfig) Replace(cfg config.Configuration) config.CommitResponse {
-	return config.CommitResponse{}
+func (c *mockedConfig) Replace(cfg config.Configuration) error {
+	return nil
 }
 
 func (c *mockedConfig) Subscribe(cm config.Committer) {}
@@ -48,3 +48,7 @@ func (c *mockedConfig) Devices() map[protocol.DeviceID]config.DeviceConfiguratio
 func (c *mockedConfig) Save() error {
 	return nil
 }
+
+func (c *mockedConfig) RequiresRestart() bool {
+	return false
+}

cmd/syncthing/summaryservice.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/syncthing/syncthing/lib/events"
+	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/thejerf/suture"
 )
@@ -59,7 +60,7 @@ func (c *folderSummaryService) Stop() {
 // listenForUpdates subscribes to the event bus and makes note of folders that
 // need their data recalculated.
 func (c *folderSummaryService) listenForUpdates() {
-	sub := events.Default.Subscribe(events.LocalIndexUpdated | events.RemoteIndexUpdated | events.StateChanged | events.RemoteDownloadProgress)
+	sub := events.Default.Subscribe(events.LocalIndexUpdated | events.RemoteIndexUpdated | events.StateChanged | events.RemoteDownloadProgress | events.DeviceConnected)
 	defer events.Default.Unsubscribe(sub)
 
 	for {
@@ -67,8 +68,31 @@ func (c *folderSummaryService) listenForUpdates() {
 
 		select {
 		case ev := <-sub.C():
-			// Whenever the local or remote index is updated for a given
-			// folder we make a note of it.
+			if ev.Type == events.DeviceConnected {
+				// When a device connects we schedule a refresh of all
+				// folders shared with that device.
+
+				data := ev.Data.(map[string]string)
+				deviceID, _ := protocol.DeviceIDFromString(data["id"])
+
+				c.foldersMut.Lock()
+			nextFolder:
+				for _, folder := range c.cfg.Folders() {
+					for _, dev := range folder.Devices {
+						if dev.DeviceID == deviceID {
+							c.folders[folder.ID] = struct{}{}
+							continue nextFolder
+						}
+					}
+				}
+				c.foldersMut.Unlock()
+
+				continue
+			}
+
+			// The other events all have a "folder" attribute that they
+			// affect. Whenever the local or remote index is updated for a
+			// given folder we make a note of it.
 
 			data := ev.Data.(map[string]interface{})
 			folder := data["folder"].(string)

gui/default/assets/lang/lang-el.json

@@ -8,13 +8,13 @@
     "Add": "Προσθήκη",
     "Add Device": "Προσθήκη συσκευής",
     "Add Folder": "Προσθήκη φακέλου",
-    "Add Remote Device": "Add Remote Device",
+    "Add Remote Device": "Προσθήκη Απομακρυσμένης Συσκευής",
     "Add new folder?": "Προσθήκη νέου φακέλου;",
     "Address": "Διεύθυνση",
     "Addresses": "Διευθύνσεις",
     "Advanced": "Προχωρημένες",
     "Advanced Configuration": "Προχωρημένες ρυθμίσεις",
-    "Advanced settings": "Advanced settings",
+    "Advanced settings": "Προχωρημένες ρυθμίσεις",
     "All Data": "Όλα τα δεδομένα",
     "Allow Anonymous Usage Reporting?": "Να επιτρέπεται η αποστολή ανώνυμων στοιχείων χρήσης;",
     "Alphabetic": "Αλφαβητικά",
@@ -32,15 +32,15 @@
     "Comment, when used at the start of a line": "Σχόλιο, όταν χρησιμοποιείται στην αρχή μιας γραμμής",
     "Compression": "Συμπίεση",
     "Connection Error": "Σφάλμα σύνδεσης",
-    "Connection Type": "Connection Type",
+    "Connection Type": "Τύπος Σύνδεσης",
     "Copied from elsewhere": "Έχει αντιγραφεί από κάπου αλλού",
     "Copied from original": "Έχει αντιγραφεί από το πρωτότυπο",
-    "Copyright © 2014-2016 the following Contributors:": "Copyright © 2014-2016 the following Contributors:",
+    "Copyright © 2014-2016 the following Contributors:": "Copyright © 2014-2016 οι παρακάτω Συνεισφέροντες:",
     "Copyright © 2015 the following Contributors:": "Copyright © 2015 από τους παρακάτω συνεισφορείς:",
     "Danger!": "Προσοχή!",
     "Delete": "Διαγραφή",
     "Deleted": "Διαγραμμένα",
-    "Device \"{%name%}\" ({%device%} at {%address%}) wants to connect. Add new device?": "Device \"{{name}}\" ({{device}} at {{address}}) wants to connect. Add new device?",
+    "Device \"{%name%}\" ({%device%} at {%address%}) wants to connect. Add new device?": "Η συσκευή \"{{name}}\" ({{device}} στη {{address}}) επιθυμεί να συνδεθεί. Προσθήκη της νέας συσκευής?",
     "Device ID": "Ταυτότητα συσκευής",
     "Device Identification": "Ταυτότητα συσκευής",
     "Device Name": "Όνομα συσκευής",
@@ -98,7 +98,7 @@
     "Keep Versions": "Διατήρηση εκδόσεων",
     "Largest First": "Το μεγαλύτερο πρώτα",
     "Last File Received": "Πιο πρόσφατο αρχείο",
-    "Last Scan": "Last Scan",
+    "Last Scan": "Τελευταία Σάρωση",
     "Last seen": "Τελευταία φορά συνδεδεμένος",
     "Later": "Αργότερα",
     "Listeners": "Listeners",
@@ -193,7 +193,7 @@
     "Syncthing seems to be down, or there is a problem with your Internet connection. Retrying…": "Το Syncthing φαίνεται πως είναι απενεργοποιημένο ή υπάρχει πρόβλημα στη σύνδεσή σου στο διαδίκτυο. Προσπαθώ πάλι…",
     "Syncthing seems to be experiencing a problem processing your request. Please refresh the page or restart Syncthing if the problem persists.": "Το Syncthing φαίνεται να αντιμετωπίζει ένα πρόβλημα με την επεξεργασία του αιτήματός σου. Παρακαλούμε, αν το πρόβλημα συνεχίζει, ανανέωσε την σελίδα ή επανεκκίνησε το Syncthing.",
     "The Syncthing admin interface is configured to allow remote access without a password.": "Η διεπαφή διαχείρισης του Syncthing είναι ρυθμισμένη να επιτρέπει την πρόσβαση χωρίς κωδικό.",
-    "The aggregated statistics are publicly available at the URL below.": "The aggregated statistics are publicly available at the URL below.",
+    "The aggregated statistics are publicly available at the URL below.": "Τα στατιστικά που έχουν συλλεγεί είναι δημόσια διαθέσιμα στη παρακάτω διεύθυνση.",
     "The aggregated statistics are publicly available at {%url%}.": "Τα στατιστικά που έχουν συλλεγεί είναι δημόσια διαθέσιμα στο {{url}}.",
     "The configuration has been saved but not activated. Syncthing must restart to activate the new configuration.": "Οι ρυθμίσεις έχουν αποθηκευτεί αλλά δεν έχουν ενεργοποιηθεί. Πρέπει να επανεκκινήσεις το Syncthing για να ισχύσουν οι νέες ρυθμίσεις.",
     "The device ID cannot be blank.": "Η ταυτότητα της συσκευής δεν μπορεί να είναι κενή",

gui/default/assets/lang/lang-fr-CA.json

@@ -1,24 +1,24 @@
 {
-    "A device with that ID is already added.": "A device with that ID is already added.",
+    "A device with that ID is already added.": "La machine portant cette ID est déjà présente.",
     "A negative number of days doesn't make sense.": "Un nombre négatif de jours n'a pas de sens.",
     "A new major version may not be compatible with previous versions.": "Une nouvelle version majeure peut présenter des incompatibilités avec les versions antérieures.",
     "API Key": "Clé API",
     "About": "À propos",
     "Actions": "Actions",
     "Add": "Ajouter",
-    "Add Device": "Ajouter un périphérique",
-    "Add Folder": "Ajouter un répertoire",
-    "Add Remote Device": "Add Remote Device",
-    "Add new folder?": "Ajouter un nouveau dossier ?",
+    "Add Device": "Ajouter une machine",
+    "Add Folder": "Ajouter un partage",
+    "Add Remote Device": "Ajouter une machine",
+    "Add new folder?": "Ajouter un nouveau partage ?",
     "Address": "Adresse",
     "Addresses": "Adresses",
     "Advanced": "Avancé",
     "Advanced Configuration": "Configuration avancée",
-    "Advanced settings": "Advanced settings",
+    "Advanced settings": "Réglages experts",
     "All Data": "Toutes les données",
     "Allow Anonymous Usage Reporting?": "Autoriser le rapport anonyme de statistiques d'utilisation ?",
     "Alphabetic": "Alphabétique",
-    "An external command handles the versioning. It has to remove the file from the synced folder.": "Une commande externe gère les versions de fichiers. Elle supprime les fichiers dans le dossier synchronisé.",
+    "An external command handles the versioning. It has to remove the file from the synced folder.": "Une commande externe gère les versions de fichiers. Elle supprime les fichiers dans le répertoire synchronisé.",
     "Anonymous Usage Reporting": "Rapport anonyme de statistiques d'utilisation",
     "Any devices configured on an introducer device will be added to this device as well.": "Toute machine ajoutée depuis une machine introductrice sera aussi ajoutée sur cette machine.",
     "Automatic upgrades": "Mises à jour automatiques",
@@ -32,32 +32,32 @@
     "Comment, when used at the start of a line": "Commentaire lorsque utilisé en début de ligne",
     "Compression": "Compression",
     "Connection Error": "Erreur de connexion",
-    "Connection Type": "Connection Type",
+    "Connection Type": "Type de connexion",
     "Copied from elsewhere": "Copié d'ailleurs",
     "Copied from original": "Copié depuis l'original",
-    "Copyright © 2014-2016 the following Contributors:": "Copyright © 2014-2016 the following Contributors:",
+    "Copyright © 2014-2016 the following Contributors:": "Les contributeurs suivants, Copyright © 2014-2016 :",
     "Copyright © 2015 the following Contributors:": "Copyright © 2015 Les contributeurs suivants:",
-    "Danger!": "Danger!",
+    "Danger!": "Attention !",
     "Delete": "Supprimer",
     "Deleted": "Supprimé",
-    "Device \"{%name%}\" ({%device%} at {%address%}) wants to connect. Add new device?": "Device \"{{name}}\" ({{device}} at {{address}}) wants to connect. Add new device?",
-    "Device ID": "ID du périphérique",
+    "Device \"{%name%}\" ({%device%} at {%address%}) wants to connect. Add new device?": "La machine \"{{name}}\" ({{device}} à l'IP {{address}}) souhaite se connecter. L'acceptez-vous ?",
+    "Device ID": "ID de la machine",
     "Device Identification": "Identification de l'appareil",
-    "Device Name": "Nom du périphérique",
+    "Device Name": "Nom de la machine",
     "Device {%device%} ({%address%}) wants to connect. Add new device?": "L'appareil {{device}} ({{address}}) veut se connecter. Voulez-vous ajouter cette appareil ?",
     "Devices": "Appareil",
     "Disconnected": "Déconnecté",
-    "Discovery": "Discovery",
+    "Discovery": "Découverte",
     "Documentation": "Documentation",
     "Download Rate": "Débit de réception",
     "Downloaded": "Téléchargé",
     "Downloading": "En cours de téléchargement",
-    "Edit": "Éditer",
-    "Edit Device": "Éditer le périphérique",
-    "Edit Folder": "Éditer le répertoire",
+    "Edit": "Modifier",
+    "Edit Device": "Modifier la machine",
+    "Edit Folder": "Modifier le partage",
     "Editing": "Édition",
-    "Enable NAT traversal": "Enable NAT traversal",
-    "Enable Relaying": "Enable Relaying",
+    "Enable NAT traversal": "Activer transfert d'adresses NAT",
+    "Enable Relaying": "Activer le relayage",
     "Enable UPnP": "Activer l'UPnP",
     "Enter comma separated  (\"tcp://ip:port\", \"tcp://host:port\") addresses or \"dynamic\" to perform automatic discovery of the address.": "Entrer les adresses (\"tcp://ip:port\" ou \"tcp://host:port\") séparées par une virgule ou \"dynamic\" afin d'activer la recherche automatique de l'adresse.",
     "Enter ignore patterns, one per line.": "Entrer les masques de filtrage, un par ligne.",
@@ -67,16 +67,16 @@
     "File Pull Order": "Ordre de récupération de fichier",
     "File Versioning": "Versions de fichier",
     "File permission bits are ignored when looking for changes. Use on FAT file systems.": "Les bits de permission de fichier sont ignorés lors de la recherche de changements. Utilisé sur les systèmes de fichiers FAT.",
-    "Files are moved to .stversions folder when replaced or deleted by Syncthing.": "Les fichiers sont déplacés vers le dossier .stversions quand ils sont remplacés ou effacés par Syncthing.",
-    "Files are moved to date stamped versions in a .stversions folder when replaced or deleted by Syncthing.": "Les fichiers sont déplacés, avec horodatage, dans le dossier .stversions quand ils sont remplacés ou supprimés par Syncthing.",
+    "Files are moved to .stversions folder when replaced or deleted by Syncthing.": "Les fichiers sont déplacés vers le répertoire .stversions quand ils sont remplacés ou effacés par Syncthing.",
+    "Files are moved to date stamped versions in a .stversions folder when replaced or deleted by Syncthing.": "Les fichiers sont déplacés, avec horodatage, dans le répertoire .stversions quand ils sont remplacés ou supprimés par Syncthing.",
     "Files are protected from changes made on other devices, but changes made on this device will be sent to the rest of the cluster.": "Les fichiers sont protégés des changements réalisés sur les autres appareils, mais les changements réalisés sur cet appareil seront transférés aux autres appareils.",
-    "Folder": "Dossier",
-    "Folder ID": "ID du répertoire",
-    "Folder Label": "Folder Label",
-    "Folder Master": "Répertoire maître",
-    "Folder Path": "Chemin du répertoire",
-    "Folder Type": "Folder Type",
-    "Folders": "Dossiers",
+    "Folder": "Partage",
+    "Folder ID": "ID du partage",
+    "Folder Label": "Étiquette du partage",
+    "Folder Master": "Partage maître",
+    "Folder Path": "Chemin racine du partage",
+    "Folder Type": "Type de partage",
+    "Folders": "Partages",
     "GUI": "GUI",
     "GUI Authentication Password": "Mot de passe d'authentification GUI",
     "GUI Authentication User": "Utilisateur autorisé GUI",
@@ -84,29 +84,29 @@
     "Generate": "Générer",
     "Global Discovery": "Recherche globale",
     "Global Discovery Server": "Serveur global de recherche",
-    "Global Discovery Servers": "Global Discovery Servers",
+    "Global Discovery Servers": "Serveurs de découverte globale",
     "Global State": "État global",
     "Help": "Aide",
     "Home page": "Page d'accueil",
     "Ignore": "Ignorer",
-    "Ignore Patterns": "Modèles à éviter",
+    "Ignore Patterns": "Masques d'exclusion",
     "Ignore Permissions": "Ignorer les permissions",
     "Incoming Rate Limit (KiB/s)": "Limite du débit entrant (KiB/s)",
-    "Incorrect configuration may damage your folder contents and render Syncthing inoperable.": "Une configuration incorrecte peut créer des dommages dans vos dossiers et mettre hors-service Syncthing",
+    "Incorrect configuration may damage your folder contents and render Syncthing inoperable.": "Une configuration incorrecte peut créer des dommages dans vos répertoires et mettre Syncthing hors-service.",
     "Introducer": "Initiateur",
     "Inversion of the given condition (i.e. do not exclude)": "Inverser la condition donnée (i.e. ne pas exclure)",
     "Keep Versions": "Conserver les versions",
     "Largest First": "Les plus volumineux en premier",
     "Last File Received": "Dernier fichier reçu",
-    "Last Scan": "Last Scan",
+    "Last Scan": "Dernière analyse",
     "Last seen": "Dernière apparition",
     "Later": "Plus tard",
-    "Listeners": "Listeners",
+    "Listeners": "Systèmes à l'écoute",
     "Local Discovery": "Recherche locale",
     "Local State": "État local",
     "Local State (Total)": "État local (Total)",
     "Major Upgrade": "Mise à jour majeure",
-    "Master": "Master",
+    "Master": "Maître",
     "Maximum Age": "Ancienneté maximum",
     "Metadata Only": "Métadonnées uniquement",
     "Minimum Free Disk Space": "Espace disque libre minimum",
@@ -114,7 +114,7 @@
     "Multi level wildcard (matches multiple directory levels)": "Astérisque à plusieurs niveaux (correspond aux répertoires et sous-répertoires)",
     "Never": "Jamais",
     "New Device": "Nouvel appareil",
-    "New Folder": "Nouveau dossier",
+    "New Folder": "Nouveau partage",
     "Newest First": "Les plus récents en premier",
     "No": "Non",
     "No File Versioning": "Pas de version de fichier",
@@ -123,60 +123,60 @@
     "OK": "OK",
     "Off": "Éteint",
     "Oldest First": "Les plus anciens en premier",
-    "Optional descriptive label for the folder. Can be different on each device.": "Optional descriptive label for the folder. Can be different on each device.",
+    "Optional descriptive label for the folder. Can be different on each device.": "Étiquette conviviale pour le partage. Elle peut être différente sur chaque machine.",
     "Options": "Options",
     "Out of Sync": "Désynchronisé",
     "Out of Sync Items": "Objets non synchronisés",
     "Outgoing Rate Limit (KiB/s)": "Limite du débit sortant (KiB/s)",
     "Override Changes": "Écraser les changements",
-    "Path to the folder on the local computer. Will be created if it does not exist. The tilde character (~) can be used as a shortcut for": "Le chemin du dossier sur l'ordinateur local sera créé si il n'existe pas. Le caractère tilde (~) peut être utilisé comme raccourci vers",
+    "Path to the folder on the local computer. Will be created if it does not exist. The tilde character (~) can be used as a shortcut for": "Le chemin du répertoire sur l'ordinateur local sera créé si il n'existe pas. Le caractère tilde (~) peut être utilisé comme raccourci vers",
     "Path where versions should be stored (leave empty for the default .stversions folder in the folder).": "Chemin où les versions doivent être conservées (laisser vide pour le chemin par défaut de .stversions dans le répertoire)",
     "Pause": "Pause",
     "Paused": "En pause",
     "Please consult the release notes before performing a major upgrade.": "Veuillez consulter les notes de version avant de réaliser une mise à jour majeure.",
-    "Please set a GUI Authentication User and Password in the Settings dialog.": "Please set a GUI Authentication User and Password in the Settings dialog.",
+    "Please set a GUI Authentication User and Password in the Settings dialog.": "Veuillez définir un nom d'utilisateur et un mot de passe dans les réglages.",
     "Please wait": "Merci de patienter",
     "Preview": "Aperçu",
     "Preview Usage Report": "Aperçu du rapport de statistiques d'utilisation",
     "Quick guide to supported patterns": "Guide rapide des masques supportés",
     "RAM Utilization": "Utilisation de la RAM",
     "Random": "Aléatoire",
-    "Relay Servers": "Relay Servers",
+    "Relay Servers": "Serveurs relais",
     "Relayed via": "Relayée par",
     "Relays": "Relais",
     "Release Notes": "Notes de version",
-    "Remote Devices": "Remote Devices",
+    "Remote Devices": "Machines distantes",
     "Remove": "Enlever",
-    "Required identifier for the folder. Must be the same on all cluster devices.": "Required identifier for the folder. Must be the same on all cluster devices.",
-    "Rescan": "Rescanner",
+    "Required identifier for the folder. Must be the same on all cluster devices.": "Identifiant obligatoire du partage. Il doit être identique chez chaque participant.",
+    "Rescan": "Réanalyser",
     "Rescan All": "Réanalyser tout",
-    "Rescan Interval": "Intervalle de scan",
+    "Rescan Interval": "Intervalle d'analyse",
     "Restart": "Redémarrer",
     "Restart Needed": "Redémarrage nécessaire",
     "Restarting": "Redémarrage",
     "Resume": "Résumer",
     "Reused": "Réutilisé",
     "Save": "Sauver",
-    "Scan Time Remaining": "Scan Time Remaining",
-    "Scanning": "En cours de scan",
-    "Select the devices to share this folder with.": "Sélectionner les appareils avec qui partager ce dossier.",
-    "Select the folders to share with this device.": "Sélectionner les dossiers à partager avec cet appareil.",
+    "Scan Time Remaining": "Temps d'analyse restant",
+    "Scanning": "Analyse en cours",
+    "Select the devices to share this folder with.": "Sélectionner les machines invitées à ce partage.",
+    "Select the folders to share with this device.": "Sélectionner les partages auxquels cette machine participe.",
     "Settings": "Configuration",
     "Share": "Partager",
-    "Share Folder": "Partager le dossier",
-    "Share Folders With Device": "Partager des dossiers avec des appareils",
+    "Share Folder": "Partager",
+    "Share Folders With Device": "Partages avec cette machine",
     "Share With Devices": "Partage avec des appareils",
-    "Share this folder?": "Voulez-vous partager ce dossier ?",
+    "Share this folder?": "Acceptez-vous ce partage ?",
     "Shared With": "Partagé avec",
-    "Short identifier for the folder. Must be the same on all cluster devices.": "Identifiant court du dossier. Il doit être le même sur l'ensemble des appareils du groupe.",
-    "Show ID": "Montrer l'ID",
-    "Show QR": "Show QR",
+    "Short identifier for the folder. Must be the same on all cluster devices.": "Identifiant court du partage. Il sera le même sur l'ensemble des machines du groupe.",
+    "Show ID": "Afficher mon ID",
+    "Show QR": "Afficher l'image QR",
     "Shown instead of Device ID in the cluster status. Will be advertised to other devices as an optional default name.": "Affiché à la place de l'ID de l'appareil dans le groupe. Sera proposé aux autres appareils comme nom optionnel par défaut.",
     "Shown instead of Device ID in the cluster status. Will be updated to the name the device advertises if left empty.": "Affiché à la place de l'ID de l'appareil dans le groupe. Si laissé vide, il sera mis à jour par le nom proposé par l'appareil distant.",
-    "Shutdown": "Éteindre",
-    "Shutdown Complete": "Extinction terminée",
+    "Shutdown": "Arrêter",
+    "Shutdown Complete": "Arrêté !",
     "Simple File Versioning": "Suivi simple des versions de fichier",
-    "Single level wildcard (matches within a directory only)": "Astérisque à un seul niveau (correspond uniquement à l’intérieur du dossier)",
+    "Single level wildcard (matches within a directory only)": "Astérisque à un seul niveau (correspond uniquement à l’intérieur du répertoire)",
     "Smallest First": "Les plus petits en premier",
     "Source Code": "Code source",
     "Staggered File Versioning": "Versions échelonnées de fichier",
@@ -192,20 +192,20 @@
     "Syncthing is upgrading.": "Syncthing est cours de mise à jour.",
     "Syncthing seems to be down, or there is a problem with your Internet connection. Retrying…": "Syncthing semble être éteint, ou il y a un problème avec votre connexion Internet. Nouvelle tentative ...",
     "Syncthing seems to be experiencing a problem processing your request. Please refresh the page or restart Syncthing if the problem persists.": "Syncthing semble avoir un problème pour traiter votre demande. S'il vous plait, rafraichissez la page ou redémarrer Syncthing si le problème persiste.",
-    "The Syncthing admin interface is configured to allow remote access without a password.": "The Syncthing admin interface is configured to allow remote access without a password.",
-    "The aggregated statistics are publicly available at the URL below.": "The aggregated statistics are publicly available at the URL below.",
+    "The Syncthing admin interface is configured to allow remote access without a password.": "L'interface d'administration de Syncthing est configuré pour accepter l'accès distant sans mot de passe !",
+    "The aggregated statistics are publicly available at the URL below.": "Les statistiques aggrégées sont publiquement disponibles à l'adresse ci-dessous.",
     "The aggregated statistics are publicly available at {%url%}.": "Les statistiques agrégées sont disponibles publiquement à l'adresse {{url}}.",
     "The configuration has been saved but not activated. Syncthing must restart to activate the new configuration.": "La configuration a été enregistrée mais pas activée. Syncthing doit redémarrer afin d'activer la nouvelle configuration.",
     "The device ID cannot be blank.": "L'ID de l'appareil ne peut être vide.",
-    "The device ID to enter here can be found in the \"Actions > Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).": "The device ID to enter here can be found in the \"Actions > Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).",
-    "The device ID to enter here can be found in the \"Edit > Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).": "L'ID de l'appareil à entrer peut être trouvé dans le menu \"Éditer > Montrer l'ID\" des autres appareils. Les espaces et les tirets sont optionnels (ils seront ignorés).",
-    "The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.": "Le rapport d'utilisation chiffré est envoyé quotidiennement. Il sert à répertorier les plateformes utilisées, la taille des dossiers et les versions de l'application. Si les données rapportées sont modifiées cette boite de dialogue vous redemandera votre confirmation.",
+    "The device ID to enter here can be found in the \"Actions > Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).": "L'ID de machine à saisir ici se trouve dans le menu \"Actions > Afficher mon ID\" de la machine distante. Les tirets et espaces sont optionnels (et ignorés).",
+    "The device ID to enter here can be found in the \"Edit > Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).": "L'ID de machine à entrer se trouve dans le menu \"Éditer > Montrer l'ID\" de la machine distante. Les espaces et les tirets sont optionnels (ils seront ignorés).",
+    "The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.": "Le rapport d'utilisation chiffré est envoyé quotidiennement. Il sert à répertorier les plateformes utilisées, la taille des partages et les versions de l'application. Si les données rapportées sont modifiées cette boite de dialogue vous redemandera votre confirmation.",
     "The entered device ID does not look valid. It should be a 52 or 56 character string consisting of letters and numbers, with spaces and dashes being optional.": "L'ID de l'appareil inséré ne semble pas être valide. Il devrait ressembler à une chaîne de 52 ou 56 caractères comprenant des lettres, des chiffres et potentiellement des espaces et des traits d'union.",
-    "The first command line parameter is the folder path and the second parameter is the relative path in the folder.": "Le premier paramètre de ligne de commande est le chemin du dossier, et le second est le chemin relatif dans le dossier.",
-    "The folder ID cannot be blank.": "L'identifiant (ID) du dossier ne peut être vide.",
-    "The folder ID must be a short identifier (64 characters or less) consisting of letters, numbers and the dot (.), dash (-) and underscode (_) characters only.": "L'ID du dossier doit être un identifiant court (64 caractères ou moins) comprenant uniquement des lettres, chiffre, points (.), traits d'union (-) et tirets bas (_).",
-    "The folder ID must be unique.": "L'ID du répertoire doit être unique.",
-    "The folder path cannot be blank.": "Le chemin du répertoire ne peut pas être vide.",
+    "The first command line parameter is the folder path and the second parameter is the relative path in the folder.": "Le premier paramètre de ligne de commande est le chemin du répertoire partagé, et le second est le chemin relatif dans le répertoire.",
+    "The folder ID cannot be blank.": "L'identifiant du partage ne peut être vide.",
+    "The folder ID must be a short identifier (64 characters or less) consisting of letters, numbers and the dot (.), dash (-) and underscode (_) characters only.": "L'ID du partage doit être un identifiant court (64 caractères ou moins) comprenant uniquement des lettres, chiffre, points (.), traits d'union (-) et tirets bas (_).",
+    "The folder ID must be unique.": "L'ID du partage doit être unique.",
+    "The folder path cannot be blank.": "Le chemin vers le répertoire ne peut pas être vide.",
     "The following intervals are used: for the first hour a version is kept every 30 seconds, for the first day a version is kept every hour, for the first 30 days a version is kept every day, until the maximum age a version is kept every week.": "Les intervalles suivant sont utilisés: la première heure une version est conservée chaque 30 secondes, le premier jour une version est conservée chaque heure, les premiers 30 jours une version est conservée chaque jour, jusqu'à la limite d'âge maximum une version est conservée chaque semaine.",
     "The following items could not be synchronized.": "Les éléments suivants ne peuvent pas être synchronisés.",
     "The maximum age must be a number and cannot be blank.": "L'âge maximum doit être un nombre et ne peut être vide.",
@@ -219,8 +219,8 @@
     "The rate limit must be a non-negative number (0: no limit)": "La limite de débit ne doit pas être négative (0: Aucune limite)",
     "The rescan interval must be a non-negative number of seconds.": "L'intervalle d'analyse ne doit pas être un nombre négatif de secondes.",
     "They are retried automatically and will be synced when the error is resolved.": "Ils seront réessayés automatiquement et synchronisés quand l'erreur sera résolue.",
-    "This Device": "This Device",
-    "This can easily give hackers access to read and change any files on your computer.": "This can easily give hackers access to read and change any files on your computer.",
+    "This Device": "Cette machine",
+    "This can easily give hackers access to read and change any files on your computer.": "Ceci peut aisément permettre à un intrus de lire et modifier n'importe quel fichier de votre ordinateur.  ",
     "This is a major version upgrade.": "Ceci est une mise à jour majeure.",
     "Trash Can File Versioning": "Gestion des versions de fichier style poubelle.",
     "Unknown": "Inconnu",
@@ -237,15 +237,15 @@
     "Version": "Version",
     "Versions Path": "Emplacement des versions",
     "Versions are automatically deleted if they are older than the maximum age or exceed the number of files allowed in an interval.": "Les versions seront supprimées automatiquement, si elles dépassent la durée maximum de conservation, ou si leur nombre est supérieur à la valeur autorisée dans l'intervalle.",
-    "Warning, this path is a subdirectory of an existing folder \"{%otherFolder%}\".": "Warning, this path is a subdirectory of an existing folder \"{{otherFolder}}\".",
+    "Warning, this path is a subdirectory of an existing folder \"{%otherFolder%}\".": "Attention, ce chemin est un sous-répertoire du partage existant \"{{otherFolder}}\". Ceci peut causer des problèmes tels que duplications de fichiers ou suppressions intempestives sur les autres machines.",
     "When adding a new device, keep in mind that this device must be added on the other side too.": "Lorsqu'un appareil est ajouté, gardez à l'esprit que cet appareil doit aussi être ajouté de l'autre coté.",
-    "When adding a new folder, keep in mind that the Folder ID is used to tie folders together between devices. They are case sensitive and must match exactly between all devices.": "Lorsqu'un nouveau répertoire est ajouté, gardez à l'esprit que son ID est utilisé pour lier les répertoires à travers les appareils. Les ID sont sensibles à la casse et doivent être identiques à travers tous les nœuds.",
+    "When adding a new folder, keep in mind that the Folder ID is used to tie folders together between devices. They are case sensitive and must match exactly between all devices.": "Lorsqu'un nouveau partage est ajouté, gardez à l'esprit que son ID est utilisée pour lier les répertoires à travers les machines. L'ID est sensible à la casse et sera forcément la même sur toutes les machines participant à ce partage.",
     "Yes": "Oui",
     "You must keep at least one version.": "Vous devez garder au minimum une version.",
     "days": "Jours",
     "full documentation": "documentation complète",
     "items": "éléments",
-    "{%device%} wants to share folder \"{%folder%}\".": "{{device}} veut partager le dossier \"{{folder}}\".",
-    "{%device%} wants to share folder \"{%folderLabel%}\" ({%folder%}).": "{{device}} wants to share folder \"{{folderLabel}}\" ({{folder}}).",
-    "{%device%} wants to share folder \"{%folderlabel%}\" ({%folder%}).": "{{device}} wants to share folder \"{{folderlabel}}\" ({{folder}})."
+    "{%device%} wants to share folder \"{%folder%}\".": "{{device}} vous invite au partage \"{{folderLabel}}\".",
+    "{%device%} wants to share folder \"{%folderLabel%}\" ({%folder%}).": "{{device}} vous invite au partage \"{{folderLabel}}\" ({{folder}}).",
+    "{%device%} wants to share folder \"{%folderlabel%}\" ({%folder%}).": "{{device}} vous invite au partage \"{{folderLabel}}\" ({{folder}})."
 }

gui/default/assets/lang/lang-fr.json

@@ -40,7 +40,7 @@
     "Danger!": "Attention !",
     "Delete": "Supprimer",
     "Deleted": "Supprimé",
-    "Device \"{%name%}\" ({%device%} at {%address%}) wants to connect. Add new device?": "La machine \"{{name}}\" ({{device}} sur {{address}}) veut se connecter. Ajouter cette nouvelle machine ?",
+    "Device \"{%name%}\" ({%device%} at {%address%}) wants to connect. Add new device?": "La machine \"{{name}}\" ({{device}} à {{address}}) veut se connecter. L'acceptez-vous ?",
     "Device ID": "ID de la machine",
     "Device Identification": "Identifiant de la machine",
     "Device Name": "Nom de la machine",
@@ -52,12 +52,12 @@
     "Download Rate": "Vitesse de réception",
     "Downloaded": "Téléchargé",
     "Downloading": "En cours de téléchargement",
-    "Edit": "Éditer",
-    "Edit Device": "Éditer la machine",
-    "Edit Folder": "Éditer le dossier",
-    "Editing": "Édition",
-    "Enable NAT traversal": "Activer transfert d'adresses (NAT)",
-    "Enable Relaying": "Activer le relayage",
+    "Edit": "Modifier",
+    "Edit Device": "Modifier la machine",
+    "Edit Folder": "Modifier le dossier",
+    "Editing": "Modifications",
+    "Enable NAT traversal": "Activer la translation d'adresses (NAT)",
+    "Enable Relaying": "Relayage possible",
     "Enable UPnP": "Activer UPnP",
     "Enter comma separated  (\"tcp://ip:port\", \"tcp://host:port\") addresses or \"dynamic\" to perform automatic discovery of the address.": "Entrer les adresses (\"tcp://ip:port\" ou \"tcp://host:port\") séparées par une virgule ou \"dynamic\" afin d'activer la recherche automatique de l'adresse.",
     "Enter ignore patterns, one per line.": "Entrer les masques de filtrage, un par ligne.",
@@ -72,10 +72,10 @@
     "Files are protected from changes made on other devices, but changes made on this device will be sent to the rest of the cluster.": "Les fichiers sont protégés des changements réalisés sur les autres machines, mais les changements réalisés sur celle-ci seront transférés aux autres machines.",
     "Folder": "Dossier",
     "Folder ID": "ID du dossier",
-    "Folder Label": "Étiquette du dossier",
+    "Folder Label": "Étiquette du partage",
     "Folder Master": "Dossier maître",
     "Folder Path": "Chemin du dossier",
-    "Folder Type": "Type de répertoire",
+    "Folder Type": "Type de partage",
     "Folders": "Dossiers",
     "GUI": "GUI",
     "GUI Authentication Password": "Mot de passe d'authentification GUI",
@@ -89,7 +89,7 @@
     "Help": "Aide",
     "Home page": "Page d'accueil",
     "Ignore": "Ignorer",
-    "Ignore Patterns": "Modèles à éviter",
+    "Ignore Patterns": "Masques d'exclusion",
     "Ignore Permissions": "Ignorer les permissions",
     "Incoming Rate Limit (KiB/s)": "Limite du débit de réception (Ko/s)",
     "Incorrect configuration may damage your folder contents and render Syncthing inoperable.": "Une configuration incorrecte peut créer des dommages dans vos dossiers et mettre hors-service Syncthing",
@@ -106,7 +106,7 @@
     "Local State": "État local",
     "Local State (Total)": "État local (Total)",
     "Major Upgrade": "Mise à jour majeure",
-    "Master": "Maitre",
+    "Master": "Maître",
     "Maximum Age": "Ancienneté maximum",
     "Metadata Only": "Métadonnées uniquement",
     "Minimum Free Disk Space": "Espace disque libre minimum",
@@ -123,7 +123,7 @@
     "OK": "OK",
     "Off": "Éteint",
     "Oldest First": "Les plus anciens en premier",
-    "Optional descriptive label for the folder. Can be different on each device.": "Étiquette optionnelle pour le dossier. Peut être différente pour chaque machine.",
+    "Optional descriptive label for the folder. Can be different on each device.": "Étiquette optionnelle pour le partage. Peut être différente sur chaque machine.",
     "Options": "Options",
     "Out of Sync": "Désynchronisé",
     "Out of Sync Items": "Fichiers non synchronisés",
@@ -134,7 +134,7 @@
     "Pause": "Pause",
     "Paused": "En pause",
     "Please consult the release notes before performing a major upgrade.": "Veuillez consulter les notes de version avant de réaliser une mise à jour majeure.",
-    "Please set a GUI Authentication User and Password in the Settings dialog.": "SVP, mettez un nom d'utilisateur et un mot de passe dans la fenêtre de paramétrage.",
+    "Please set a GUI Authentication User and Password in the Settings dialog.": "Veuillez définir un nom d'utilisateur et un mot de passe dans la fenêtre de Configuration.",
     "Please wait": "Merci de patienter",
     "Preview": "Aperçu",
     "Preview Usage Report": "Aperçu du rapport de statistiques d'utilisation",
@@ -147,7 +147,7 @@
     "Release Notes": "Notes de version",
     "Remote Devices": "Machines distantes",
     "Remove": "Enlever",
-    "Required identifier for the folder. Must be the same on all cluster devices.": "Identifiant pour le dossier. Doit être le même sur l'ensemble des machines du cluster.",
+    "Required identifier for the folder. Must be the same on all cluster devices.": "Identifiant du partage. Doit être le même sur l'ensemble des machines concernées.",
     "Rescan": "Réanalyse",
     "Rescan All": "Réanalyser tout",
     "Rescan Interval": "Intervalle d'analyse",
@@ -157,7 +157,7 @@
     "Resume": "Résumer",
     "Reused": "Réutilisé",
     "Save": "Sauver",
-    "Scan Time Remaining": "Intervalle entre chaque analyse",
+    "Scan Time Remaining": "Temps d'analyse restant",
     "Scanning": "Analyse en cours",
     "Select the devices to share this folder with.": "Sélectionner les machines avec qui partager ce dossier.",
     "Select the folders to share with this device.": "Sélectionner les dossiers à partager avec cette machine.",
@@ -169,12 +169,12 @@
     "Share this folder?": "Voulez-vous partager ce dossier ?",
     "Shared With": "Partagé avec",
     "Short identifier for the folder. Must be the same on all cluster devices.": "Identifiant court du dossier. Il doit être le même sur l'ensemble des machines du groupe.",
-    "Show ID": "Afficher l'ID",
+    "Show ID": "Afficher mon ID",
     "Show QR": "Afficher le QR",
     "Shown instead of Device ID in the cluster status. Will be advertised to other devices as an optional default name.": "Affiché à la place de l'ID de la machine dans le groupe. Sera proposé aux autres machines comme nom optionnel par défaut.",
     "Shown instead of Device ID in the cluster status. Will be updated to the name the device advertises if left empty.": "Affiché à la place de l'ID de la machine dans le groupe. Si laissé vide, il sera mis à jour par le nom proposé par la machine distante.",
-    "Shutdown": "Éteindre",
-    "Shutdown Complete": "Extinction terminée",
+    "Shutdown": "Arrêter",
+    "Shutdown Complete": "Arrêté !",
     "Simple File Versioning": "Suivi simple des versions de fichier",
     "Single level wildcard (matches within a directory only)": "Astérisque à un seul niveau (correspond uniquement à l’intérieur du dossier)",
     "Smallest First": "Les plus petits en premier",
@@ -192,13 +192,13 @@
     "Syncthing is upgrading.": "Syncthing est cours de mise à jour.",
     "Syncthing seems to be down, or there is a problem with your Internet connection. Retrying…": "Syncthing semble être éteint, ou il y a un problème avec votre connexion Internet. Nouvelle tentative ...",
     "Syncthing seems to be experiencing a problem processing your request. Please refresh the page or restart Syncthing if the problem persists.": "Syncthing semble avoir un problème pour traiter votre demande. S'il vous plait, rafraichissez la page ou redémarrer Syncthing si le problème persiste.",
-    "The Syncthing admin interface is configured to allow remote access without a password.": "L'interface d'administration de Syncthing est paramétrée pour autoriser les accès à distance sans mot de passe.",
+    "The Syncthing admin interface is configured to allow remote access without a password.": "L'interface d'administration de Syncthing est paramétrée pour autoriser les accès à distance sans mot de passe !!!",
     "The aggregated statistics are publicly available at the URL below.": "Les statistiques agrégées sont disponibles publiquement à l'adresse ci-dessous.",
     "The aggregated statistics are publicly available at {%url%}.": "Les statistiques agrégées sont disponibles publiquement à l'adresse {{url}}.",
     "The configuration has been saved but not activated. Syncthing must restart to activate the new configuration.": "La configuration a été enregistrée mais pas activée. Syncthing doit redémarrer afin d'activer la nouvelle configuration.",
     "The device ID cannot be blank.": "L'ID de la machine ne peut être vide.",
-    "The device ID to enter here can be found in the \"Actions > Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).": "L'ID de la machine à indiquer ici se trouve dans \"Actions > Afficher ID\" sur l'autre machine. Espaces et tirets sont optionnels (ignorés).",
-    "The device ID to enter here can be found in the \"Edit > Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).": "L'ID de la machine à entrer peut être trouvé dans le menu \"Éditer > Montrer l'ID\" des autres machines. Les espaces et les tirets sont optionnels (ils seront ignorés).",
+    "The device ID to enter here can be found in the \"Actions > Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).": "L'ID de machine à saisir se trouve dans le menu \"Actions > Afficher mon ID\" de la machine distante. Espaces et tirets sont optionnels (ignorés).",
+    "The device ID to enter here can be found in the \"Edit > Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).": "L'ID de machine à saisir se trouve dans le menu \"Modifications > Afficher mon ID\" de la machine distante. Les espaces et les tirets sont optionnels (ils seront ignorés).",
     "The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.": "Le rapport d'utilisation chiffré est envoyé quotidiennement. Il sert à répertorier les plateformes utilisées, la taille des dossiers et les versions de l'application. Si les données rapportées sont modifiées cette boite de dialogue vous redemandera votre confirmation.",
     "The entered device ID does not look valid. It should be a 52 or 56 character string consisting of letters and numbers, with spaces and dashes being optional.": "L'ID de la machine inséré ne semble pas être valide. Il devrait ressembler à une chaîne de 52 ou 56 caractères comprenant des lettres, des chiffres et potentiellement des espaces et des traits d'union.",
     "The first command line parameter is the folder path and the second parameter is the relative path in the folder.": "Le premier paramètre de ligne de commande est le chemin du dossier, et le second est le chemin relatif dans le dossier.",
@@ -220,7 +220,7 @@
     "The rescan interval must be a non-negative number of seconds.": "L'intervalle d'analyse ne doit pas être un nombre négatif de secondes.",
     "They are retried automatically and will be synced when the error is resolved.": "Ils seront réessayés automatiquement et synchronisés quand l'erreur sera résolue.",
     "This Device": "Cette machine",
-    "This can easily give hackers access to read and change any files on your computer.": "Cela permet facilement aux pirates de lire et modifier n'importe quel fichier de votre machine.",
+    "This can easily give hackers access to read and change any files on your computer.": "Ceci peut aisément permettre à un intrus de lire et modifier n'importe quel fichier de votre ordinateur.",
     "This is a major version upgrade.": "Ceci est une mise à jour majeure.",
     "Trash Can File Versioning": "Gestion des versions de fichier style poubelle.",
     "Unknown": "Inconnu",
@@ -237,7 +237,7 @@
     "Version": "Version",
     "Versions Path": "Emplacement des versions",
     "Versions are automatically deleted if they are older than the maximum age or exceed the number of files allowed in an interval.": "Les versions seront supprimées automatiquement, si elles dépassent la durée maximum de conservation, ou si leur nombre est supérieur à la valeur autorisée dans l'intervalle.",
-    "Warning, this path is a subdirectory of an existing folder \"{%otherFolder%}\".": "Attention, ce chemin est un sous-répertoire du dossier existant \"{{otherFolder}}\".",
+    "Warning, this path is a subdirectory of an existing folder \"{%otherFolder%}\".": "[b]Attention, ce chemin est un sous-répertoire du partage existant \"{{otherFolder}}\". Ceci peut causer des problèmes tels que duplications de fichiers ou suppressions intempestives sur les autres machines.[/b]",
     "When adding a new device, keep in mind that this device must be added on the other side too.": "Lorsqu'une machine est ajoutée, gardez à l'esprit que cette machine doit aussi être ajoutée de l'autre coté.",
     "When adding a new folder, keep in mind that the Folder ID is used to tie folders together between devices. They are case sensitive and must match exactly between all devices.": "Lorsqu'un nouveau dossier est ajouté, gardez à l'esprit que son ID est utilisé pour lier les dossiers à travers les machines. Les ID sont sensibles à la casse et doivent être identiques à travers tous les nœuds.",
     "Yes": "Oui",
@@ -246,6 +246,6 @@
     "full documentation": "documentation complète",
     "items": "fichiers",
     "{%device%} wants to share folder \"{%folder%}\".": "{{device}} veut partager le dossier \"{{folder}}\".",
-    "{%device%} wants to share folder \"{%folderLabel%}\" ({%folder%}).": "{{device}} veut partager le dossier \"{{folderLabel}}\" ({{folder}}).",
-    "{%device%} wants to share folder \"{%folderlabel%}\" ({%folder%}).": "{{device}} veut partager le dossier \"{{folderLabel}}\" ({{folder}})."
+    "{%device%} wants to share folder \"{%folderLabel%}\" ({%folder%}).": "{{device}} vous invite au partage \"{{folderLabel}}\" ({{folder}}).",
+    "{%device%} wants to share folder \"{%folderlabel%}\" ({%folder%}).": "{{device}} vous invite au partage \"{{folderLabel}}\" ({{folder}})."
 }

gui/default/assets/lang/lang-hu.json

@@ -33,8 +33,8 @@
     "Compression": "Tömörítés",
     "Connection Error": "Kapcsolódási hiba",
     "Connection Type": "Kapcsolat típus",
-    "Copied from elsewhere": "Másolva máshonnan",
-    "Copied from original": "Másolva az eredetiről",
+    "Copied from elsewhere": "Máshonnan másolva",
+    "Copied from original": "Eredetiről másolva",
     "Copyright © 2014-2016 the following Contributors:": "Szerzői jog © 2014-2016 az alábbi közreműködők:",
     "Copyright © 2015 the following Contributors:": "Copyright © 2015 az alábbi közreműködők:",
     "Danger!": "Veszély!",
@@ -210,7 +210,7 @@
     "The following items could not be synchronized.": "A következő elemek nem szinkronizálhatóak.",
     "The maximum age must be a number and cannot be blank.": "A maximális kornak számnak kell lenni és nem lehet üres",
     "The maximum time to keep a version (in days, set to 0 to keep versions forever).": "A verziók megtartásának maximális ideje (napokban, ha 0-t adsz meg örökre megmaradnak).",
-    "The minimum free disk space percentage must be a non-negative number between 0 and 100 (inclusive).": "A minimális szabad terület százalékos. nem-negatív érték 0 és 100 között",
+    "The minimum free disk space percentage must be a non-negative number between 0 and 100 (inclusive).": "A minimális szabad terület százalékos, nem-negatív értéke 0 és 100 között",
     "The number of days must be a number and cannot be blank.": "A napok száma szám kell legyen és nem lehet üres.",
     "The number of days to keep files in the trash can. Zero means forever.": "A napok száma ameddig a fájlok meg lesznek tartva a szemetesben. A 0 azt jelenti örökre.",
     "The number of old versions to keep, per file.": "A megtartott régi verziók száma, fájlonként.",
@@ -220,7 +220,7 @@
     "The rescan interval must be a non-negative number of seconds.": "Az átnézési intervallum nullánál nagyobb másodperc érték kell legyen",
     "They are retried automatically and will be synced when the error is resolved.": "A hiba javítása után automatikusan újra megpróbálja a szinkronizálást.",
     "This Device": "Ez az eszköz",
-    "This can easily give hackers access to read and change any files on your computer.": "Így a hekkerek könnyedén hozzáférhetnek a számítógépen található fájlokhoz. ",
+    "This can easily give hackers access to read and change any files on your computer.": "Így a hekkerek könnyedén hozzáférést szerezhetnek a gépeden tárolt fájlok olvasásához és módosításához.",
     "This is a major version upgrade.": "Ez egy főverzió frissítés.",
     "Trash Can File Versioning": "Szemetes fájl verziókövetés",
     "Unknown": "Ismeretlen",
@@ -241,7 +241,7 @@
     "When adding a new device, keep in mind that this device must be added on the other side too.": "Amikor új eszközt adsz hozzá, tartsd észben, hogy a másik oldalon ezt az eszközt is hozzá kell adni.",
     "When adding a new folder, keep in mind that the Folder ID is used to tie folders together between devices. They are case sensitive and must match exactly between all devices.": "Amikor új mappát adsz hozzá, tartsd észben, hogy a mappa azonosító arra való hogy összekösd a mappákat az eszközeiden. Az azonosító kisbetű-nagybetű érzékeny és pontosan egyeznie kell az eszközökön.",
     "Yes": "Igen",
-    "You must keep at least one version.": "Legalább egy verziót meg kell tartanod",
+    "You must keep at least one version.": "Legalább egy verziót meg kell tartanod.",
     "days": "nap",
     "full documentation": "teljes dokumentáció",
     "items": "elem",

gui/default/assets/lang/lang-ko-KR.json

@@ -3,7 +3,7 @@
     "A negative number of days doesn't make sense.": "음수로는 지정할 수 없습니다.",
     "A new major version may not be compatible with previous versions.": "새로운 메이저 버전은 이전 버전과 호환되지 않을 수 있습니다.",
     "API Key": "API 키",
-    "About": " 정보",
+    "About": "정보",
     "Actions": "동작",
     "Add": "추가",
     "Add Device": "기기 추가",

gui/default/assets/lang/lang-pt-BR.json

@@ -60,7 +60,7 @@
     "Enable Relaying": "Habilitar retransmissão",
     "Enable UPnP": "Habilitar UPnP",
     "Enter comma separated  (\"tcp://ip:port\", \"tcp://host:port\") addresses or \"dynamic\" to perform automatic discovery of the address.": "Insira endereços (\"tcp://ip:porta\", \"tcp://host:porta\") separados por vírgula ou \"dynamic\" para executar a descoberta automática do endereço.",
-    "Enter ignore patterns, one per line.": "Insira os padrões de exclusão, um por linha.",
+    "Enter ignore patterns, one per line.": "Insira os filtros, um por linha.",
     "Error": "Erro",
     "External File Versioning": "Versionamento externo de arquivo",
     "Failed Items": "Itens com falha",
@@ -89,7 +89,7 @@
     "Help": "Ajuda",
     "Home page": "Página inicial",
     "Ignore": "Ignorar",
-    "Ignore Patterns": "Padrões de exclusão",
+    "Ignore Patterns": "Filtros",
     "Ignore Permissions": "Ignorar permissões",
     "Incoming Rate Limit (KiB/s)": "Limite de velocidade de recepção (KiB/s)",
     "Incorrect configuration may damage your folder contents and render Syncthing inoperable.": "A configuração incorreta poderá causar danos aos seus dados e tornar o Syncthing inoperante.",

gui/default/assets/lang/lang-sv.json

@@ -1,6 +1,6 @@
 {
-    "A device with that ID is already added.": "En enhet med det ID är redan tillagt.",
-    "A negative number of days doesn't make sense.": "Ett negativt antal dagar är inte troligt.",
+    "A device with that ID is already added.": "En enhet med det ID:t är redan tillagt.",
+    "A negative number of days doesn't make sense.": "Ett negativt antal dagar är inte rimligt.",
     "A new major version may not be compatible with previous versions.": "En ny huvudversion kan eventuellt vara inkompatibel med tidigare versioner.",
     "API Key": "API-nyckel",
     "About": "Om",
@@ -18,8 +18,8 @@
     "All Data": "All data",
     "Allow Anonymous Usage Reporting?": "Tillåt anonym användarstatistiksrapportering?",
     "Alphabetic": "Alfabetisk",
-    "An external command handles the versioning. It has to remove the file from the synced folder.": "Ett externt kommando sköter versionshanteringen. Det måste ta bort filen från den synkroniserade mappen.",
-    "Anonymous Usage Reporting": "Anonym användarstatistik",
+    "An external command handles the versioning. It has to remove the file from the synced folder.": "Ett externt kommando sköter versionshanteringen. Den behöver ta bort filen från den synkroniserade mappen.",
+    "Anonymous Usage Reporting": "Anonym användarstatistik rapportering",
     "Any devices configured on an introducer device will be added to this device as well.": "Enheter konfigurerade på en introduktörsenhet kommer också att läggas till den här enheten.",
     "Automatic upgrades": "Automatiska uppgraderingar",
     "Be careful!": "Var aktsam!",
@@ -29,87 +29,87 @@
     "Clean out after": "Rensa efteråt",
     "Close": "Stäng",
     "Command": "Kommando",
-    "Comment, when used at the start of a line": "Kommentar, vid början av en rad.",
+    "Comment, when used at the start of a line": "Kommentar, vid användning i början av en rad.",
     "Compression": "Komprimering",
     "Connection Error": "Anslutningsproblem",
     "Connection Type": "Anslutningstyp",
     "Copied from elsewhere": "Kopierat utifrån",
-    "Copied from original": "Oförändrat",
-    "Copyright © 2014-2016 the following Contributors:": "Copyright © 2014-2016 följande bidragande:",
+    "Copied from original": "Kopierat från original",
+    "Copyright © 2014-2016 the following Contributors:": "Copyright © 2014-2016 följande bidragare:",
     "Copyright © 2015 the following Contributors:": "Copyright © 2015 följande medverkande:",
     "Danger!": "Fara!",
     "Delete": "Ta bort",
     "Deleted": "Borttaget",
     "Device \"{%name%}\" ({%device%} at {%address%}) wants to connect. Add new device?": "Enhet \"{{name}}\" ({{device}} på {{address}}) vill ansluta. Lägg till ny enhet?",
-    "Device ID": "Enhets-ID",
+    "Device ID": "Enhets ID",
     "Device Identification": "Enhetsidentifikation",
-    "Device Name": "Enhetsnamn",
+    "Device Name": "Enhetens namn",
     "Device {%device%} ({%address%}) wants to connect. Add new device?": "Enheten {{device}} ({{address}}) vill ansluta. Lägg till ny enhet?",
     "Devices": "Enheter",
-    "Disconnected": "Ej ansluten",
-    "Discovery": "Uppslagning",
+    "Disconnected": "Frånkopplad",
+    "Discovery": "Upptäckt",
     "Documentation": "Dokumentation",
     "Download Rate": "Nedladdningshastighet",
-    "Downloaded": "Nerladdat",
-    "Downloading": "Laddar ner",
+    "Downloaded": "Hämtat",
+    "Downloading": "Hämtar",
     "Edit": "Redigera",
     "Edit Device": "Redigera enhet",
     "Edit Folder": "Redigera katalog",
     "Editing": "Redigerar",
     "Enable NAT traversal": "Aktivera NAT traversering",
     "Enable Relaying": "Aktivera reläa",
-    "Enable UPnP": "Använd UPnP",
+    "Enable UPnP": "Aktivera UPnP",
     "Enter comma separated  (\"tcp://ip:port\", \"tcp://host:port\") addresses or \"dynamic\" to perform automatic discovery of the address.": "Ange kommaseparerade (\"tcp://ip:port\", \"tcp://host:port\")-adresser eller ordet \"dynamic\" för att använda automatisk uppslagning.",
-    "Enter ignore patterns, one per line.": "Ange filmönster, ett per rad.",
+    "Enter ignore patterns, one per line.": "Ange ignorera mönster, en per rad.",
     "Error": "Fel",
-    "External File Versioning": "Extern versionshantering",
-    "Failed Items": "Misslyckade filer",
-    "File Pull Order": "Hämtningsprioritering av filer",
-    "File Versioning": "Versionshantering",
+    "External File Versioning": "Extern filversionshantering",
+    "Failed Items": "Misslyckade objekt",
+    "File Pull Order": "Filhämtningsprioritering",
+    "File Versioning": "Filversionshantering",
     "File permission bits are ignored when looking for changes. Use on FAT file systems.": "Filrättigheter ignoreras vid sökning efter förändringar. Används på FAT-filsystem.",
     "Files are moved to .stversions folder when replaced or deleted by Syncthing.": "Filer flyttas till katalogen .stversions om de ersätts eller raderas av Syncthing.",
     "Files are moved to date stamped versions in a .stversions folder when replaced or deleted by Syncthing.": "Filer flyttas till datummärkta versioner i en .stversions-mapp när de ersatts eller raderats av Syncthing.",
     "Files are protected from changes made on other devices, but changes made on this device will be sent to the rest of the cluster.": "Filer skyddas från ändringar gjorda på andra enheter, men ändringar som görs på den här noden skickas till de andra klustermedlemmarna.",
     "Folder": "Katalog",
     "Folder ID": "Katalog-ID",
-    "Folder Label": "Katalog etikett",
-    "Folder Master": "Huvudlagring",
-    "Folder Path": "Sökväg",
+    "Folder Label": "Katalog-etikett",
+    "Folder Master": "Huvudkatalog",
+    "Folder Path": "Katalog-sökväg",
     "Folder Type": "Katalogtyp",
     "Folders": "Kataloger",
-    "GUI": "GUI",
-    "GUI Authentication Password": "GUI-lösenord",
-    "GUI Authentication User": "GUI-användare",
-    "GUI Listen Addresses": "GUI-adress",
-    "Generate": "Skapa",
-    "Global Discovery": "Global uppslagning",
-    "Global Discovery Server": "Global uppslagningsserver",
-    "Global Discovery Servers": "Globala uppslagningsservrar",
+    "GUI": "Grafiskt gränssnitt",
+    "GUI Authentication Password": "Lösenord för GUI",
+    "GUI Authentication User": "Användare för GUI",
+    "GUI Listen Addresses": "Lyssningsadresser för GUI",
+    "Generate": "Generera",
+    "Global Discovery": "Global upptäckt",
+    "Global Discovery Server": "Global upptäcktsserver",
+    "Global Discovery Servers": "Globala upptäcktsservrar",
     "Global State": "Global status",
     "Help": "Hjälp",
     "Home page": "Hemsida",
     "Ignore": "Ignorera",
-    "Ignore Patterns": "Ignorerade filmönster",
-    "Ignore Permissions": "Ignorera filrättigheter",
-    "Incoming Rate Limit (KiB/s)": "Max nedladdningshastighet (KiB/s)",
+    "Ignore Patterns": "Ignorera mönster",
+    "Ignore Permissions": "Ignorera rättigheter",
+    "Incoming Rate Limit (KiB/s)": "Nedladdningshastighetsgräns (KiB/s)",
     "Incorrect configuration may damage your folder contents and render Syncthing inoperable.": "Inkorrekt konfiguration kan skada innehållet i katalogen and få Syncthing att sluta fungera.",
     "Introducer": "introduktör",
     "Inversion of the given condition (i.e. do not exclude)": "Vänder på villkoret, d.v.s. exkluderar inte.",
     "Keep Versions": "Behåll versioner",
-    "Largest First": "Störst först",
-    "Last File Received": "Senast Mottagna Fil",
-    "Last Scan": "Senast skanning",
-    "Last seen": "Senast online",
+    "Largest First": "Största först",
+    "Last File Received": "Senaste fil mottagen",
+    "Last Scan": "Senaste skanning",
+    "Last seen": "Senast sedd",
     "Later": "Senare",
     "Listeners": "Lyssnare",
-    "Local Discovery": "Lokal uppslagning",
+    "Local Discovery": "Lokal upptäckt",
     "Local State": "Lokal status",
-    "Local State (Total)": "Lokal status (Total)",
+    "Local State (Total)": "Lokal status (totalt)",
     "Major Upgrade": "Stor uppgradering",
     "Master": "Huvud",
-    "Maximum Age": "Högsta ålder",
+    "Maximum Age": "Maximum ålder",
     "Metadata Only": "Endast metadata",
-    "Minimum Free Disk Space": "Minimum ledigt diskutrymme",
+    "Minimum Free Disk Space": "Minsta ledigt diskutrymme",
     "Move to top of queue": "Flytta till överst i kön",
     "Multi level wildcard (matches multiple directory levels)": "Jokertecken som representerar noll eller fler godtyckliga tecken, även över kataloggränser.",
     "Never": "Aldrig",
@@ -117,7 +117,7 @@
     "New Folder": "Ny katalog",
     "Newest First": "Nyast först",
     "No": "Nej",
-    "No File Versioning": "Ingen versionshantering",
+    "No File Versioning": "Ingen filversionshantering",
     "Normal": "Normal",
     "Notice": "Observera",
     "OK": "OK",
@@ -126,39 +126,39 @@
     "Optional descriptive label for the folder. Can be different on each device.": "Valfri beskrivande etikett för katalogen. Kan vara olika på varje enhet.",
     "Options": "Alternativ",
     "Out of Sync": "Osynkroniserad",
-    "Out of Sync Items": "Osynkroniserade poster",
-    "Outgoing Rate Limit (KiB/s)": "Max uppladdningshastighet (KiB/s)",
-    "Override Changes": "Skriv över ändringar",
+    "Out of Sync Items": "Osynkroniserade objekt",
+    "Outgoing Rate Limit (KiB/s)": "Uppladdningshastighetsgräns (KiB/s)",
+    "Override Changes": "Åsidosätt förändringar",
     "Path to the folder on the local computer. Will be created if it does not exist. The tilde character (~) can be used as a shortcut for": "Sökväg till katalogen på din dator. Kommer att skapas om det inte finns. Tecknet tilde (~) kan användas som en genväg för",
     "Path where versions should be stored (leave empty for the default .stversions folder in the folder).": "Sökväg där versioner sparas (lämna tomt för att använda .stversions i den ordinarie katalogen).",
     "Pause": "Paus",
     "Paused": "Pausad",
     "Please consult the release notes before performing a major upgrade.": "Läs igenom versionsnyheterna innan den stora uppgraderingen.",
-    "Please set a GUI Authentication User and Password in the Settings dialog.": "Ställ in ett grafiskt användarautentisering och lösenord i dialogrutan Inställningar.",
+    "Please set a GUI Authentication User and Password in the Settings dialog.": "Ställ in ett grafiska gränssnittets användarautentisering och lösenord i inställningsdialogrutan.",
     "Please wait": "Var god vänta",
     "Preview": "Förhandsgranska",
     "Preview Usage Report": "Förhandsgranska statistik",
-    "Quick guide to supported patterns": "Snabb guide till filmönster som stöds",
-    "RAM Utilization": "Minnesanvändning",
+    "Quick guide to supported patterns": "Snabb handledning till mönster som stöds",
+    "RAM Utilization": "RAM-användning",
     "Random": "Slumpmässig",
     "Relay Servers": "Reläservrar",
     "Relayed via": "Vidarbefordras via",
-    "Relays": "Vidarbefordringar",
+    "Relays": "Reläservrar",
     "Release Notes": "Versionsanteckningar",
     "Remote Devices": "Fjärrenheter",
     "Remove": "Ta bort",
     "Required identifier for the folder. Must be the same on all cluster devices.": "Krävs identifierare för katalogen. Måste vara densamma på alla kluster enheter.",
-    "Rescan": "Uppdatera",
-    "Rescan All": "Uppdatera alla",
-    "Rescan Interval": "Uppdateringsintervall",
+    "Rescan": "Skanna om",
+    "Rescan All": "Skanna om alla",
+    "Rescan Interval": "Omskanningsintervall",
     "Restart": "Starta om",
     "Restart Needed": "Omstart behövs",
     "Restarting": "Startar om",
     "Resume": "Återuppta",
-    "Reused": "Återanvänt",
+    "Reused": "Återanvänd",
     "Save": "Spara",
-    "Scan Time Remaining": "Granska återstående tid",
-    "Scanning": "Uppdaterar",
+    "Scan Time Remaining": "Återstående skanningstid",
+    "Scanning": "Skannar",
     "Select the devices to share this folder with.": "Ange enheterna att dela den här katalogen med.",
     "Select the folders to share with this device.": "Välj kataloger att dela med den här enheten.",
     "Settings": "Inställningar",
@@ -166,67 +166,67 @@
     "Share Folder": "Dela katalog",
     "Share Folders With Device": "Dela kataloger med enhet",
     "Share With Devices": "Dela med enheter",
-    "Share this folder?": "Dela den här katalogen?",
+    "Share this folder?": "Dela denna katalog?",
     "Shared With": "Delad med",
     "Short identifier for the folder. Must be the same on all cluster devices.": "Kort identifieringssträng för katalogen. Måste vara samma på alla enheter i klustret.",
     "Show ID": "Visa ID",
     "Show QR": "Visa QR",
-    "Shown instead of Device ID in the cluster status. Will be advertised to other devices as an optional default name.": "Visas i stället för enhets-ID. Skickas till andra enheter som namn på denna enhet.",
-    "Shown instead of Device ID in the cluster status. Will be updated to the name the device advertises if left empty.": "Visas i stället för enhets-ID. Sätts till namnet på den andra enheten vid första anslutning om det lämnas tomt.",
+    "Shown instead of Device ID in the cluster status. Will be advertised to other devices as an optional default name.": "Visas i stället för enhetens ID. Skickas till andra enheter som namn på denna enhet.",
+    "Shown instead of Device ID in the cluster status. Will be updated to the name the device advertises if left empty.": "Visas i stället för enhetens ID. Sätts till namnet på den andra enheten vid första anslutning om det lämnas tomt.",
     "Shutdown": "Stäng av",
     "Shutdown Complete": "Avstängning klar",
-    "Simple File Versioning": "Enkel versionshantering",
+    "Simple File Versioning": "Enkel filversionshantering",
     "Single level wildcard (matches within a directory only)": "Jokertecken som representerar noll eller fler godtyckliga tecken i ett filnamn.",
     "Smallest First": "Minst först",
     "Source Code": "Källkod",
-    "Staggered File Versioning": "Versionshantering i intervall",
-    "Start Browser": "Starta browser",
+    "Staggered File Versioning": "Filversionshantering i intervall",
+    "Start Browser": "Starta webbläsare",
     "Statistics": "Statistik",
     "Stopped": "Stoppad",
     "Support": "Support",
-    "Sync Protocol Listen Addresses": "Address för inkommande anslutningar",
+    "Sync Protocol Listen Addresses": "Synkroniseringsprotokollets lyssningsadresser",
     "Syncing": "Synkroniserar",
     "Syncthing has been shut down.": "Syncthing har stängts.",
     "Syncthing includes the following software or portions thereof:": "Syncthing innehåller följande mjukvarupaket eller delar av dem:",
     "Syncthing is restarting.": "Syncthing startar om.",
     "Syncthing is upgrading.": "Syncthing uppgraderas.",
     "Syncthing seems to be down, or there is a problem with your Internet connection. Retrying…": "Syncthing verkar avstängd, eller finns det problem med din Internetanslutning. Försöker igen...",
-    "Syncthing seems to be experiencing a problem processing your request. Please refresh the page or restart Syncthing if the problem persists.": "Syncthing verkar ha drabbats av ett problem. Uppdatera sidan eller starta om Syncthing om problemet kvarstår.",
+    "Syncthing seems to be experiencing a problem processing your request. Please refresh the page or restart Syncthing if the problem persists.": "Syncthing verkar ha drabbats av ett problem med behandlingen av din begäran. Uppdatera sidan eller starta om Syncthing om problemet kvarstår.",
     "The Syncthing admin interface is configured to allow remote access without a password.": "Syncthing administratör gränssnittet är konfigurerat för att tillåta fjärrtillträde utan ett lösenord.",
     "The aggregated statistics are publicly available at the URL below.": "Den aggregerade statistiken är offentligt tillgängliga på webbadressen nedan.",
     "The aggregated statistics are publicly available at {%url%}.": "Sammanställd statistik finns publikt tillgänglig på {{url}}.",
     "The configuration has been saved but not activated. Syncthing must restart to activate the new configuration.": "Konfigurationen har sparats men inte aktiverats. Syncthing måste startas om för att aktivera den nya konfigurationen.",
-    "The device ID cannot be blank.": "Enhets-ID kan inte vara tomt.",
-    "The device ID to enter here can be found in the \"Actions > Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).": "Enhets-ID som behövs här kan du hitta i \"Redigera > Visa ID\"-dialogen på den andra enheten. Mellanrum och bindestreck är valfria (ignoreras).",
-    "The device ID to enter here can be found in the \"Edit > Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).": "Enhets-ID som behövs här kan du hitta i \"Redigera > Visa ID\"-dialogen på den andra enheten. Mellanrum och bindestreck är valfria (ignoreras).",
+    "The device ID cannot be blank.": "Enhetens ID kan inte vara tomt.",
+    "The device ID to enter here can be found in the \"Actions > Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).": "Enhetens ID som behövs här kan du hitta i \"Redigera > Visa ID\"-dialogen på den andra enheten. Mellanrum och bindestreck är valfria (ignoreras).",
+    "The device ID to enter here can be found in the \"Edit > Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).": "Enhetens ID som behövs här kan du hitta i \"Redigera > Visa ID\"-dialogen på den andra enheten. Mellanrum och bindestreck är valfria (ignoreras).",
     "The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.": "Den krypterade användarstatistiken skickas dagligen. Den används för att spåra vanliga plattformar, katalogstorlekar och versioner. Om datan som rapporteras ändras så kommer du att bli tillfrågad igen.",
-    "The entered device ID does not look valid. It should be a 52 or 56 character string consisting of letters and numbers, with spaces and dashes being optional.": "Det inmatade enhets-ID:t verkar inte korrekt. Det ska vara en 52 eller 56 teckens sträng bestående av siffror och bokstäver, eventuellt med mellanrum och bindestreck.",
+    "The entered device ID does not look valid. It should be a 52 or 56 character string consisting of letters and numbers, with spaces and dashes being optional.": "Det inmatade enhetens ID verkar inte korrekt. Det ska vara en 52 eller 56 teckens sträng bestående av siffror och bokstäver, eventuellt med mellanrum och bindestreck.",
     "The first command line parameter is the folder path and the second parameter is the relative path in the folder.": "Den första kommandoparametern är sökvägen till mappen och den andra parametern är den relativa sökvägen i mappen.",
-    "The folder ID cannot be blank.": "Ange ett enhets-ID.",
-    "The folder ID must be a short identifier (64 characters or less) consisting of letters, numbers and the dot (.), dash (-) and underscode (_) characters only.": "Katalog-ID:t måste vara en kort sträng (64 tecken eller mindre), bestående av endast bokstäver, siffror, punkt (.), bindestreck (-) och understreck (_).",
-    "The folder ID must be unique.": "Katalog-ID:t måste vara unikt.",
-    "The folder path cannot be blank.": "Ange en sökväg.",
+    "The folder ID cannot be blank.": "Katalogens ID får inte vara tomt.",
+    "The folder ID must be a short identifier (64 characters or less) consisting of letters, numbers and the dot (.), dash (-) and underscode (_) characters only.": "Katalogens ID måste vara en kort sträng (64 tecken eller mindre), bestående av endast bokstäver, siffror, punkt (.), bindestreck (-) och understreck (_).",
+    "The folder ID must be unique.": "Katalogens ID måste vara unikt.",
+    "The folder path cannot be blank.": "Katalogsökvägen får inte vara tom.",
     "The following intervals are used: for the first hour a version is kept every 30 seconds, for the first day a version is kept every hour, for the first 30 days a version is kept every day, until the maximum age a version is kept every week.": "De följande intervallen används: varje 30 sekunder under den första timmen; varje timme under den första dagen; varje dag för de första 30 dagarna; varje vecka tills den maximala åldersgränsen uppnås.",
-    "The following items could not be synchronized.": "Följande filer kunde inte synkroniseras.",
+    "The following items could not be synchronized.": "Följande objekt kunde inte synkroniseras.",
     "The maximum age must be a number and cannot be blank.": "Åldersgränsen måste vara ett tal och kan inte lämnas tomt.",
     "The maximum time to keep a version (in days, set to 0 to keep versions forever).": "Den längsta tiden att behålla en version (i dagar, sätt till 0 för att behålla versioner för evigt).",
     "The minimum free disk space percentage must be a non-negative number between 0 and 100 (inclusive).": "Minimum ledigt diskutrymme i procent måste vara en icke negativ siffra mellan 0 och 100 (inklusive).",
     "The number of days must be a number and cannot be blank.": "Antalet dagar måste vara en siffra och får inte vara tomt.",
-    "The number of days to keep files in the trash can. Zero means forever.": "Antal dagar som filer ligger kvar i papperskorgen. Noll betyder för alltid.",
+    "The number of days to keep files in the trash can. Zero means forever.": "Antalet dagar som filer ligger kvar i papperskorgen. Noll betyder för alltid.",
     "The number of old versions to keep, per file.": "Antalet gamla versioner som ska behållas, per fil.",
     "The number of versions must be a number and cannot be blank.": "Antalet versioner måste vara ett nummer och kan inte lämnas tomt.",
-    "The path cannot be blank.": "Ange en sökväg",
+    "The path cannot be blank.": "Sökvägen kan inte vara tom.",
     "The rate limit must be a non-negative number (0: no limit)": "Frekvensgränsen måste vara ett icke-negativt tal (0: ingen gräns)",
     "The rescan interval must be a non-negative number of seconds.": "Förnyelseintervallet måste vara ett positivt antal sekunder",
     "They are retried automatically and will be synced when the error is resolved.": "De omprövas automatiskt och kommer att synkroniseras när felet är löst.",
     "This Device": "Denna enhet",
     "This can easily give hackers access to read and change any files on your computer.": "Detta kan lätt ge hackare tillgång till att läsa och ändra några filer på datorn.",
     "This is a major version upgrade.": "Det här är en stor uppgradering.",
-    "Trash Can File Versioning": "Versionshantering på filer i papperskorgen",
-    "Unknown": "Okänt",
+    "Trash Can File Versioning": "Papperskorgs filversionshantering",
+    "Unknown": "Okänd",
     "Unshared": "Inte delad",
     "Unused": "Oanvänd",
-    "Up to Date": "Helt uppdaterad",
+    "Up to Date": "Uppdaterad",
     "Updated": "Uppdaterad",
     "Upgrade": "Uppgradering",
     "Upgrade To {%version%}": "Uppgradera till {{version}}",
@@ -235,16 +235,16 @@
     "Uptime": "Tid sedan start",
     "Use HTTPS for GUI": "Använd HTTPS för GUI",
     "Version": "Version",
-    "Versions Path": "Katalog för versioner",
+    "Versions Path": "Sökväg för versioner",
     "Versions are automatically deleted if they are older than the maximum age or exceed the number of files allowed in an interval.": "Versioner tas bort automatiskt när de är äldre än den maximala åldersgränsen eller överstiger frekvensen i sitt interval.",
     "Warning, this path is a subdirectory of an existing folder \"{%otherFolder%}\".": "Varning, denna sökväg är en underkatalog till en befintlig katalog \"{{otherFolder}}\".",
     "When adding a new device, keep in mind that this device must be added on the other side too.": "När du lägger till en ny enhet, kom ihåg att den här enheten måste läggas till på den andra enheten också.",
-    "When adding a new folder, keep in mind that the Folder ID is used to tie folders together between devices. They are case sensitive and must match exactly between all devices.": "När du lägger till ny katalog, tänk på att katalog-ID:t knyter ihop katalogen mellan olika noder. De måste vara exakt desamma mellan noder och stora eller små bokstäver har betydelse.",
+    "When adding a new folder, keep in mind that the Folder ID is used to tie folders together between devices. They are case sensitive and must match exactly between all devices.": "När du lägger till ny katalog, tänk på att katalogens ID knyter ihop katalogen mellan olika noder. De måste vara exakt desamma mellan noder och stora eller små bokstäver har betydelse.",
     "Yes": "Ja",
     "You must keep at least one version.": "Du måste behålla åtminstone en version.",
     "days": "dagar",
     "full documentation": "fullständig dokumentation",
-    "items": "poster",
+    "items": "objekt",
     "{%device%} wants to share folder \"{%folder%}\".": "{{device}} vill dela katalogen \"{{folder}}\".",
     "{%device%} wants to share folder \"{%folderLabel%}\" ({%folder%}).": "{{device}} vill dela katalogen \"{{folderLabel}}\" ({{folder}}).",
     "{%device%} wants to share folder \"{%folderlabel%}\" ({%folder%}).": "{{device}} vill dela katalogen \"{{folderlabel}}\" ({{folder}})."

gui/default/syncthing/core/syncthingController.js

@@ -1453,7 +1453,7 @@ angular.module('syncthing.core')
                 }
             }
 
-            folders.sort();
+            folders.sort(folderCompare);
             return folders;
         };
 

gui/default/syncthing/usagereport/usageReportPreviewModalView.html

@@ -3,9 +3,8 @@
     <p translate>
       The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.
     </p>
-    <p translate translate-value-url="<a href=&quot;https://data.syncthing.net&quot; target=&quot;_blank&quot;>https://data.syncthing.net</a>">
-      The aggregated statistics are publicly available at {%url%}.
-    </p>
+    <p translate>The aggregated statistics are publicly available at the URL below.</p>
+    <p><a href="https://data.syncthing.net/" target="_blank">https://data.syncthing.net/</a></p>
     <form>
       <textarea class="form-control" rows="20">{{reportData | json}}</textarea>
     </form>

lib/config/commit_test.go

@@ -11,12 +11,18 @@ import (
 	"testing"
 )
 
-type requiresRestart struct{}
+type requiresRestart struct {
+	committed chan struct{}
+}
 
 func (requiresRestart) VerifyConfiguration(_, _ Configuration) error {
 	return nil
 }
-func (requiresRestart) CommitConfiguration(_, _ Configuration) bool {
+func (c requiresRestart) CommitConfiguration(_, _ Configuration) bool {
+	select {
+	case c.committed <- struct{}{}:
+	default:
+	}
 	return false
 }
 func (requiresRestart) String() string {
@@ -28,7 +34,7 @@ type validationError struct{}
 func (validationError) VerifyConfiguration(_, _ Configuration) error {
 	return errors.New("some error")
 }
-func (validationError) CommitConfiguration(_, _ Configuration) bool {
+func (c validationError) CommitConfiguration(_, _ Configuration) bool {
 	return true
 }
 func (validationError) String() string {
@@ -44,11 +50,11 @@ func TestReplaceCommit(t *testing.T) {
 	// Replace config. We should get back a clean response and the config
 	// should change.
 
-	resp := w.Replace(Configuration{Version: 1})
-	if resp.ValidationError != nil {
-		t.Fatal("Should not have a validation error")
+	err := w.Replace(Configuration{Version: 1})
+	if err != nil {
+		t.Fatal("Should not have a validation error:", err)
 	}
-	if resp.RequiresRestart {
+	if w.RequiresRestart() {
 		t.Fatal("Should not require restart")
 	}
 	if w.Raw().Version != 1 {
@@ -58,13 +64,16 @@ func TestReplaceCommit(t *testing.T) {
 	// Now with a subscriber requiring restart. We should get a clean response
 	// but with the restart flag set, and the config should change.
 
-	w.Subscribe(requiresRestart{})
+	sub0 := requiresRestart{committed: make(chan struct{}, 1)}
+	w.Subscribe(sub0)
 
-	resp = w.Replace(Configuration{Version: 2})
-	if resp.ValidationError != nil {
-		t.Fatal("Should not have a validation error")
+	err = w.Replace(Configuration{Version: 2})
+	if err != nil {
+		t.Fatal("Should not have a validation error:", err)
 	}
-	if !resp.RequiresRestart {
+
+	<-sub0.committed
+	if !w.RequiresRestart() {
 		t.Fatal("Should require restart")
 	}
 	if w.Raw().Version != 2 {
@@ -76,12 +85,12 @@ func TestReplaceCommit(t *testing.T) {
 
 	w.Subscribe(validationError{})
 
-	resp = w.Replace(Configuration{Version: 3})
-	if resp.ValidationError == nil {
+	err = w.Replace(Configuration{Version: 3})
+	if err == nil {
 		t.Fatal("Should have a validation error")
 	}
-	if resp.RequiresRestart {
-		t.Fatal("Should not require restart")
+	if !w.RequiresRestart() {
+		t.Fatal("Should still require restart")
 	}
 	if w.Raw().Version != 2 {
 		t.Fatal("Config should not have changed")

lib/config/config.go

@@ -105,7 +105,9 @@ func ReadJSON(r io.Reader, myID protocol.DeviceID) (Configuration, error) {
 		return Configuration{}, err
 	}
 
-	err = json.Unmarshal(bs, &cfg)
+	if err := json.Unmarshal(bs, &cfg); err != nil {
+		return Configuration{}, err
+	}
 	cfg.OriginalVersion = cfg.Version
 
 	if err := cfg.prepare(myID); err != nil {

lib/config/config_test.go

@@ -549,6 +549,9 @@ func TestPullOrder(t *testing.T) {
 	t.Logf("%s", buf.Bytes())
 
 	cfg, err = ReadXML(buf, device1)
+	if err != nil {
+		t.Fatal(err)
+	}
 	wrapper = Wrap("testdata/pullorder.xml", cfg)
 	folders = wrapper.Folders()
 

lib/config/wrapper.go

@@ -8,6 +8,7 @@ package config
 
 import (
 	"os"
+	"sync/atomic"
 
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/osutil"
@@ -41,11 +42,6 @@ type Committer interface {
 	String() string
 }
 
-type CommitResponse struct {
-	ValidationError error
-	RequiresRestart bool
-}
-
 // A wrapper around a Configuration that manages loads, saves and published
 // notifications of changes to registered Handlers
 
@@ -58,6 +54,8 @@ type Wrapper struct {
 	replaces  chan Configuration
 	subs      []Committer
 	mut       sync.Mutex
+
+	requiresRestart uint32 // an atomic bool
 }
 
 // Wrap wraps an existing Configuration structure and ties it to a file on
@@ -128,32 +126,21 @@ func (w *Wrapper) Raw() Configuration {
 }
 
 // Replace swaps the current configuration object for the given one.
-func (w *Wrapper) Replace(cfg Configuration) CommitResponse {
+func (w *Wrapper) Replace(cfg Configuration) error {
 	w.mut.Lock()
 	defer w.mut.Unlock()
+
 	return w.replaceLocked(cfg)
 }
 
-func (w *Wrapper) replaceLocked(to Configuration) CommitResponse {
+func (w *Wrapper) replaceLocked(to Configuration) error {
 	from := w.cfg
 
 	for _, sub := range w.subs {
 		l.Debugln(sub, "verifying configuration")
 		if err := sub.VerifyConfiguration(from, to); err != nil {
 			l.Debugln(sub, "rejected config:", err)
-			return CommitResponse{
-				ValidationError: err,
-			}
-		}
-	}
-
-	allOk := true
-	for _, sub := range w.subs {
-		l.Debugln(sub, "committing configuration")
-		ok := sub.CommitConfiguration(from, to)
-		if !ok {
-			l.Debugln(sub, "requires restart")
-			allOk = false
+			return err
 		}
 	}
 
@@ -161,8 +148,22 @@ func (w *Wrapper) replaceLocked(to Configuration) CommitResponse {
 	w.deviceMap = nil
 	w.folderMap = nil
 
-	return CommitResponse{
-		RequiresRestart: !allOk,
+	w.notifyListeners(from, to)
+
+	return nil
+}
+
+func (w *Wrapper) notifyListeners(from, to Configuration) {
+	for _, sub := range w.subs {
+		go w.notifyListener(sub, from, to)
+	}
+}
+
+func (w *Wrapper) notifyListener(sub Committer, from, to Configuration) {
+	l.Debugln(sub, "committing configuration")
+	if !sub.CommitConfiguration(from, to) {
+		l.Debugln(sub, "requires restart")
+		w.setRequiresRestart()
 	}
 }
 
@@ -182,7 +183,7 @@ func (w *Wrapper) Devices() map[protocol.DeviceID]DeviceConfiguration {
 
 // SetDevice adds a new device to the configuration, or overwrites an existing
 // device with the same ID.
-func (w *Wrapper) SetDevice(dev DeviceConfiguration) CommitResponse {
+func (w *Wrapper) SetDevice(dev DeviceConfiguration) error {
 	w.mut.Lock()
 	defer w.mut.Unlock()
 
@@ -218,7 +219,7 @@ func (w *Wrapper) Folders() map[string]FolderConfiguration {
 
 // SetFolder adds a new folder to the configuration, or overwrites an existing
 // folder with the same ID.
-func (w *Wrapper) SetFolder(fld FolderConfiguration) CommitResponse {
+func (w *Wrapper) SetFolder(fld FolderConfiguration) error {
 	w.mut.Lock()
 	defer w.mut.Unlock()
 
@@ -246,7 +247,7 @@ func (w *Wrapper) Options() OptionsConfiguration {
 }
 
 // SetOptions replaces the current options configuration object.
-func (w *Wrapper) SetOptions(opts OptionsConfiguration) CommitResponse {
+func (w *Wrapper) SetOptions(opts OptionsConfiguration) error {
 	w.mut.Lock()
 	defer w.mut.Unlock()
 	newCfg := w.cfg.Copy()
@@ -262,7 +263,7 @@ func (w *Wrapper) GUI() GUIConfiguration {
 }
 
 // SetGUI replaces the current GUI configuration object.
-func (w *Wrapper) SetGUI(gui GUIConfiguration) CommitResponse {
+func (w *Wrapper) SetGUI(gui GUIConfiguration) error {
 	w.mut.Lock()
 	defer w.mut.Unlock()
 	newCfg := w.cfg.Copy()
@@ -332,3 +333,11 @@ func (w *Wrapper) ListenAddresses() []string {
 	}
 	return util.UniqueStrings(addresses)
 }
+
+func (w *Wrapper) RequiresRestart() bool {
+	return atomic.LoadUint32(&w.requiresRestart) != 0
+}
+
+func (w *Wrapper) setRequiresRestart() {
+	atomic.StoreUint32(&w.requiresRestart, 1)
+}

lib/connections/service.go

@@ -162,8 +162,17 @@ next:
 		if err != nil {
 			if protocol.IsVersionMismatch(err) {
 				// The error will be a relatively user friendly description
-				// of what's wrong with the version compatibility
-				msg := fmt.Sprintf("Connecting to %s (%s): %s", remoteID, c.RemoteAddr(), err)
+				// of what's wrong with the version compatibility. By
+				// default identify the other side by device ID and IP.
+				remote := fmt.Sprintf("%v (%v)", remoteID, c.RemoteAddr())
+				if hello.DeviceName != "" {
+					// If the name was set in the hello return, use that to
+					// give the user more info about which device is the
+					// affected one. It probably says more than the remote
+					// IP.
+					remote = fmt.Sprintf("%q (%s %s, %v)", hello.DeviceName, hello.ClientName, hello.ClientVersion, remoteID)
+				}
+				msg := fmt.Sprintf("Connecting to %s: %s", remote, err)
 				warningFor(remoteID, msg)
 			} else {
 				// It's something else - connection reset or whatever

lib/connections/structs.go

@@ -70,7 +70,7 @@ type Model interface {
 	ConnectedTo(remoteID protocol.DeviceID) bool
 	IsPaused(remoteID protocol.DeviceID) bool
 	OnHello(protocol.DeviceID, net.Addr, protocol.HelloResult)
-	GetHello(protocol.DeviceID) protocol.Version13HelloMessage
+	GetHello(protocol.DeviceID) protocol.HelloIntf
 }
 
 // serviceFunc wraps a function to create a suture.Service without stop

lib/db/blockmap_test.go

@@ -73,7 +73,7 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
 
 	m := NewBlockMap(db, db.folderIdx.ID([]byte("folder1")))
 
-	f3.Flags |= protocol.FlagDirectory
+	f3.Type = protocol.FileInfoTypeDirectory
 
 	err := m.Add([]protocol.FileInfo{f1, f2, f3})
 	if err != nil {
@@ -99,9 +99,11 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
 		return true
 	})
 
-	f3.Flags = f1.Flags
-	f1.Flags |= protocol.FlagDeleted
-	f2.Flags |= protocol.FlagInvalid
+	f3.Permissions = f1.Permissions
+	f3.Deleted = f1.Deleted
+	f3.Invalid = f1.Invalid
+	f1.Deleted = true
+	f2.Invalid = true
 
 	// Should remove
 	err = m.Update([]protocol.FileInfo{f1, f2, f3})
@@ -145,9 +147,15 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
 		t.Fatal("db not empty")
 	}
 
-	f1.Flags = 0
-	f2.Flags = 0
-	f3.Flags = 0
+	f1.Deleted = false
+	f1.Invalid = false
+	f1.Permissions = 0
+	f2.Deleted = false
+	f2.Invalid = false
+	f2.Permissions = 0
+	f3.Deleted = false
+	f3.Invalid = false
+	f3.Permissions = 0
 }
 
 func TestBlockFinderLookup(t *testing.T) {
@@ -187,7 +195,7 @@ func TestBlockFinderLookup(t *testing.T) {
 		t.Fatal("Incorrect count", counter)
 	}
 
-	f1.Flags |= protocol.FlagDeleted
+	f1.Deleted = true
 
 	err = m1.Update([]protocol.FileInfo{f1})
 	if err != nil {
@@ -212,7 +220,7 @@ func TestBlockFinderLookup(t *testing.T) {
 		t.Fatal("Incorrect count")
 	}
 
-	f1.Flags = 0
+	f1.Deleted = false
 }
 
 func TestBlockFinderFix(t *testing.T) {

lib/db/leveldb.go

@@ -4,9 +4,6 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at http://mozilla.org/MPL/2.0/.
 
-//go:generate -command genxdr go run ../../vendor/github.com/calmh/xdr/cmd/genxdr/main.go
-//go:generate genxdr -o leveldb_xdr.go leveldb.go
-
 package db
 
 import (
@@ -14,27 +11,10 @@ import (
 	"fmt"
 
 	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/opt"
 )
 
-var (
-	clockTick int64
-	clockMut  = sync.NewMutex()
-)
-
-func clock(v int64) int64 {
-	clockMut.Lock()
-	defer clockMut.Unlock()
-	if v > clockTick {
-		clockTick = v + 1
-	} else {
-		clockTick++
-	}
-	return clockTick
-}
-
 const (
 	KeyTypeDevice = iota
 	KeyTypeGlobal
@@ -44,27 +24,19 @@ const (
 	KeyTypeVirtualMtime
 	KeyTypeFolderIdx
 	KeyTypeDeviceIdx
+	KeyTypeIndexID
 )
 
-type fileVersion struct {
-	version protocol.Vector
-	device  []byte
-}
-
-type VersionList struct {
-	versions []fileVersion
-}
-
 func (l VersionList) String() string {
 	var b bytes.Buffer
 	var id protocol.DeviceID
 	b.WriteString("{")
-	for i, v := range l.versions {
+	for i, v := range l.Versions {
 		if i > 0 {
 			b.WriteString(", ")
 		}
-		copy(id[:], v.device)
-		fmt.Fprintf(&b, "{%d, %v}", v.version, id)
+		copy(id[:], v.Device)
+		fmt.Fprintf(&b, "{%d, %v}", v.Version, id)
 	}
 	b.WriteString("}")
 	return b.String()
@@ -101,7 +73,7 @@ func getFile(db dbReader, key []byte) (protocol.FileInfo, bool) {
 	}
 
 	var f protocol.FileInfo
-	err = f.UnmarshalXDR(bs)
+	err = f.Unmarshal(bs)
 	if err != nil {
 		panic(err)
 	}

lib/db/leveldb_convert.go

@@ -1,114 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package db
-
-import (
-	"bytes"
-
-	"github.com/syndtr/goleveldb/leveldb"
-)
-
-// convertKeyFormat converts from the v0.12 to the v0.13 database format, to
-// avoid having to do rescan. The change is in the key format for folder
-// labels, so we basically just iterate over the database rewriting keys as
-// necessary.
-func convertKeyFormat(from, to *leveldb.DB) error {
-	l.Infoln("Converting database key format")
-	blocks, files, globals, unchanged := 0, 0, 0, 0
-
-	dbi := newDBInstance(to)
-	i := from.NewIterator(nil, nil)
-	for i.Next() {
-		key := i.Key()
-		switch key[0] {
-		case KeyTypeBlock:
-			folder, file := oldFromBlockKey(key)
-			folderIdx := dbi.folderIdx.ID([]byte(folder))
-			hash := key[1+64:]
-			newKey := blockKeyInto(nil, hash, folderIdx, file)
-			if err := to.Put(newKey, i.Value(), nil); err != nil {
-				return err
-			}
-			blocks++
-
-		case KeyTypeDevice:
-			newKey := dbi.deviceKey(oldDeviceKeyFolder(key), oldDeviceKeyDevice(key), oldDeviceKeyName(key))
-			if err := to.Put(newKey, i.Value(), nil); err != nil {
-				return err
-			}
-			files++
-
-		case KeyTypeGlobal:
-			newKey := dbi.globalKey(oldGlobalKeyFolder(key), oldGlobalKeyName(key))
-			if err := to.Put(newKey, i.Value(), nil); err != nil {
-				return err
-			}
-			globals++
-
-		case KeyTypeVirtualMtime:
-			// Cannot be converted, we drop it instead :(
-
-		default:
-			if err := to.Put(key, i.Value(), nil); err != nil {
-				return err
-			}
-			unchanged++
-		}
-	}
-
-	l.Infof("Converted %d blocks, %d files, %d globals (%d unchanged).", blocks, files, globals, unchanged)
-
-	return nil
-}
-
-func oldDeviceKeyFolder(key []byte) []byte {
-	folder := key[1 : 1+64]
-	izero := bytes.IndexByte(folder, 0)
-	if izero < 0 {
-		return folder
-	}
-	return folder[:izero]
-}
-
-func oldDeviceKeyDevice(key []byte) []byte {
-	return key[1+64 : 1+64+32]
-}
-
-func oldDeviceKeyName(key []byte) []byte {
-	return key[1+64+32:]
-}
-
-func oldGlobalKeyName(key []byte) []byte {
-	return key[1+64:]
-}
-
-func oldGlobalKeyFolder(key []byte) []byte {
-	folder := key[1 : 1+64]
-	izero := bytes.IndexByte(folder, 0)
-	if izero < 0 {
-		return folder
-	}
-	return folder[:izero]
-}
-
-func oldFromBlockKey(data []byte) (string, string) {
-	if len(data) < 1+64+32+1 {
-		panic("Incorrect key length")
-	}
-	if data[0] != KeyTypeBlock {
-		panic("Incorrect key type")
-	}
-
-	file := string(data[1+64+32:])
-
-	slice := data[1 : 1+64]
-	izero := bytes.IndexByte(slice, 0)
-	if izero > -1 {
-		return string(slice[:izero]), file
-	}
-	return string(slice), file
-}

lib/db/leveldb_convert_test.go

@@ -1,136 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package db
-
-import (
-	"archive/zip"
-	"io"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/syndtr/goleveldb/leveldb"
-)
-
-func TestLabelConversion(t *testing.T) {
-	os.RemoveAll("testdata/oldformat.db")
-	defer os.RemoveAll("testdata/oldformat.db")
-	os.RemoveAll("testdata/newformat.db")
-	defer os.RemoveAll("testdata/newformat.db")
-
-	if err := unzip("testdata/oldformat.db.zip", "testdata"); err != nil {
-		t.Fatal(err)
-	}
-
-	odb, err := leveldb.OpenFile("testdata/oldformat.db", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	ldb, err := leveldb.OpenFile("testdata/newformat.db", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err = convertKeyFormat(odb, ldb); err != nil {
-		t.Fatal(err)
-	}
-	ldb.Close()
-	odb.Close()
-
-	inst, err := Open("testdata/newformat.db")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	fs := NewFileSet("default", inst)
-	files, deleted, _ := fs.GlobalSize()
-	if files+deleted != 953 {
-		// Expected number of global entries determined by
-		// ../../bin/stindex testdata/oldformat.db/ | grep global | grep -c default
-		t.Errorf("Conversion error, global list differs (%d != 953)", files+deleted)
-	}
-
-	files, deleted, _ = fs.LocalSize()
-	if files+deleted != 953 {
-		t.Errorf("Conversion error, device list differs (%d != 953)", files+deleted)
-	}
-
-	f := NewBlockFinder(inst)
-	// [block] F:"default" H:1c25dea9003cc16216e2a22900be1ec1cc5aaf270442904e2f9812c314e929d8 N:"f/f2/f25f1b3e6e029231b933531b2138796d" I:3
-	h := []byte{0x1c, 0x25, 0xde, 0xa9, 0x00, 0x3c, 0xc1, 0x62, 0x16, 0xe2, 0xa2, 0x29, 0x00, 0xbe, 0x1e, 0xc1, 0xcc, 0x5a, 0xaf, 0x27, 0x04, 0x42, 0x90, 0x4e, 0x2f, 0x98, 0x12, 0xc3, 0x14, 0xe9, 0x29, 0xd8}
-	found := 0
-	f.Iterate([]string{"default"}, h, func(folder, file string, idx int32) bool {
-		if folder == "default" && file == filepath.FromSlash("f/f2/f25f1b3e6e029231b933531b2138796d") && idx == 3 {
-			found++
-		}
-		return true
-	})
-	if found != 1 {
-		t.Errorf("Found %d blocks instead of expected 1", found)
-	}
-
-	inst.Close()
-}
-
-func unzip(src, dest string) error {
-	r, err := zip.OpenReader(src)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err := r.Close(); err != nil {
-			panic(err)
-		}
-	}()
-
-	os.MkdirAll(dest, 0755)
-
-	// Closure to address file descriptors issue with all the deferred .Close() methods
-	extractAndWriteFile := func(f *zip.File) error {
-		rc, err := f.Open()
-		if err != nil {
-			return err
-		}
-		defer func() {
-			if err := rc.Close(); err != nil {
-				panic(err)
-			}
-		}()
-
-		path := filepath.Join(dest, f.Name)
-
-		if f.FileInfo().IsDir() {
-			os.MkdirAll(path, f.Mode())
-		} else {
-			f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
-			if err != nil {
-				return err
-			}
-			defer func() {
-				if err := f.Close(); err != nil {
-					panic(err)
-				}
-			}()
-
-			_, err = io.Copy(f, rc)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
-	for _, f := range r.File {
-		err := extractAndWriteFile(f)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

lib/db/leveldb_dbinstance.go

@@ -10,12 +10,10 @@ import (
 	"bytes"
 	"encoding/binary"
 	"os"
-	"path/filepath"
 	"sort"
 	"strings"
 	"sync/atomic"
 
-	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syndtr/goleveldb/leveldb"
@@ -26,7 +24,7 @@ import (
 	"github.com/syndtr/goleveldb/leveldb/util"
 )
 
-type deletionHandler func(t readWriteTransaction, folder, device, name []byte, dbi iterator.Iterator) int64
+type deletionHandler func(t readWriteTransaction, folder, device, name []byte, dbi iterator.Iterator)
 
 type Instance struct {
 	committed int64 // this must be the first attribute in the struct to ensure 64 bit alignment on 32 bit plaforms
@@ -48,16 +46,6 @@ func Open(file string) (*Instance, error) {
 		WriteBuffer:            4 << 20,
 	}
 
-	if _, err := os.Stat(file); os.IsNotExist(err) {
-		// The file we are looking to open does not exist. This may be the
-		// first launch so we should look for an old version and try to
-		// convert it.
-		if err := checkConvertDatabase(file); err != nil {
-			l.Infoln("Converting old database:", err)
-			l.Infoln("Will rescan from scratch.")
-		}
-	}
-
 	db, err := leveldb.OpenFile(file, opts)
 	if leveldbIsCorrupted(err) {
 		db, err = leveldb.RecoverFile(file, opts)
@@ -98,7 +86,7 @@ func (db *Instance) Committed() int64 {
 	return atomic.LoadInt64(&db.committed)
 }
 
-func (db *Instance) genericReplace(folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker, deleteFn deletionHandler) int64 {
+func (db *Instance) genericReplace(folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker, deleteFn deletionHandler) {
 	sort.Sort(fileList(fs)) // sort list on name, same as in the database
 
 	t := db.newReadWriteTransaction()
@@ -109,7 +97,6 @@ func (db *Instance) genericReplace(folder, device []byte, fs []protocol.FileInfo
 
 	moreDb := dbi.Next()
 	fsi := 0
-	var maxLocalVer int64
 
 	isLocalDevice := bytes.Equal(device, protocol.LocalDeviceID[:])
 	for {
@@ -136,9 +123,7 @@ func (db *Instance) genericReplace(folder, device []byte, fs []protocol.FileInfo
 		case moreFs && (!moreDb || cmp == -1):
 			l.Debugln("generic replace; missing - insert")
 			// Database is missing this file. Insert it.
-			if lv := t.insertFile(folder, device, fs[fsi]); lv > maxLocalVer {
-				maxLocalVer = lv
-			}
+			t.insertFile(folder, device, fs[fsi])
 			if isLocalDevice {
 				localSize.addFile(fs[fsi])
 			}
@@ -151,16 +136,14 @@ func (db *Instance) genericReplace(folder, device []byte, fs []protocol.FileInfo
 
 		case moreFs && moreDb && cmp == 0:
 			// File exists on both sides - compare versions. We might get an
-			// update with the same version and different flags if a device has
-			// marked a file as invalid, so handle that too.
+			// update with the same version if a device has marked a file as
+			// invalid, so handle that too.
 			l.Debugln("generic replace; exists - compare")
 			var ef FileInfoTruncated
-			ef.UnmarshalXDR(dbi.Value())
-			if !fs[fsi].Version.Equal(ef.Version) || fs[fsi].Flags != ef.Flags {
+			ef.Unmarshal(dbi.Value())
+			if !fs[fsi].Version.Equal(ef.Version) || fs[fsi].Invalid != ef.Invalid {
 				l.Debugln("generic replace; differs - insert")
-				if lv := t.insertFile(folder, device, fs[fsi]); lv > maxLocalVer {
-					maxLocalVer = lv
-				}
+				t.insertFile(folder, device, fs[fsi])
 				if isLocalDevice {
 					localSize.removeFile(ef)
 					localSize.addFile(fs[fsi])
@@ -179,9 +162,7 @@ func (db *Instance) genericReplace(folder, device []byte, fs []protocol.FileInfo
 
 		case moreDb && (!moreFs || cmp == 1):
 			l.Debugln("generic replace; exists - remove")
-			if lv := deleteFn(t, folder, device, oldName, dbi); lv > maxLocalVer {
-				maxLocalVer = lv
-			}
+			deleteFn(t, folder, device, oldName, dbi)
 			moreDb = dbi.Next()
 		}
 
@@ -189,26 +170,21 @@ func (db *Instance) genericReplace(folder, device []byte, fs []protocol.FileInfo
 		// growing too large and thus allocating unnecessarily much memory.
 		t.checkFlush()
 	}
-
-	return maxLocalVer
 }
 
-func (db *Instance) replace(folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker) int64 {
-	// TODO: Return the remaining maxLocalVer?
-	return db.genericReplace(folder, device, fs, localSize, globalSize, func(t readWriteTransaction, folder, device, name []byte, dbi iterator.Iterator) int64 {
+func (db *Instance) replace(folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker) {
+	db.genericReplace(folder, device, fs, localSize, globalSize, func(t readWriteTransaction, folder, device, name []byte, dbi iterator.Iterator) {
 		// Database has a file that we are missing. Remove it.
 		l.Debugf("delete; folder=%q device=%v name=%q", folder, protocol.DeviceIDFromBytes(device), name)
 		t.removeFromGlobal(folder, device, name, globalSize)
 		t.Delete(dbi.Key())
-		return 0
 	})
 }
 
-func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker) int64 {
+func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, localSize, globalSize *sizeTracker) {
 	t := db.newReadWriteTransaction()
 	defer t.close()
 
-	var maxLocalVer int64
 	var fk []byte
 	isLocalDevice := bytes.Equal(device, protocol.LocalDeviceID[:])
 	for _, f := range fs {
@@ -220,9 +196,7 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, l
 				localSize.addFile(f)
 			}
 
-			if lv := t.insertFile(folder, device, f); lv > maxLocalVer {
-				maxLocalVer = lv
-			}
+			t.insertFile(folder, device, f)
 			if f.IsInvalid() {
 				t.removeFromGlobal(folder, device, name, globalSize)
 			} else {
@@ -232,21 +206,18 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, l
 		}
 
 		var ef FileInfoTruncated
-		err = ef.UnmarshalXDR(bs)
+		err = ef.Unmarshal(bs)
 		if err != nil {
 			panic(err)
 		}
-		// Flags might change without the version being bumped when we set the
-		// invalid flag on an existing file.
-		if !ef.Version.Equal(f.Version) || ef.Flags != f.Flags {
+		// The Invalid flag might change without the version being bumped.
+		if !ef.Version.Equal(f.Version) || ef.Invalid != f.Invalid {
 			if isLocalDevice {
 				localSize.removeFile(ef)
 				localSize.addFile(f)
 			}
 
-			if lv := t.insertFile(folder, device, f); lv > maxLocalVer {
-				maxLocalVer = lv
-			}
+			t.insertFile(folder, device, f)
 			if f.IsInvalid() {
 				t.removeFromGlobal(folder, device, name, globalSize)
 			} else {
@@ -258,8 +229,6 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, l
 		// growing too large and thus allocating unnecessarily much memory.
 		t.checkFlush()
 	}
-
-	return maxLocalVer
 }
 
 func (db *Instance) withHave(folder, device, prefix []byte, truncate bool, fn Iterator) {
@@ -308,7 +277,7 @@ func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte,
 		// struct, which in turn references the buffer it was unmarshalled
 		// from. dbi.Value() just returns an internal slice that it reuses, so
 		// we need to copy it.
-		err := f.UnmarshalXDR(append([]byte{}, dbi.Value()...))
+		err := f.Unmarshal(append([]byte{}, dbi.Value()...))
 		if err != nil {
 			panic(err)
 		}
@@ -347,16 +316,16 @@ func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, boo
 	}
 
 	var vl VersionList
-	err = vl.UnmarshalXDR(bs)
+	err = vl.Unmarshal(bs)
 	if err != nil {
 		panic(err)
 	}
-	if len(vl.versions) == 0 {
+	if len(vl.Versions) == 0 {
 		l.Debugln(k)
 		panic("no versions?")
 	}
 
-	k = db.deviceKey(folder, vl.versions[0].device, file)
+	k = db.deviceKey(folder, vl.Versions[0].Device, file)
 	bs, err = t.Get(k, nil)
 	if err != nil {
 		panic(err)
@@ -384,11 +353,11 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
 	var fk []byte
 	for dbi.Next() {
 		var vl VersionList
-		err := vl.UnmarshalXDR(dbi.Value())
+		err := vl.Unmarshal(dbi.Value())
 		if err != nil {
 			panic(err)
 		}
-		if len(vl.versions) == 0 {
+		if len(vl.Versions) == 0 {
 			l.Debugln(dbi.Key())
 			panic("no versions?")
 		}
@@ -398,13 +367,13 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
 			return
 		}
 
-		fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.versions[0].device, name)
+		fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.Versions[0].Device, name)
 		bs, err := t.Get(fk, nil)
 		if err != nil {
 			l.Debugf("folder: %q (%x)", folder, folder)
 			l.Debugf("key: %q (%x)", dbi.Key(), dbi.Key())
 			l.Debugf("vl: %v", vl)
-			l.Debugf("vl.versions[0].device: %x", vl.versions[0].device)
+			l.Debugf("vl.Versions[0].Device: %x", vl.Versions[0].Device)
 			l.Debugf("name: %q (%x)", name, name)
 			l.Debugf("fk: %q", fk)
 			l.Debugf("fk: %x %x %x",
@@ -436,17 +405,17 @@ func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
 	}
 
 	var vl VersionList
-	err = vl.UnmarshalXDR(bs)
+	err = vl.Unmarshal(bs)
 	if err != nil {
 		panic(err)
 	}
 
 	var devices []protocol.DeviceID
-	for _, v := range vl.versions {
-		if !v.version.Equal(vl.versions[0].version) {
+	for _, v := range vl.Versions {
+		if !v.Version.Equal(vl.Versions[0].Version) {
 			break
 		}
-		n := protocol.DeviceIDFromBytes(v.device)
+		n := protocol.DeviceIDFromBytes(v.Device)
 		devices = append(devices, n)
 	}
 
@@ -464,11 +433,11 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
 nextFile:
 	for dbi.Next() {
 		var vl VersionList
-		err := vl.UnmarshalXDR(dbi.Value())
+		err := vl.Unmarshal(dbi.Value())
 		if err != nil {
 			panic(err)
 		}
-		if len(vl.versions) == 0 {
+		if len(vl.Versions) == 0 {
 			l.Debugln(dbi.Key())
 			panic("no versions?")
 		}
@@ -476,29 +445,29 @@ nextFile:
 		have := false // If we have the file, any version
 		need := false // If we have a lower version of the file
 		var haveVersion protocol.Vector
-		for _, v := range vl.versions {
-			if bytes.Equal(v.device, device) {
+		for _, v := range vl.Versions {
+			if bytes.Equal(v.Device, device) {
 				have = true
-				haveVersion = v.version
+				haveVersion = v.Version
 				// XXX: This marks Concurrent (i.e. conflicting) changes as
 				// needs. Maybe we should do that, but it needs special
 				// handling in the puller.
-				need = !v.version.GreaterEqual(vl.versions[0].version)
+				need = !v.Version.GreaterEqual(vl.Versions[0].Version)
 				break
 			}
 		}
 
 		if need || !have {
 			name := db.globalKeyName(dbi.Key())
-			needVersion := vl.versions[0].version
+			needVersion := vl.Versions[0].Version
 
 		nextVersion:
-			for i := range vl.versions {
-				if !vl.versions[i].version.Equal(needVersion) {
+			for i := range vl.Versions {
+				if !vl.Versions[i].Version.Equal(needVersion) {
 					// We haven't found a valid copy of the file with the needed version.
 					continue nextFile
 				}
-				fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.versions[i].device, name)
+				fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.Versions[i].Device, name)
 				bs, err := t.Get(fk, nil)
 				if err != nil {
 					var id protocol.DeviceID
@@ -528,7 +497,7 @@ nextFile:
 					continue nextFile
 				}
 
-				l.Debugf("need folder=%q device=%v name=%q need=%v have=%v haveV=%d globalV=%d", folder, protocol.DeviceIDFromBytes(device), name, need, have, haveVersion, vl.versions[0].version)
+				l.Debugf("need folder=%q device=%v name=%q need=%v have=%v haveV=%d globalV=%d", folder, protocol.DeviceIDFromBytes(device), name, need, have, haveVersion, vl.Versions[0].Version)
 
 				if cont := fn(gf); !cont {
 					return
@@ -601,7 +570,7 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
 	for dbi.Next() {
 		gk := dbi.Key()
 		var vl VersionList
-		err := vl.UnmarshalXDR(dbi.Value())
+		err := vl.Unmarshal(dbi.Value())
 		if err != nil {
 			panic(err)
 		}
@@ -613,8 +582,8 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
 
 		name := db.globalKeyName(gk)
 		var newVL VersionList
-		for i, version := range vl.versions {
-			fk = db.deviceKeyInto(fk[:cap(fk)], folder, version.device, name)
+		for i, version := range vl.Versions {
+			fk = db.deviceKeyInto(fk[:cap(fk)], folder, version.Device, name)
 
 			_, err := t.Get(fk, nil)
 			if err == leveldb.ErrNotFound {
@@ -623,10 +592,10 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
 			if err != nil {
 				panic(err)
 			}
-			newVL.versions = append(newVL.versions, version)
+			newVL.Versions = append(newVL.Versions, version)
 
 			if i == 0 {
-				fi, ok := t.getFile(folder, version.device, name)
+				fi, ok := t.getFile(folder, version.Device, name)
 				if !ok {
 					panic("nonexistent global master file")
 				}
@@ -634,8 +603,8 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
 			}
 		}
 
-		if len(newVL.versions) != len(vl.versions) {
-			t.Put(dbi.Key(), newVL.MustMarshalXDR())
+		if len(newVL.Versions) != len(vl.Versions) {
+			t.Put(dbi.Key(), mustMarshal(&newVL))
 			t.checkFlush()
 		}
 	}
@@ -712,15 +681,46 @@ func (db *Instance) globalKeyFolder(key []byte) []byte {
 	return folder
 }
 
+func (db *Instance) getIndexID(device, folder []byte) protocol.IndexID {
+	key := db.indexIDKey(device, folder)
+	cur, err := db.Get(key, nil)
+	if err != nil {
+		return 0
+	}
+
+	var id protocol.IndexID
+	if err := id.Unmarshal(cur); err != nil {
+		return 0
+	}
+
+	return id
+}
+
+func (db *Instance) setIndexID(device, folder []byte, id protocol.IndexID) {
+	key := db.indexIDKey(device, folder)
+	bs, _ := id.Marshal() // marshalling can't fail
+	if err := db.Put(key, bs, nil); err != nil {
+		panic("storing index ID: " + err.Error())
+	}
+}
+
+func (db *Instance) indexIDKey(device, folder []byte) []byte {
+	k := make([]byte, keyPrefixLen+keyDeviceLen+keyFolderLen)
+	k[0] = KeyTypeIndexID
+	binary.BigEndian.PutUint32(k[keyPrefixLen:], db.deviceIdx.ID(device))
+	binary.BigEndian.PutUint32(k[keyPrefixLen+keyDeviceLen:], db.folderIdx.ID(folder))
+	return k
+}
+
 func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) {
 	if truncate {
 		var tf FileInfoTruncated
-		err := tf.UnmarshalXDR(bs)
+		err := tf.Unmarshal(bs)
 		return tf, err
 	}
 
 	var tf protocol.FileInfo
-	err := tf.UnmarshalXDR(bs)
+	err := tf.Unmarshal(bs)
 	return tf, err
 }
 
@@ -740,50 +740,6 @@ func leveldbIsCorrupted(err error) bool {
 	return false
 }
 
-// checkConvertDatabase tries to convert an existing old (v0.11) database to
-// new (v0.13) format.
-func checkConvertDatabase(dbFile string) error {
-	oldLoc := filepath.Join(filepath.Dir(dbFile), "index-v0.11.0.db")
-	if _, err := os.Stat(oldLoc); os.IsNotExist(err) {
-		// The old database file does not exist; that's ok, continue as if
-		// everything succeeded.
-		return nil
-	} else if err != nil {
-		// Any other error is weird.
-		return err
-	}
-
-	// There exists a database in the old format. We run a one time
-	// conversion from old to new.
-
-	fromDb, err := leveldb.OpenFile(oldLoc, nil)
-	if err != nil {
-		return err
-	}
-
-	toDb, err := leveldb.OpenFile(dbFile, nil)
-	if err != nil {
-		return err
-	}
-
-	err = convertKeyFormat(fromDb, toDb)
-	if err != nil {
-		return err
-	}
-
-	err = toDb.Close()
-	if err != nil {
-		return err
-	}
-
-	// We've done this one, we don't want to do it again (if the user runs
-	// -reset or so). We don't care too much about errors any more at this stage.
-	fromDb.Close()
-	osutil.Rename(oldLoc, oldLoc+".converted")
-
-	return nil
-}
-
 // A smallIndex is an in memory bidirectional []byte to uint32 map. It gives
 // fast lookups in both directions and persists to the database. Don't use for
 // storing more items than fit comfortably in RAM.

lib/db/leveldb_transactions.go

@@ -74,18 +74,12 @@ func (t readWriteTransaction) flush() {
 	atomic.AddInt64(&t.db.committed, int64(t.Batch.Len()))
 }
 
-func (t readWriteTransaction) insertFile(folder, device []byte, file protocol.FileInfo) int64 {
+func (t readWriteTransaction) insertFile(folder, device []byte, file protocol.FileInfo) {
 	l.Debugf("insert; folder=%q device=%v %v", folder, protocol.DeviceIDFromBytes(device), file)
 
-	if file.LocalVersion == 0 {
-		file.LocalVersion = clock(0)
-	}
-
 	name := []byte(file.Name)
 	nk := t.db.deviceKey(folder, device, name)
-	t.Put(nk, file.MustMarshalXDR())
-
-	return file.LocalVersion
+	t.Put(nk, mustMarshal(&file))
 }
 
 // updateGlobal adds this device+version to the version list for the given
@@ -105,14 +99,14 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
 	var hasOldFile bool
 	// Remove the device from the current version list
 	if len(svl) != 0 {
-		err = fl.UnmarshalXDR(svl)
+		err = fl.Unmarshal(svl)
 		if err != nil {
 			panic(err)
 		}
 
-		for i := range fl.versions {
-			if bytes.Equal(fl.versions[i].device, device) {
-				if fl.versions[i].version.Equal(file.Version) {
+		for i := range fl.Versions {
+			if bytes.Equal(fl.Versions[i].Device, device) {
+				if fl.Versions[i].Version.Equal(file.Version) {
 					// No need to do anything
 					return false
 				}
@@ -120,29 +114,29 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
 				if i == 0 {
 					// Keep the current newest file around so we can subtract it from
 					// the globalSize if we replace it.
-					oldFile, hasOldFile = t.getFile(folder, fl.versions[0].device, name)
+					oldFile, hasOldFile = t.getFile(folder, fl.Versions[0].Device, name)
 				}
 
-				fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
+				fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...)
 				break
 			}
 		}
 	}
 
-	nv := fileVersion{
-		device:  device,
-		version: file.Version,
+	nv := FileVersion{
+		Device:  device,
+		Version: file.Version,
 	}
 
 	insertedAt := -1
 	// Find a position in the list to insert this file. The file at the front
 	// of the list is the newer, the "global".
-	for i := range fl.versions {
-		switch fl.versions[i].version.Compare(file.Version) {
+	for i := range fl.Versions {
+		switch fl.Versions[i].Version.Compare(file.Version) {
 		case protocol.Equal, protocol.Lesser:
 			// The version at this point in the list is equal to or lesser
 			// ("older") than us. We insert ourselves in front of it.
-			fl.versions = insertVersion(fl.versions, i, nv)
+			fl.Versions = insertVersion(fl.Versions, i, nv)
 			insertedAt = i
 			goto done
 
@@ -153,12 +147,12 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
 			// "Greater" in the condition above is just based on the device
 			// IDs in the version vector, which is not the only thing we use
 			// to determine the winner.)
-			of, ok := t.getFile(folder, fl.versions[i].device, name)
+			of, ok := t.getFile(folder, fl.Versions[i].Device, name)
 			if !ok {
 				panic("file referenced in version list does not exist")
 			}
 			if file.WinsConflict(of) {
-				fl.versions = insertVersion(fl.versions, i, nv)
+				fl.Versions = insertVersion(fl.Versions, i, nv)
 				insertedAt = i
 				goto done
 			}
@@ -166,8 +160,8 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
 	}
 
 	// We didn't find a position for an insert above, so append to the end.
-	fl.versions = append(fl.versions, nv)
-	insertedAt = len(fl.versions) - 1
+	fl.Versions = append(fl.Versions, nv)
+	insertedAt = len(fl.Versions) - 1
 
 done:
 	if insertedAt == 0 {
@@ -178,9 +172,9 @@ done:
 			if hasOldFile {
 				// We have the old file that was removed at the head of the list.
 				globalSize.removeFile(oldFile)
-			} else if len(fl.versions) > 1 {
+			} else if len(fl.Versions) > 1 {
 				// The previous newest version is now at index 1, grab it from there.
-				oldFile, ok := t.getFile(folder, fl.versions[1].device, name)
+				oldFile, ok := t.getFile(folder, fl.Versions[1].Device, name)
 				if !ok {
 					panic("file referenced in version list does not exist")
 				}
@@ -190,7 +184,7 @@ done:
 	}
 
 	l.Debugf("new global after update: %v", fl)
-	t.Put(gk, fl.MustMarshalXDR())
+	t.Put(gk, mustMarshal(&fl))
 
 	return true
 }
@@ -210,14 +204,14 @@ func (t readWriteTransaction) removeFromGlobal(folder, device, file []byte, glob
 	}
 
 	var fl VersionList
-	err = fl.UnmarshalXDR(svl)
+	err = fl.Unmarshal(svl)
 	if err != nil {
 		panic(err)
 	}
 
 	removed := false
-	for i := range fl.versions {
-		if bytes.Equal(fl.versions[i].device, device) {
+	for i := range fl.Versions {
+		if bytes.Equal(fl.Versions[i].Device, device) {
 			if i == 0 && globalSize != nil {
 				f, ok := t.getFile(folder, device, file)
 				if !ok {
@@ -226,18 +220,18 @@ func (t readWriteTransaction) removeFromGlobal(folder, device, file []byte, glob
 				globalSize.removeFile(f)
 				removed = true
 			}
-			fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
+			fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...)
 			break
 		}
 	}
 
-	if len(fl.versions) == 0 {
+	if len(fl.Versions) == 0 {
 		t.Delete(gk)
 	} else {
 		l.Debugf("new global after remove: %v", fl)
-		t.Put(gk, fl.MustMarshalXDR())
+		t.Put(gk, mustMarshal(&fl))
 		if removed {
-			f, ok := t.getFile(folder, fl.versions[0].device, file)
+			f, ok := t.getFile(folder, fl.Versions[0].Device, file)
 			if !ok {
 				panic("new global is nonexistent file")
 			}
@@ -246,9 +240,21 @@ func (t readWriteTransaction) removeFromGlobal(folder, device, file []byte, glob
 	}
 }
 
-func insertVersion(vl []fileVersion, i int, v fileVersion) []fileVersion {
-	t := append(vl, fileVersion{})
+func insertVersion(vl []FileVersion, i int, v FileVersion) []FileVersion {
+	t := append(vl, FileVersion{})
 	copy(t[i+1:], t[i:])
 	t[i] = v
 	return t
 }
+
+type marshaller interface {
+	Marshal() ([]byte, error)
+}
+
+func mustMarshal(f marshaller) []byte {
+	bs, err := f.Marshal()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}

lib/db/leveldb_xdr.go

@@ -1,142 +0,0 @@
-// ************************************************************
-// This file is automatically generated by genxdr. Do not edit.
-// ************************************************************
-
-package db
-
-import (
-	"github.com/calmh/xdr"
-)
-
-/*
-
-fileVersion Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                       Vector Structure                        \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                 device (length + padded data)                 \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct fileVersion {
-	Vector version;
-	opaque device<>;
-}
-
-*/
-
-func (o fileVersion) XDRSize() int {
-	return o.version.XDRSize() +
-		4 + len(o.device) + xdr.Padding(len(o.device))
-}
-
-func (o fileVersion) MarshalXDR() ([]byte, error) {
-	buf := make([]byte, o.XDRSize())
-	m := &xdr.Marshaller{Data: buf}
-	return buf, o.MarshalXDRInto(m)
-}
-
-func (o fileVersion) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o fileVersion) MarshalXDRInto(m *xdr.Marshaller) error {
-	if err := o.version.MarshalXDRInto(m); err != nil {
-		return err
-	}
-	m.MarshalBytes(o.device)
-	return m.Error
-}
-
-func (o *fileVersion) UnmarshalXDR(bs []byte) error {
-	u := &xdr.Unmarshaller{Data: bs}
-	return o.UnmarshalXDRFrom(u)
-}
-func (o *fileVersion) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	(&o.version).UnmarshalXDRFrom(u)
-	o.device = u.UnmarshalBytes()
-	return u.Error
-}
-
-/*
-
-VersionList Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                      Number of versions                       |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\              Zero or more fileVersion Structures              \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct VersionList {
-	fileVersion versions<>;
-}
-
-*/
-
-func (o VersionList) XDRSize() int {
-	return 4 + xdr.SizeOfSlice(o.versions)
-}
-
-func (o VersionList) MarshalXDR() ([]byte, error) {
-	buf := make([]byte, o.XDRSize())
-	m := &xdr.Marshaller{Data: buf}
-	return buf, o.MarshalXDRInto(m)
-}
-
-func (o VersionList) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o VersionList) MarshalXDRInto(m *xdr.Marshaller) error {
-	m.MarshalUint32(uint32(len(o.versions)))
-	for i := range o.versions {
-		if err := o.versions[i].MarshalXDRInto(m); err != nil {
-			return err
-		}
-	}
-	return m.Error
-}
-
-func (o *VersionList) UnmarshalXDR(bs []byte) error {
-	u := &xdr.Unmarshaller{Data: bs}
-	return o.UnmarshalXDRFrom(u)
-}
-func (o *VersionList) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	_versionsSize := int(u.UnmarshalUint32())
-	if _versionsSize < 0 {
-		return xdr.ElementSizeExceeded("versions", _versionsSize, 0)
-	} else if _versionsSize == 0 {
-		o.versions = nil
-	} else {
-		if _versionsSize <= len(o.versions) {
-			o.versions = o.versions[:_versionsSize]
-		} else {
-			o.versions = make([]fileVersion, _versionsSize)
-		}
-		for i := range o.versions {
-			(&o.versions[i]).UnmarshalXDRFrom(u)
-		}
-	}
-	return u.Error
-}

lib/db/set.go

@@ -14,6 +14,7 @@ package db
 
 import (
 	stdsync "sync"
+	"sync/atomic"
 
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
@@ -21,19 +22,22 @@ import (
 )
 
 type FileSet struct {
-	localVersion map[protocol.DeviceID]int64
-	mutex        sync.Mutex
+	localVersion int64 // Our local version counter
 	folder       string
 	db           *Instance
 	blockmap     *BlockMap
 	localSize    sizeTracker
 	globalSize   sizeTracker
+
+	remoteLocalVersion map[protocol.DeviceID]int64 // Highest seen local versions for other devices
+	updateMutex        sync.Mutex                  // protects remoteLocalVersion and database updates
 }
 
 // FileIntf is the set of methods implemented by both protocol.FileInfo and
-// protocol.FileInfoTruncated.
+// FileInfoTruncated.
 type FileIntf interface {
-	Size() int64
+	FileSize() int64
+	FileName() string
 	IsDeleted() bool
 	IsInvalid() bool
 	IsDirectory() bool
@@ -42,7 +46,7 @@ type FileIntf interface {
 }
 
 // The Iterator is called with either a protocol.FileInfo or a
-// protocol.FileInfoTruncated (depending on the method) and returns true to
+// FileInfoTruncated (depending on the method) and returns true to
 // continue iteration, false to stop.
 type Iterator func(f FileIntf) bool
 
@@ -64,7 +68,7 @@ func (s *sizeTracker) addFile(f FileIntf) {
 	} else {
 		s.files++
 	}
-	s.bytes += f.Size()
+	s.bytes += f.FileSize()
 	s.mut.Unlock()
 }
 
@@ -79,7 +83,7 @@ func (s *sizeTracker) removeFile(f FileIntf) {
 	} else {
 		s.files--
 	}
-	s.bytes -= f.Size()
+	s.bytes -= f.FileSize()
 	if s.deleted < 0 || s.files < 0 {
 		panic("bug: removed more than added")
 	}
@@ -94,11 +98,11 @@ func (s *sizeTracker) Size() (files, deleted int, bytes int64) {
 
 func NewFileSet(folder string, db *Instance) *FileSet {
 	var s = FileSet{
-		localVersion: make(map[protocol.DeviceID]int64),
-		folder:       folder,
-		db:           db,
-		blockmap:     NewBlockMap(db, db.folderIdx.ID([]byte(folder))),
-		mutex:        sync.NewMutex(),
+		remoteLocalVersion: make(map[protocol.DeviceID]int64),
+		folder:             folder,
+		db:                 db,
+		blockmap:           NewBlockMap(db, db.folderIdx.ID([]byte(folder))),
+		updateMutex:        sync.NewMutex(),
 	}
 
 	s.db.checkGlobals([]byte(folder), &s.globalSize)
@@ -106,16 +110,17 @@ func NewFileSet(folder string, db *Instance) *FileSet {
 	var deviceID protocol.DeviceID
 	s.db.withAllFolderTruncated([]byte(folder), func(device []byte, f FileInfoTruncated) bool {
 		copy(deviceID[:], device)
-		if f.LocalVersion > s.localVersion[deviceID] {
-			s.localVersion[deviceID] = f.LocalVersion
-		}
 		if deviceID == protocol.LocalDeviceID {
+			if f.LocalVersion > s.localVersion {
+				s.localVersion = f.LocalVersion
+			}
 			s.localSize.addFile(f)
+		} else if f.LocalVersion > s.remoteLocalVersion[deviceID] {
+			s.remoteLocalVersion[deviceID] = f.LocalVersion
 		}
 		return true
 	})
 	l.Debugf("loaded localVersion for %q: %#v", folder, s.localVersion)
-	clock(s.localVersion[protocol.LocalDeviceID])
 
 	return &s
 }
@@ -123,13 +128,25 @@ func NewFileSet(folder string, db *Instance) *FileSet {
 func (s *FileSet) Replace(device protocol.DeviceID, fs []protocol.FileInfo) {
 	l.Debugf("%s Replace(%v, [%d])", s.folder, device, len(fs))
 	normalizeFilenames(fs)
-	s.mutex.Lock()
-	defer s.mutex.Unlock()
-	s.localVersion[device] = s.db.replace([]byte(s.folder), device[:], fs, &s.localSize, &s.globalSize)
-	if len(fs) == 0 {
-		// Reset the local version if all files were removed.
-		s.localVersion[device] = 0
+
+	s.updateMutex.Lock()
+	defer s.updateMutex.Unlock()
+
+	if device == protocol.LocalDeviceID {
+		if len(fs) == 0 {
+			s.localVersion = 0
+		} else {
+			// Always overwrite LocalVersion on updated files to ensure
+			// correct ordering. The caller is supposed to leave it set to
+			// zero anyhow.
+			for i := range fs {
+				fs[i].LocalVersion = atomic.AddInt64(&s.localVersion, 1)
+			}
+		}
+	} else {
+		s.remoteLocalVersion[device] = maxLocalVersion(fs)
 	}
+	s.db.replace([]byte(s.folder), device[:], fs, &s.localSize, &s.globalSize)
 	if device == protocol.LocalDeviceID {
 		s.blockmap.Drop()
 		s.blockmap.Add(fs)
@@ -139,12 +156,15 @@ func (s *FileSet) Replace(device protocol.DeviceID, fs []protocol.FileInfo) {
 func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
 	l.Debugf("%s Update(%v, [%d])", s.folder, device, len(fs))
 	normalizeFilenames(fs)
-	s.mutex.Lock()
-	defer s.mutex.Unlock()
+
+	s.updateMutex.Lock()
+	defer s.updateMutex.Unlock()
+
 	if device == protocol.LocalDeviceID {
 		discards := make([]protocol.FileInfo, 0, len(fs))
 		updates := make([]protocol.FileInfo, 0, len(fs))
-		for _, newFile := range fs {
+		for i, newFile := range fs {
+			fs[i].LocalVersion = atomic.AddInt64(&s.localVersion, 1)
 			existingFile, ok := s.db.getFile([]byte(s.folder), device[:], []byte(newFile.Name))
 			if !ok || !existingFile.Version.Equal(newFile.Version) {
 				discards = append(discards, existingFile)
@@ -153,10 +173,10 @@ func (s *FileSet) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
 		}
 		s.blockmap.Discard(discards)
 		s.blockmap.Update(updates)
+	} else {
+		s.remoteLocalVersion[device] = maxLocalVersion(fs)
 	}
-	if lv := s.db.updateFiles([]byte(s.folder), device[:], fs, &s.localSize, &s.globalSize); lv > s.localVersion[device] {
-		s.localVersion[device] = lv
-	}
+	s.db.updateFiles([]byte(s.folder), device[:], fs, &s.localSize, &s.globalSize)
 }
 
 func (s *FileSet) WithNeed(device protocol.DeviceID, fn Iterator) {
@@ -229,9 +249,13 @@ func (s *FileSet) Availability(file string) []protocol.DeviceID {
 }
 
 func (s *FileSet) LocalVersion(device protocol.DeviceID) int64 {
-	s.mutex.Lock()
-	defer s.mutex.Unlock()
-	return s.localVersion[device]
+	if device == protocol.LocalDeviceID {
+		return atomic.LoadInt64(&s.localVersion)
+	}
+
+	s.updateMutex.Lock()
+	defer s.updateMutex.Unlock()
+	return s.remoteLocalVersion[device]
 }
 
 func (s *FileSet) LocalSize() (files, deleted int, bytes int64) {
@@ -242,6 +266,37 @@ func (s *FileSet) GlobalSize() (files, deleted int, bytes int64) {
 	return s.globalSize.Size()
 }
 
+func (s *FileSet) IndexID(device protocol.DeviceID) protocol.IndexID {
+	id := s.db.getIndexID(device[:], []byte(s.folder))
+	if id == 0 && device == protocol.LocalDeviceID {
+		// No index ID set yet. We create one now.
+		id = protocol.NewIndexID()
+		s.db.setIndexID(device[:], []byte(s.folder), id)
+	}
+	return id
+}
+
+func (s *FileSet) SetIndexID(device protocol.DeviceID, id protocol.IndexID) {
+	if device == protocol.LocalDeviceID {
+		panic("do not explicitly set index ID for local device")
+	}
+	s.db.setIndexID(device[:], []byte(s.folder), id)
+}
+
+// maxLocalVersion returns the highest of the LocalVersion numbers found in
+// the given slice of FileInfos. This should really be the LocalVersion of
+// the last item, but Syncthing v0.14.0 and other implementations may not
+// implement update sorting....
+func maxLocalVersion(fs []protocol.FileInfo) int64 {
+	var max int64
+	for _, f := range fs {
+		if f.LocalVersion > max {
+			max = f.LocalVersion
+		}
+	}
+	return max
+}
+
 // DropFolder clears out all information related to the given folder from the
 // database.
 func DropFolder(db *Instance, folder string) {

lib/db/set_test.go

@@ -88,7 +88,7 @@ func (l fileList) String() string {
 	var b bytes.Buffer
 	b.WriteString("[]protocol.FileList{\n")
 	for _, f := range l {
-		fmt.Fprintf(&b, "  %q: #%d, %d bytes, %d blocks, flags=%o\n", f.Name, f.Version, f.Size(), len(f.Blocks), f.Flags)
+		fmt.Fprintf(&b, "  %q: #%d, %d bytes, %d blocks, perms=%o\n", f.Name, f.Version, f.Size, len(f.Blocks), f.Permissions)
 	}
 	b.WriteString("}")
 	return b.String()
@@ -100,35 +100,35 @@ func TestGlobalSet(t *testing.T) {
 	m := db.NewFileSet("test", ldb)
 
 	local0 := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(3)},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(8)},
+		protocol.FileInfo{Name: "a", LocalVersion: 1, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", LocalVersion: 2, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", LocalVersion: 3, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(3)},
+		protocol.FileInfo{Name: "d", LocalVersion: 4, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "z", LocalVersion: 5, Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(8)},
 	}
 	local1 := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(3)},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(3)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "z", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Deleted: true},
 	}
 	localTot := fileList{
 		local0[0],
 		local0[1],
 		local0[2],
 		local0[3],
-		protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted},
+		protocol.FileInfo{Name: "z", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Deleted: true},
 	}
 
 	remote0 := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(5)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(5)},
 	}
 	remote1 := fileList{
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(6)},
-		protocol.FileInfo{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(6)},
+		protocol.FileInfo{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(7)},
 	}
 	remoteTot := fileList{
 		remote0[0],
@@ -178,7 +178,7 @@ func TestGlobalSet(t *testing.T) {
 		} else {
 			globalFiles++
 		}
-		globalBytes += f.Size()
+		globalBytes += f.FileSize()
 	}
 	gsFiles, gsDeleted, gsBytes := m.GlobalSize()
 	if gsFiles != globalFiles {
@@ -208,7 +208,7 @@ func TestGlobalSet(t *testing.T) {
 		} else {
 			haveFiles++
 		}
-		haveBytes += f.Size()
+		haveBytes += f.FileSize()
 	}
 	lsFiles, lsDeleted, lsBytes := m.LocalSize()
 	if lsFiles != haveFiles {
@@ -303,23 +303,23 @@ func TestNeedWithInvalid(t *testing.T) {
 	s := db.NewFileSet("test", ldb)
 
 	localHave := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
 	}
 	remote0Have := fileList{
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), Invalid: true},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
 	}
 	remote1Have := fileList{
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(5), Invalid: true},
+		protocol.FileInfo{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), Invalid: true},
 	}
 
 	expectedNeed := fileList{
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
 	}
 
 	s.Replace(protocol.LocalDeviceID, localHave)
@@ -340,10 +340,10 @@ func TestUpdateToInvalid(t *testing.T) {
 	s := db.NewFileSet("test", ldb)
 
 	localHave := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), Invalid: true},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
 	}
 
 	s.Replace(protocol.LocalDeviceID, localHave)
@@ -355,7 +355,7 @@ func TestUpdateToInvalid(t *testing.T) {
 		t.Errorf("Have incorrect before invalidation;\n A: %v !=\n E: %v", have, localHave)
 	}
 
-	localHave[1] = protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagInvalid}
+	localHave[1] = protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Invalid: true}
 	s.Update(protocol.LocalDeviceID, localHave[1:2])
 
 	have = fileList(haveList(s, protocol.LocalDeviceID))
@@ -372,16 +372,16 @@ func TestInvalidAvailability(t *testing.T) {
 	s := db.NewFileSet("test", ldb)
 
 	remote0Have := fileList{
-		protocol.FileInfo{Name: "both", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "r1only", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "r0only", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
-		protocol.FileInfo{Name: "none", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "both", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "r1only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), Invalid: true},
+		protocol.FileInfo{Name: "r0only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "none", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), Invalid: true},
 	}
 	remote1Have := fileList{
-		protocol.FileInfo{Name: "both", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "r1only", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
-		protocol.FileInfo{Name: "r0only", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "none", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "both", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "r1only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "r0only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(5), Invalid: true},
+		protocol.FileInfo{Name: "none", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), Invalid: true},
 	}
 
 	s.Replace(remoteDevice0, remote0Have)
@@ -410,17 +410,17 @@ func TestGlobalReset(t *testing.T) {
 	m := db.NewFileSet("test", ldb)
 
 	local := []protocol.FileInfo{
-		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	remote := []protocol.FileInfo{
-		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
-		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
+		{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	m.Replace(protocol.LocalDeviceID, local)
@@ -448,23 +448,23 @@ func TestNeed(t *testing.T) {
 	m := db.NewFileSet("test", ldb)
 
 	local := []protocol.FileInfo{
-		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	remote := []protocol.FileInfo{
-		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
-		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
+		{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	shouldNeed := []protocol.FileInfo{
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
-		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
+		{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	m.Replace(protocol.LocalDeviceID, local)
@@ -486,18 +486,18 @@ func TestLocalVersion(t *testing.T) {
 	m := db.NewFileSet("test", ldb)
 
 	local1 := []protocol.FileInfo{
-		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	local2 := []protocol.FileInfo{
 		local1[0],
 		// [1] deleted
 		local1[2],
-		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1002}}},
-		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
+		{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	m.Replace(protocol.LocalDeviceID, local1)
@@ -515,17 +515,17 @@ func TestListDropFolder(t *testing.T) {
 
 	s0 := db.NewFileSet("test0", ldb)
 	local1 := []protocol.FileInfo{
-		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 	s0.Replace(protocol.LocalDeviceID, local1)
 
 	s1 := db.NewFileSet("test1", ldb)
 	local2 := []protocol.FileInfo{
-		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1002}}},
-		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1002}}},
-		{Name: "f", Version: protocol.Vector{{ID: myID, Value: 1002}}},
+		{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
+		{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
+		{Name: "f", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
 	}
 	s1.Replace(remoteDevice0, local2)
 
@@ -566,24 +566,24 @@ func TestGlobalNeedWithInvalid(t *testing.T) {
 	s := db.NewFileSet("test1", ldb)
 
 	rem0 := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Invalid: true},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
 	}
 	s.Replace(remoteDevice0, rem0)
 
 	rem1 := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Invalid: true},
 	}
 	s.Replace(remoteDevice1, rem1)
 
 	total := fileList{
 		// There's a valid copy of each file, so it should be merged
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
 	}
 
 	need := fileList(needList(s, protocol.LocalDeviceID))
@@ -609,7 +609,7 @@ func TestLongPath(t *testing.T) {
 	name := b.String() // 5000 characters
 
 	local := []protocol.FileInfo{
-		{Name: string(name), Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: string(name), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	s.Replace(protocol.LocalDeviceID, local)
@@ -633,7 +633,7 @@ func TestCommitted(t *testing.T) {
 	s := db.NewFileSet("test", ldb)
 
 	local := []protocol.FileInfo{
-		{Name: string("file"), Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: string("file"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	// Adding a file should increase the counter
@@ -659,12 +659,12 @@ func TestCommitted(t *testing.T) {
 
 func BenchmarkUpdateOneFile(b *testing.B) {
 	local0 := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(3)},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(3)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(4)},
 		// A longer name is more realistic and causes more allocations
-		protocol.FileInfo{Name: "zajksdhaskjdh/askjdhaskjdashkajshd/kasjdhaskjdhaskdjhaskdjash/dkjashdaksjdhaskdjahskdjh", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(8)},
+		protocol.FileInfo{Name: "zajksdhaskjdh/askjdhaskjdashkajshd/kasjdhaskjdhaskdjhaskdjash/dkjashdaksjdhaskdjahskdjh", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(8)},
 	}
 
 	ldb, err := db.Open("testdata/benchmarkupdate.db")
@@ -687,3 +687,35 @@ func BenchmarkUpdateOneFile(b *testing.B) {
 
 	b.ReportAllocs()
 }
+
+func TestIndexID(t *testing.T) {
+	ldb := db.OpenMemory()
+
+	s := db.NewFileSet("test", ldb)
+
+	// The Index ID for some random device is zero by default.
+	id := s.IndexID(remoteDevice0)
+	if id != 0 {
+		t.Errorf("index ID for remote device should default to zero, not %d", id)
+	}
+
+	// The Index ID for someone else should be settable
+	s.SetIndexID(remoteDevice0, 42)
+	id = s.IndexID(remoteDevice0)
+	if id != 42 {
+		t.Errorf("index ID for remote device should be remembered; got %d, expected %d", id, 42)
+	}
+
+	// Our own index ID should be generated randomly.
+	id = s.IndexID(protocol.LocalDeviceID)
+	if id == 0 {
+		t.Errorf("index ID for local device should be random, not zero")
+	}
+	t.Logf("random index ID is 0x%016x", id)
+
+	// But of course always the same after that.
+	again := s.IndexID(protocol.LocalDeviceID)
+	if again != id {
+		t.Errorf("index ID changed; %d != %d", again, id)
+	}
+}

lib/db/structs.go

@@ -0,0 +1,57 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+//go:generate go run ../../script/protofmt.go structs.proto
+//go:generate protoc --proto_path=../../../../../:../../../../gogo/protobuf/protobuf:. --gogofast_out=. structs.proto
+
+package db
+
+import (
+	"fmt"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+func (f FileInfoTruncated) String() string {
+	return fmt.Sprintf("File{Name:%q, Permissions:0%o, Modified:%d, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v}",
+		f.Name, f.Permissions, f.Modified, f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions)
+}
+
+func (f FileInfoTruncated) IsDeleted() bool {
+	return f.Deleted
+}
+
+func (f FileInfoTruncated) IsInvalid() bool {
+	return f.Invalid
+}
+
+func (f FileInfoTruncated) IsDirectory() bool {
+	return f.Type == protocol.FileInfoTypeDirectory
+}
+
+func (f FileInfoTruncated) IsSymlink() bool {
+	switch f.Type {
+	case protocol.FileInfoTypeSymlinkDirectory, protocol.FileInfoTypeSymlinkFile, protocol.FileInfoTypeSymlinkUnknown:
+		return true
+	default:
+		return false
+	}
+}
+
+func (f FileInfoTruncated) HasPermissionBits() bool {
+	return !f.NoPermissions
+}
+
+func (f FileInfoTruncated) FileSize() int64 {
+	if f.IsDirectory() || f.IsDeleted() {
+		return 128
+	}
+	return f.Size
+}
+
+func (f FileInfoTruncated) FileName() string {
+	return f.Name
+}

lib/db/structs.pb.go

@@ -0,0 +1,914 @@
+// Code generated by protoc-gen-gogo.
+// source: structs.proto
+// DO NOT EDIT!
+
+/*
+	Package db is a generated protocol buffer package.
+
+	It is generated from these files:
+		structs.proto
+
+	It has these top-level messages:
+		FileVersion
+		VersionList
+		FileInfoTruncated
+*/
+package db
+
+import proto "github.com/gogo/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import _ "github.com/gogo/protobuf/gogoproto"
+import protocol "github.com/syncthing/syncthing/lib/protocol"
+
+import io "io"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+const _ = proto.GoGoProtoPackageIsVersion1
+
+type FileVersion struct {
+	Version protocol.Vector `protobuf:"bytes,1,opt,name=version" json:"version"`
+	Device  []byte          `protobuf:"bytes,2,opt,name=device,proto3" json:"device,omitempty"`
+}
+
+func (m *FileVersion) Reset()                    { *m = FileVersion{} }
+func (m *FileVersion) String() string            { return proto.CompactTextString(m) }
+func (*FileVersion) ProtoMessage()               {}
+func (*FileVersion) Descriptor() ([]byte, []int) { return fileDescriptorStructs, []int{0} }
+
+type VersionList struct {
+	Versions []FileVersion `protobuf:"bytes,1,rep,name=versions" json:"versions"`
+}
+
+func (m *VersionList) Reset()                    { *m = VersionList{} }
+func (*VersionList) ProtoMessage()               {}
+func (*VersionList) Descriptor() ([]byte, []int) { return fileDescriptorStructs, []int{1} }
+
+// Must be the same as FileInfo but without the blocks field
+type FileInfoTruncated struct {
+	Name          string                `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	Type          protocol.FileInfoType `protobuf:"varint,2,opt,name=type,proto3,enum=protocol.FileInfoType" json:"type,omitempty"`
+	Size          int64                 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"`
+	Permissions   uint32                `protobuf:"varint,4,opt,name=permissions,proto3" json:"permissions,omitempty"`
+	Modified      int64                 `protobuf:"varint,5,opt,name=modified,proto3" json:"modified,omitempty"`
+	Deleted       bool                  `protobuf:"varint,6,opt,name=deleted,proto3" json:"deleted,omitempty"`
+	Invalid       bool                  `protobuf:"varint,7,opt,name=invalid,proto3" json:"invalid,omitempty"`
+	NoPermissions bool                  `protobuf:"varint,8,opt,name=no_permissions,json=noPermissions,proto3" json:"no_permissions,omitempty"`
+	Version       protocol.Vector       `protobuf:"bytes,9,opt,name=version" json:"version"`
+	LocalVersion  int64                 `protobuf:"varint,10,opt,name=local_version,json=localVersion,proto3" json:"local_version,omitempty"`
+}
+
+func (m *FileInfoTruncated) Reset()                    { *m = FileInfoTruncated{} }
+func (*FileInfoTruncated) ProtoMessage()               {}
+func (*FileInfoTruncated) Descriptor() ([]byte, []int) { return fileDescriptorStructs, []int{2} }
+
+func init() {
+	proto.RegisterType((*FileVersion)(nil), "db.FileVersion")
+	proto.RegisterType((*VersionList)(nil), "db.VersionList")
+	proto.RegisterType((*FileInfoTruncated)(nil), "db.FileInfoTruncated")
+}
+func (m *FileVersion) Marshal() (data []byte, err error) {
+	size := m.ProtoSize()
+	data = make([]byte, size)
+	n, err := m.MarshalTo(data)
+	if err != nil {
+		return nil, err
+	}
+	return data[:n], nil
+}
+
+func (m *FileVersion) MarshalTo(data []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	data[i] = 0xa
+	i++
+	i = encodeVarintStructs(data, i, uint64(m.Version.ProtoSize()))
+	n1, err := m.Version.MarshalTo(data[i:])
+	if err != nil {
+		return 0, err
+	}
+	i += n1
+	if len(m.Device) > 0 {
+		data[i] = 0x12
+		i++
+		i = encodeVarintStructs(data, i, uint64(len(m.Device)))
+		i += copy(data[i:], m.Device)
+	}
+	return i, nil
+}
+
+func (m *VersionList) Marshal() (data []byte, err error) {
+	size := m.ProtoSize()
+	data = make([]byte, size)
+	n, err := m.MarshalTo(data)
+	if err != nil {
+		return nil, err
+	}
+	return data[:n], nil
+}
+
+func (m *VersionList) MarshalTo(data []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.Versions) > 0 {
+		for _, msg := range m.Versions {
+			data[i] = 0xa
+			i++
+			i = encodeVarintStructs(data, i, uint64(msg.ProtoSize()))
+			n, err := msg.MarshalTo(data[i:])
+			if err != nil {
+				return 0, err
+			}
+			i += n
+		}
+	}
+	return i, nil
+}
+
+func (m *FileInfoTruncated) Marshal() (data []byte, err error) {
+	size := m.ProtoSize()
+	data = make([]byte, size)
+	n, err := m.MarshalTo(data)
+	if err != nil {
+		return nil, err
+	}
+	return data[:n], nil
+}
+
+func (m *FileInfoTruncated) MarshalTo(data []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.Name) > 0 {
+		data[i] = 0xa
+		i++
+		i = encodeVarintStructs(data, i, uint64(len(m.Name)))
+		i += copy(data[i:], m.Name)
+	}
+	if m.Type != 0 {
+		data[i] = 0x10
+		i++
+		i = encodeVarintStructs(data, i, uint64(m.Type))
+	}
+	if m.Size != 0 {
+		data[i] = 0x18
+		i++
+		i = encodeVarintStructs(data, i, uint64(m.Size))
+	}
+	if m.Permissions != 0 {
+		data[i] = 0x20
+		i++
+		i = encodeVarintStructs(data, i, uint64(m.Permissions))
+	}
+	if m.Modified != 0 {
+		data[i] = 0x28
+		i++
+		i = encodeVarintStructs(data, i, uint64(m.Modified))
+	}
+	if m.Deleted {
+		data[i] = 0x30
+		i++
+		if m.Deleted {
+			data[i] = 1
+		} else {
+			data[i] = 0
+		}
+		i++
+	}
+	if m.Invalid {
+		data[i] = 0x38
+		i++
+		if m.Invalid {
+			data[i] = 1
+		} else {
+			data[i] = 0
+		}
+		i++
+	}
+	if m.NoPermissions {
+		data[i] = 0x40
+		i++
+		if m.NoPermissions {
+			data[i] = 1
+		} else {
+			data[i] = 0
+		}
+		i++
+	}
+	data[i] = 0x4a
+	i++
+	i = encodeVarintStructs(data, i, uint64(m.Version.ProtoSize()))
+	n2, err := m.Version.MarshalTo(data[i:])
+	if err != nil {
+		return 0, err
+	}
+	i += n2
+	if m.LocalVersion != 0 {
+		data[i] = 0x50
+		i++
+		i = encodeVarintStructs(data, i, uint64(m.LocalVersion))
+	}
+	return i, nil
+}
+
+func encodeFixed64Structs(data []byte, offset int, v uint64) int {
+	data[offset] = uint8(v)
+	data[offset+1] = uint8(v >> 8)
+	data[offset+2] = uint8(v >> 16)
+	data[offset+3] = uint8(v >> 24)
+	data[offset+4] = uint8(v >> 32)
+	data[offset+5] = uint8(v >> 40)
+	data[offset+6] = uint8(v >> 48)
+	data[offset+7] = uint8(v >> 56)
+	return offset + 8
+}
+func encodeFixed32Structs(data []byte, offset int, v uint32) int {
+	data[offset] = uint8(v)
+	data[offset+1] = uint8(v >> 8)
+	data[offset+2] = uint8(v >> 16)
+	data[offset+3] = uint8(v >> 24)
+	return offset + 4
+}
+func encodeVarintStructs(data []byte, offset int, v uint64) int {
+	for v >= 1<<7 {
+		data[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	data[offset] = uint8(v)
+	return offset + 1
+}
+func (m *FileVersion) ProtoSize() (n int) {
+	var l int
+	_ = l
+	l = m.Version.ProtoSize()
+	n += 1 + l + sovStructs(uint64(l))
+	l = len(m.Device)
+	if l > 0 {
+		n += 1 + l + sovStructs(uint64(l))
+	}
+	return n
+}
+
+func (m *VersionList) ProtoSize() (n int) {
+	var l int
+	_ = l
+	if len(m.Versions) > 0 {
+		for _, e := range m.Versions {
+			l = e.ProtoSize()
+			n += 1 + l + sovStructs(uint64(l))
+		}
+	}
+	return n
+}
+
+func (m *FileInfoTruncated) ProtoSize() (n int) {
+	var l int
+	_ = l
+	l = len(m.Name)
+	if l > 0 {
+		n += 1 + l + sovStructs(uint64(l))
+	}
+	if m.Type != 0 {
+		n += 1 + sovStructs(uint64(m.Type))
+	}
+	if m.Size != 0 {
+		n += 1 + sovStructs(uint64(m.Size))
+	}
+	if m.Permissions != 0 {
+		n += 1 + sovStructs(uint64(m.Permissions))
+	}
+	if m.Modified != 0 {
+		n += 1 + sovStructs(uint64(m.Modified))
+	}
+	if m.Deleted {
+		n += 2
+	}
+	if m.Invalid {
+		n += 2
+	}
+	if m.NoPermissions {
+		n += 2
+	}
+	l = m.Version.ProtoSize()
+	n += 1 + l + sovStructs(uint64(l))
+	if m.LocalVersion != 0 {
+		n += 1 + sovStructs(uint64(m.LocalVersion))
+	}
+	return n
+}
+
+func sovStructs(x uint64) (n int) {
+	for {
+		n++
+		x >>= 7
+		if x == 0 {
+			break
+		}
+	}
+	return n
+}
+func sozStructs(x uint64) (n int) {
+	return sovStructs(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (m *FileVersion) Unmarshal(data []byte) error {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowStructs
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: FileVersion: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: FileVersion: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if err := m.Version.Unmarshal(data[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Device", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				byteLen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Device = append(m.Device[:0], data[iNdEx:postIndex]...)
+			if m.Device == nil {
+				m.Device = []byte{}
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipStructs(data[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *VersionList) Unmarshal(data []byte) error {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowStructs
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: VersionList: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: VersionList: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Versions", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Versions = append(m.Versions, FileVersion{})
+			if err := m.Versions[len(m.Versions)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipStructs(data[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *FileInfoTruncated) Unmarshal(data []byte) error {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowStructs
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: FileInfoTruncated: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: FileInfoTruncated: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Name = string(data[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType)
+			}
+			m.Type = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.Type |= (protocol.FileInfoType(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Size", wireType)
+			}
+			m.Size = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.Size |= (int64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 4:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Permissions", wireType)
+			}
+			m.Permissions = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.Permissions |= (uint32(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 5:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Modified", wireType)
+			}
+			m.Modified = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.Modified |= (int64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 6:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Deleted", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				v |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.Deleted = bool(v != 0)
+		case 7:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Invalid", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				v |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.Invalid = bool(v != 0)
+		case 8:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field NoPermissions", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				v |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.NoPermissions = bool(v != 0)
+		case 9:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if err := m.Version.Unmarshal(data[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 10:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field LocalVersion", wireType)
+			}
+			m.LocalVersion = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.LocalVersion |= (int64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipStructs(data[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func skipStructs(data []byte) (n int, err error) {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return 0, ErrIntOverflowStructs
+			}
+			if iNdEx >= l {
+				return 0, io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		wireType := int(wire & 0x7)
+		switch wireType {
+		case 0:
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				iNdEx++
+				if data[iNdEx-1] < 0x80 {
+					break
+				}
+			}
+			return iNdEx, nil
+		case 1:
+			iNdEx += 8
+			return iNdEx, nil
+		case 2:
+			var length int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				length |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			iNdEx += length
+			if length < 0 {
+				return 0, ErrInvalidLengthStructs
+			}
+			return iNdEx, nil
+		case 3:
+			for {
+				var innerWire uint64
+				var start int = iNdEx
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return 0, ErrIntOverflowStructs
+					}
+					if iNdEx >= l {
+						return 0, io.ErrUnexpectedEOF
+					}
+					b := data[iNdEx]
+					iNdEx++
+					innerWire |= (uint64(b) & 0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				innerWireType := int(innerWire & 0x7)
+				if innerWireType == 4 {
+					break
+				}
+				next, err := skipStructs(data[start:])
+				if err != nil {
+					return 0, err
+				}
+				iNdEx = start + next
+			}
+			return iNdEx, nil
+		case 4:
+			return iNdEx, nil
+		case 5:
+			iNdEx += 4
+			return iNdEx, nil
+		default:
+			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+		}
+	}
+	panic("unreachable")
+}
+
+var (
+	ErrInvalidLengthStructs = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowStructs   = fmt.Errorf("proto: integer overflow")
+)
+
+var fileDescriptorStructs = []byte{
+	// 401 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x51, 0x4f, 0xcb, 0xd3, 0x30,
+	0x1c, 0x6e, 0xb7, 0xba, 0xf5, 0x4d, 0xdf, 0x4e, 0x0d, 0x32, 0xca, 0x0e, 0xdd, 0x98, 0x08, 0x22,
+	0xd8, 0xe9, 0xc4, 0x8b, 0xc7, 0x1d, 0x06, 0x82, 0x07, 0x29, 0x32, 0x8f, 0xa3, 0x4d, 0xb2, 0x2e,
+	0xd0, 0x26, 0xa5, 0x49, 0x07, 0xf3, 0x93, 0x78, 0xdc, 0xc7, 0xd9, 0xd1, 0x2f, 0xa0, 0xe8, 0xfc,
+	0x22, 0x66, 0x49, 0x3b, 0x7b, 0x7c, 0x0f, 0x81, 0xdf, 0x93, 0xe7, 0xcf, 0xef, 0x21, 0x01, 0xbe,
+	0x90, 0x55, 0x8d, 0xa4, 0x88, 0xca, 0x8a, 0x4b, 0x0e, 0x7b, 0x38, 0x9d, 0xbc, 0xce, 0xa8, 0xdc,
+	0xd7, 0x69, 0x84, 0x78, 0xb1, 0xc8, 0x78, 0xc6, 0x17, 0x9a, 0x4a, 0xeb, 0x9d, 0x46, 0x1a, 0xe8,
+	0xc9, 0x58, 0x26, 0xef, 0x3b, 0x72, 0x71, 0x64, 0x48, 0xee, 0x29, 0xcb, 0x3a, 0x53, 0x4e, 0x53,
+	0x93, 0x80, 0x78, 0xbe, 0x48, 0x49, 0x69, 0x6c, 0xf3, 0xaf, 0xc0, 0x5b, 0xd3, 0x9c, 0x6c, 0x48,
+	0x25, 0x28, 0x67, 0xf0, 0x0d, 0x18, 0x1e, 0xcc, 0x18, 0xd8, 0x33, 0xfb, 0xa5, 0xb7, 0x7c, 0x12,
+	0xb5, 0xa6, 0x68, 0x43, 0x90, 0xe4, 0xd5, 0xca, 0x39, 0xff, 0x9a, 0x5a, 0x71, 0x2b, 0x83, 0x63,
+	0x30, 0xc0, 0xe4, 0x40, 0x11, 0x09, 0x7a, 0xca, 0x70, 0x1f, 0x37, 0x68, 0xbe, 0x06, 0x5e, 0x13,
+	0xfa, 0x89, 0x0a, 0x09, 0xdf, 0x02, 0xb7, 0x71, 0x08, 0x95, 0xdc, 0x57, 0xc9, 0x8f, 0x23, 0x9c,
+	0x46, 0x9d, 0xdd, 0x4d, 0xf0, 0x4d, 0xf6, 0xc1, 0xf9, 0x7e, 0x9a, 0x5a, 0xf3, 0x9f, 0x3d, 0xf0,
+	0xf4, 0xaa, 0xfa, 0xc8, 0x76, 0xfc, 0x4b, 0x55, 0x33, 0x94, 0x48, 0x82, 0x21, 0x04, 0x0e, 0x4b,
+	0x0a, 0xa2, 0x4b, 0xde, 0xc5, 0x7a, 0x86, 0xaf, 0x80, 0x23, 0x8f, 0xa5, 0xe9, 0x31, 0x5a, 0x8e,
+	0xff, 0x17, 0xbf, 0xd9, 0x15, 0x1b, 0x6b, 0xcd, 0xd5, 0x2f, 0xe8, 0x37, 0x12, 0xf4, 0x95, 0xb6,
+	0x1f, 0xeb, 0x19, 0xce, 0x80, 0x57, 0x92, 0xaa, 0xa0, 0xc2, 0xb4, 0x74, 0x14, 0xe5, 0xc7, 0xdd,
+	0x2b, 0x38, 0x01, 0x6e, 0xc1, 0x31, 0xdd, 0x51, 0x82, 0x83, 0x47, 0xda, 0x79, 0xc3, 0x30, 0x00,
+	0x43, 0x4c, 0x72, 0xa2, 0xca, 0x05, 0x03, 0x45, 0xb9, 0x71, 0x0b, 0xaf, 0x0c, 0x65, 0x87, 0x24,
+	0xa7, 0x38, 0x18, 0x1a, 0xa6, 0x81, 0xf0, 0x05, 0x18, 0x31, 0xbe, 0xed, 0x2e, 0x75, 0xb5, 0xc0,
+	0x67, 0xfc, 0x73, 0x67, 0x6d, 0xe7, 0x53, 0xee, 0x1e, 0xf6, 0x29, 0xcf, 0x81, 0x9f, 0x73, 0x94,
+	0xe4, 0xdb, 0xd6, 0x07, 0x74, 0xdb, 0x7b, 0x7d, 0xd9, 0xbc, 0xb7, 0x79, 0xdf, 0xd5, 0xb3, 0xf3,
+	0x9f, 0xd0, 0x3a, 0x5f, 0x42, 0xfb, 0x87, 0x3a, 0xbf, 0x2f, 0xa1, 0x75, 0xfa, 0x1b, 0xda, 0xe9,
+	0x40, 0x2f, 0x78, 0xf7, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xe4, 0xb1, 0x7f, 0x07, 0x98, 0x02, 0x00,
+	0x00,
+}

lib/db/structs.proto

@@ -0,0 +1,35 @@
+syntax = "proto3";
+
+package db;
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+import "github.com/syncthing/syncthing/lib/protocol/bep.proto";
+
+option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.sizer_all) = false;
+option (gogoproto.protosizer_all) = true;
+
+message FileVersion {
+    protocol.Vector version = 1 [(gogoproto.nullable) = false];
+    bytes           device  = 2;
+}
+
+message VersionList {
+    option (gogoproto.goproto_stringer) = false;
+    repeated FileVersion versions = 1 [(gogoproto.nullable) = false];
+}
+
+// Must be the same as FileInfo but without the blocks field
+message FileInfoTruncated {
+    option (gogoproto.goproto_stringer) = false;
+    string                name           = 1;
+    protocol.FileInfoType type           = 2;
+    int64                 size           = 3;
+    uint32                permissions    = 4;
+    int64                 modified       = 5;
+    bool                  deleted        = 6;
+    bool                  invalid        = 7;
+    bool                  no_permissions = 8;
+    protocol.Vector       version        = 9 [(gogoproto.nullable) = false];
+    int64                 local_version  = 10;
+}

lib/db/truncated.go

@@ -1,52 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package db
-
-import (
-	"github.com/calmh/xdr"
-	"github.com/syncthing/syncthing/lib/protocol"
-)
-
-type FileInfoTruncated struct {
-	protocol.FileInfo
-}
-
-func (o *FileInfoTruncated) UnmarshalXDR(bs []byte) error {
-	return o.UnmarshalXDRFrom(&xdr.Unmarshaller{Data: bs})
-}
-
-func (o *FileInfoTruncated) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	o.Name = u.UnmarshalStringMax(8192)
-	o.Flags = u.UnmarshalUint32()
-	o.Modified = int64(u.UnmarshalUint64())
-	(&o.Version).UnmarshalXDRFrom(u)
-	o.LocalVersion = int64(u.UnmarshalUint64())
-	_BlocksSize := int(u.UnmarshalUint32())
-	if _BlocksSize < 0 {
-		return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 10000000)
-	} else if _BlocksSize == 0 {
-		o.Blocks = nil
-	} else {
-		if _BlocksSize > 10000000 {
-			return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 10000000)
-		}
-		for i := 0; i < _BlocksSize; i++ {
-			size := int64(u.UnmarshalUint32())
-			o.CachedSize += size
-			u.UnmarshalBytes()
-		}
-	}
-
-	return u.Error
-}
-
-func BlocksToSize(num int) int64 {
-	if num < 2 {
-		return protocol.BlockSize / 2
-	}
-	return int64(num-1)*protocol.BlockSize + protocol.BlockSize/2
-}

lib/discover/discover.go

@@ -26,6 +26,7 @@ type CacheEntry struct {
 	when       time.Time // When did we get the result
 	found      bool      // Is it a success (cacheTime applies) or a failure (negCacheTime applies)?
 	validUntil time.Time // Validity time, overrides normal calculation
+	instanceID int64     // for local discovery, the instance ID (random on each restart)
 }
 
 // A FinderService is a Finder that has background activity and must be run as

lib/discover/local.go

@@ -4,10 +4,14 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at http://mozilla.org/MPL/2.0/.
 
+//go:generate go run ../../script/protofmt.go local.proto
+//go:generate protoc --proto_path=../../../../../:../../../../gogo/protobuf/protobuf:. --gogofast_out=. local.proto
+
 package discover
 
 import (
 	"bytes"
+	"encoding/binary"
 	"encoding/hex"
 	"io"
 	"net"
@@ -18,6 +22,7 @@ import (
 	"github.com/syncthing/syncthing/lib/beacon"
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/thejerf/suture"
 )
 
@@ -38,6 +43,8 @@ type localClient struct {
 const (
 	BroadcastInterval = 30 * time.Second
 	CacheLifeTime     = 3 * BroadcastInterval
+	Magic             = uint32(0x2EA7D90B) // same as in BEP
+	v13Magic          = uint32(0x7D79BC40) // previous version
 )
 
 func NewLocal(id protocol.DeviceID, addr string, addrList AddressLister) (FinderService, error) {
@@ -107,25 +114,20 @@ func (c *localClient) Error() error {
 }
 
 func (c *localClient) announcementPkt() Announce {
-	var addrs []Address
-	for _, addr := range c.addrList.AllAddresses() {
-		addrs = append(addrs, Address{
-			URL: addr,
-		})
-	}
-
 	return Announce{
-		Magic: AnnouncementMagic,
-		This: Device{
-			ID:        c.myID[:],
-			Addresses: addrs,
-		},
+		ID:         c.myID[:],
+		Addresses:  c.addrList.AllAddresses(),
+		InstanceID: rand.Int63(),
 	}
 }
 
 func (c *localClient) sendLocalAnnouncements() {
+	msg := make([]byte, 4)
+	binary.BigEndian.PutUint32(msg, Magic)
+
 	var pkt = c.announcementPkt()
-	msg := pkt.MustMarshalXDR()
+	bs, _ := pkt.Marshal()
+	msg = append(msg, bs...)
 
 	for {
 		c.beacon.Send(msg)
@@ -138,26 +140,44 @@ func (c *localClient) sendLocalAnnouncements() {
 }
 
 func (c *localClient) recvAnnouncements(b beacon.Interface) {
+	warnedAbout := make(map[string]bool)
 	for {
 		buf, addr := b.Recv()
+		if len(buf) < 4 {
+			l.Debugf("discover: short packet from %s")
+			continue
+		}
+
+		magic := binary.BigEndian.Uint32(buf)
+		switch magic {
+		case Magic:
+			// All good
+
+		case v13Magic:
+			// Old version
+			if !warnedAbout[addr.String()] {
+				l.Warnf("Incompatible (v0.13) local discovery packet from %v - upgrade that device to connect", addr)
+				warnedAbout[addr.String()] = true
+			}
+			continue
+
+		default:
+			l.Debugf("discover: Incorrect magic %x from %s", magic, addr)
+			continue
+		}
 
 		var pkt Announce
-		err := pkt.UnmarshalXDR(buf)
+		err := pkt.Unmarshal(buf[4:])
 		if err != nil && err != io.EOF {
 			l.Debugf("discover: Failed to unmarshal local announcement from %s:\n%s", addr, hex.Dump(buf))
 			continue
 		}
 
-		if pkt.Magic != AnnouncementMagic {
-			l.Debugf("discover: Incorrect magic from %s: %s != %s", addr, pkt.Magic, AnnouncementMagic)
-			continue
-		}
-
-		l.Debugf("discover: Received local announcement from %s for %s", addr, protocol.DeviceIDFromBytes(pkt.This.ID))
+		l.Debugf("discover: Received local announcement from %s for %s", addr, protocol.DeviceIDFromBytes(pkt.ID))
 
 		var newDevice bool
-		if !bytes.Equal(pkt.This.ID, c.myID[:]) {
-			newDevice = c.registerDevice(addr, pkt.This)
+		if !bytes.Equal(pkt.ID, c.myID[:]) {
+			newDevice = c.registerDevice(addr, pkt)
 		}
 
 		if newDevice {
@@ -171,14 +191,16 @@ func (c *localClient) recvAnnouncements(b beacon.Interface) {
 	}
 }
 
-func (c *localClient) registerDevice(src net.Addr, device Device) bool {
+func (c *localClient) registerDevice(src net.Addr, device Announce) bool {
 	var id protocol.DeviceID
 	copy(id[:], device.ID)
 
 	// Remember whether we already had a valid cache entry for this device.
+	// If the instance ID has changed the remote device has restarted since
+	// we last heard from it, so we should treat it as a new device.
 
 	ce, existsAlready := c.Get(id)
-	isNewDevice := !existsAlready || time.Since(ce.when) > CacheLifeTime
+	isNewDevice := !existsAlready || time.Since(ce.when) > CacheLifeTime || ce.instanceID != device.InstanceID
 
 	// Any empty or unspecified addresses should be set to the source address
 	// of the announcement. We also skip any addresses we can't parse.
@@ -186,7 +208,7 @@ func (c *localClient) registerDevice(src net.Addr, device Device) bool {
 	l.Debugln("discover: Registering addresses for", id)
 	var validAddresses []string
 	for _, addr := range device.Addresses {
-		u, err := url.Parse(addr.URL)
+		u, err := url.Parse(addr)
 		if err != nil {
 			continue
 		}
@@ -197,6 +219,21 @@ func (c *localClient) registerDevice(src net.Addr, device Device) bool {
 		}
 
 		if len(tcpAddr.IP) == 0 || tcpAddr.IP.IsUnspecified() {
+			srcAddr, err := net.ResolveTCPAddr("tcp", src.String())
+			if err != nil {
+				continue
+			}
+
+			// Do not use IPv6 source address if requested scheme is tcp4
+			if u.Scheme == "tcp4" && srcAddr.IP.To4() == nil {
+				continue
+			}
+
+			// Do not use IPv4 source address if requested scheme is tcp6
+			if u.Scheme == "tcp6" && srcAddr.IP.To4() != nil {
+				continue
+			}
+
 			host, _, err := net.SplitHostPort(src.String())
 			if err != nil {
 				continue
@@ -204,17 +241,18 @@ func (c *localClient) registerDevice(src net.Addr, device Device) bool {
 			u.Host = net.JoinHostPort(host, strconv.Itoa(tcpAddr.Port))
 			l.Debugf("discover: Reconstructed URL is %#v", u)
 			validAddresses = append(validAddresses, u.String())
-			l.Debugf("discover: Replaced address %v in %s to get %s", tcpAddr.IP, addr.URL, u.String())
+			l.Debugf("discover: Replaced address %v in %s to get %s", tcpAddr.IP, addr, u.String())
 		} else {
-			validAddresses = append(validAddresses, addr.URL)
-			l.Debugf("discover: Accepted address %s verbatim", addr.URL)
+			validAddresses = append(validAddresses, addr)
+			l.Debugf("discover: Accepted address %s verbatim", addr)
 		}
 	}
 
 	c.Set(id, CacheEntry{
-		Addresses: validAddresses,
-		when:      time.Now(),
-		found:     true,
+		Addresses:  validAddresses,
+		when:       time.Now(),
+		found:      true,
+		instanceID: device.InstanceID,
 	})
 
 	if isNewDevice {

lib/discover/local.pb.go

@@ -0,0 +1,398 @@
+// Code generated by protoc-gen-gogo.
+// source: local.proto
+// DO NOT EDIT!
+
+/*
+	Package discover is a generated protocol buffer package.
+
+	It is generated from these files:
+		local.proto
+
+	It has these top-level messages:
+		Announce
+*/
+package discover
+
+import proto "github.com/gogo/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import _ "github.com/gogo/protobuf/gogoproto"
+
+import io "io"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+const _ = proto.GoGoProtoPackageIsVersion1
+
+type Announce struct {
+	ID         []byte   `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
+	Addresses  []string `protobuf:"bytes,2,rep,name=addresses" json:"addresses,omitempty"`
+	InstanceID int64    `protobuf:"varint,3,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"`
+}
+
+func (m *Announce) Reset()                    { *m = Announce{} }
+func (m *Announce) String() string            { return proto.CompactTextString(m) }
+func (*Announce) ProtoMessage()               {}
+func (*Announce) Descriptor() ([]byte, []int) { return fileDescriptorLocal, []int{0} }
+
+func init() {
+	proto.RegisterType((*Announce)(nil), "discover.Announce")
+}
+func (m *Announce) Marshal() (data []byte, err error) {
+	size := m.ProtoSize()
+	data = make([]byte, size)
+	n, err := m.MarshalTo(data)
+	if err != nil {
+		return nil, err
+	}
+	return data[:n], nil
+}
+
+func (m *Announce) MarshalTo(data []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.ID) > 0 {
+		data[i] = 0xa
+		i++
+		i = encodeVarintLocal(data, i, uint64(len(m.ID)))
+		i += copy(data[i:], m.ID)
+	}
+	if len(m.Addresses) > 0 {
+		for _, s := range m.Addresses {
+			data[i] = 0x12
+			i++
+			l = len(s)
+			for l >= 1<<7 {
+				data[i] = uint8(uint64(l)&0x7f | 0x80)
+				l >>= 7
+				i++
+			}
+			data[i] = uint8(l)
+			i++
+			i += copy(data[i:], s)
+		}
+	}
+	if m.InstanceID != 0 {
+		data[i] = 0x18
+		i++
+		i = encodeVarintLocal(data, i, uint64(m.InstanceID))
+	}
+	return i, nil
+}
+
+func encodeFixed64Local(data []byte, offset int, v uint64) int {
+	data[offset] = uint8(v)
+	data[offset+1] = uint8(v >> 8)
+	data[offset+2] = uint8(v >> 16)
+	data[offset+3] = uint8(v >> 24)
+	data[offset+4] = uint8(v >> 32)
+	data[offset+5] = uint8(v >> 40)
+	data[offset+6] = uint8(v >> 48)
+	data[offset+7] = uint8(v >> 56)
+	return offset + 8
+}
+func encodeFixed32Local(data []byte, offset int, v uint32) int {
+	data[offset] = uint8(v)
+	data[offset+1] = uint8(v >> 8)
+	data[offset+2] = uint8(v >> 16)
+	data[offset+3] = uint8(v >> 24)
+	return offset + 4
+}
+func encodeVarintLocal(data []byte, offset int, v uint64) int {
+	for v >= 1<<7 {
+		data[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	data[offset] = uint8(v)
+	return offset + 1
+}
+func (m *Announce) ProtoSize() (n int) {
+	var l int
+	_ = l
+	l = len(m.ID)
+	if l > 0 {
+		n += 1 + l + sovLocal(uint64(l))
+	}
+	if len(m.Addresses) > 0 {
+		for _, s := range m.Addresses {
+			l = len(s)
+			n += 1 + l + sovLocal(uint64(l))
+		}
+	}
+	if m.InstanceID != 0 {
+		n += 1 + sovLocal(uint64(m.InstanceID))
+	}
+	return n
+}
+
+func sovLocal(x uint64) (n int) {
+	for {
+		n++
+		x >>= 7
+		if x == 0 {
+			break
+		}
+	}
+	return n
+}
+func sozLocal(x uint64) (n int) {
+	return sovLocal(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (m *Announce) Unmarshal(data []byte) error {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowLocal
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: Announce: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: Announce: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowLocal
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				byteLen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthLocal
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.ID = append(m.ID[:0], data[iNdEx:postIndex]...)
+			if m.ID == nil {
+				m.ID = []byte{}
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowLocal
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthLocal
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Addresses = append(m.Addresses, string(data[iNdEx:postIndex]))
+			iNdEx = postIndex
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field InstanceID", wireType)
+			}
+			m.InstanceID = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowLocal
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.InstanceID |= (int64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipLocal(data[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthLocal
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func skipLocal(data []byte) (n int, err error) {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return 0, ErrIntOverflowLocal
+			}
+			if iNdEx >= l {
+				return 0, io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		wireType := int(wire & 0x7)
+		switch wireType {
+		case 0:
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowLocal
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				iNdEx++
+				if data[iNdEx-1] < 0x80 {
+					break
+				}
+			}
+			return iNdEx, nil
+		case 1:
+			iNdEx += 8
+			return iNdEx, nil
+		case 2:
+			var length int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowLocal
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				length |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			iNdEx += length
+			if length < 0 {
+				return 0, ErrInvalidLengthLocal
+			}
+			return iNdEx, nil
+		case 3:
+			for {
+				var innerWire uint64
+				var start int = iNdEx
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return 0, ErrIntOverflowLocal
+					}
+					if iNdEx >= l {
+						return 0, io.ErrUnexpectedEOF
+					}
+					b := data[iNdEx]
+					iNdEx++
+					innerWire |= (uint64(b) & 0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				innerWireType := int(innerWire & 0x7)
+				if innerWireType == 4 {
+					break
+				}
+				next, err := skipLocal(data[start:])
+				if err != nil {
+					return 0, err
+				}
+				iNdEx = start + next
+			}
+			return iNdEx, nil
+		case 4:
+			return iNdEx, nil
+		case 5:
+			iNdEx += 4
+			return iNdEx, nil
+		default:
+			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+		}
+	}
+	panic("unreachable")
+}
+
+var (
+	ErrInvalidLengthLocal = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowLocal   = fmt.Errorf("proto: integer overflow")
+)
+
+var fileDescriptorLocal = []byte{
+	// 194 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xce, 0xc9, 0x4f, 0x4e,
+	0xcc, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x48, 0xc9, 0x2c, 0x4e, 0xce, 0x2f, 0x4b,
+	0x2d, 0x92, 0xd2, 0x4d, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf,
+	0x4f, 0xcf, 0xd7, 0x07, 0x2b, 0x48, 0x2a, 0x4d, 0x03, 0xf3, 0xc0, 0x1c, 0x30, 0x0b, 0xa2, 0x51,
+	0xa9, 0x90, 0x8b, 0xc3, 0x31, 0x2f, 0x2f, 0xbf, 0x34, 0x2f, 0x39, 0x55, 0x48, 0x8c, 0x8b, 0x29,
+	0x33, 0x45, 0x82, 0x51, 0x81, 0x51, 0x83, 0xc7, 0x89, 0xed, 0xd1, 0x3d, 0x79, 0x26, 0x4f, 0x97,
+	0x20, 0xa0, 0x88, 0x90, 0x0c, 0x17, 0x67, 0x62, 0x4a, 0x4a, 0x51, 0x6a, 0x71, 0x71, 0x6a, 0xb1,
+	0x04, 0x93, 0x02, 0xb3, 0x06, 0x67, 0x10, 0x42, 0x40, 0x48, 0x9f, 0x8b, 0x3b, 0x33, 0xaf, 0xb8,
+	0x24, 0x11, 0x68, 0x42, 0x3c, 0x50, 0x3b, 0x33, 0x50, 0x3b, 0xb3, 0x13, 0x1f, 0x50, 0x3b, 0x97,
+	0x27, 0x54, 0x18, 0x68, 0x0c, 0x17, 0x4c, 0x89, 0x67, 0x8a, 0x93, 0xc8, 0x89, 0x87, 0x72, 0x0c,
+	0x27, 0x1e, 0xc9, 0x31, 0x5e, 0x00, 0xe2, 0x07, 0x8f, 0xe4, 0x18, 0x16, 0x3c, 0x96, 0x63, 0x4c,
+	0x62, 0x03, 0xbb, 0xc7, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff, 0x91, 0x3f, 0x96, 0x25, 0xd7, 0x00,
+	0x00, 0x00,
+}

lib/discover/local.proto

@@ -0,0 +1,15 @@
+syntax = "proto3";
+
+package discover;
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.sizer_all) = false;
+option (gogoproto.protosizer_all) = true;
+
+message Announce {
+    bytes           id          = 1 [(gogoproto.customname) = "ID"];
+    repeated string addresses   = 2;
+    int64           instance_id = 3 [(gogoproto.customname) = "InstanceID"];
+}

lib/discover/local_test.go

@@ -0,0 +1,81 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package discover
+
+import (
+	"net"
+	"testing"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+func TestRandomLocalInstanceID(t *testing.T) {
+	c, err := NewLocal(protocol.LocalDeviceID, ":0", &fakeAddressLister{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	go c.Serve()
+	defer c.Stop()
+
+	lc := c.(*localClient)
+
+	p0 := lc.announcementPkt()
+	p1 := lc.announcementPkt()
+	if p0.InstanceID == p1.InstanceID {
+		t.Error("each generated packet should have a new instance id")
+	}
+}
+
+func TestLocalInstanceIDShouldTriggerNew(t *testing.T) {
+	c, err := NewLocal(protocol.LocalDeviceID, ":0", &fakeAddressLister{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	lc := c.(*localClient)
+	src := &net.UDPAddr{IP: []byte{10, 20, 30, 40}, Port: 50}
+
+	new := lc.registerDevice(src, Announce{
+		ID:         []byte{10, 20, 30, 40, 50, 60, 70, 80, 90},
+		Addresses:  []string{"tcp://0.0.0.0:22000"},
+		InstanceID: 1234567890,
+	})
+
+	if !new {
+		t.Fatal("first register should be new")
+	}
+
+	new = lc.registerDevice(src, Announce{
+		ID:         []byte{10, 20, 30, 40, 50, 60, 70, 80, 90},
+		Addresses:  []string{"tcp://0.0.0.0:22000"},
+		InstanceID: 1234567890,
+	})
+
+	if new {
+		t.Fatal("second register should not be new")
+	}
+
+	new = lc.registerDevice(src, Announce{
+		ID:         []byte{42, 10, 20, 30, 40, 50, 60, 70, 80, 90},
+		Addresses:  []string{"tcp://0.0.0.0:22000"},
+		InstanceID: 1234567890,
+	})
+
+	if !new {
+		t.Fatal("new device ID should be new")
+	}
+
+	new = lc.registerDevice(src, Announce{
+		ID:         []byte{10, 20, 30, 40, 50, 60, 70, 80, 90},
+		Addresses:  []string{"tcp://0.0.0.0:22000"},
+		InstanceID: 91234567890,
+	})
+
+	if !new {
+		t.Fatal("new instance ID should be new")
+	}
+}

lib/discover/localpackets.go

@@ -1,29 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-//go:generate -command genxdr go run ../../vendor/github.com/calmh/xdr/cmd/genxdr/main.go
-//go:generate genxdr -o localpackets_xdr.go localpackets.go
-
-package discover
-
-const (
-	AnnouncementMagic = 0x7D79BC40
-)
-
-type Announce struct {
-	Magic uint32
-	This  Device
-	Extra []Device // max:16
-}
-
-type Device struct {
-	ID        []byte    // max:32
-	Addresses []Address // max:16
-}
-
-type Address struct {
-	URL string // max:2083
-}

lib/discover/localpackets_xdr.go

@@ -1,246 +0,0 @@
-// ************************************************************
-// This file is automatically generated by genxdr. Do not edit.
-// ************************************************************
-
-package discover
-
-import (
-	"github.com/calmh/xdr"
-)
-
-/*
-
-Announce Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             Magic                             |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                       Device Structure                        \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                        Number of Extra                        |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                Zero or more Device Structures                 \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct Announce {
-	unsigned int Magic;
-	Device This;
-	Device Extra<16>;
-}
-
-*/
-
-func (o Announce) XDRSize() int {
-	return 4 +
-		o.This.XDRSize() +
-		4 + xdr.SizeOfSlice(o.Extra)
-}
-
-func (o Announce) MarshalXDR() ([]byte, error) {
-	buf := make([]byte, o.XDRSize())
-	m := &xdr.Marshaller{Data: buf}
-	return buf, o.MarshalXDRInto(m)
-}
-
-func (o Announce) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o Announce) MarshalXDRInto(m *xdr.Marshaller) error {
-	m.MarshalUint32(o.Magic)
-	if err := o.This.MarshalXDRInto(m); err != nil {
-		return err
-	}
-	if l := len(o.Extra); l > 16 {
-		return xdr.ElementSizeExceeded("Extra", l, 16)
-	}
-	m.MarshalUint32(uint32(len(o.Extra)))
-	for i := range o.Extra {
-		if err := o.Extra[i].MarshalXDRInto(m); err != nil {
-			return err
-		}
-	}
-	return m.Error
-}
-
-func (o *Announce) UnmarshalXDR(bs []byte) error {
-	u := &xdr.Unmarshaller{Data: bs}
-	return o.UnmarshalXDRFrom(u)
-}
-func (o *Announce) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	o.Magic = u.UnmarshalUint32()
-	(&o.This).UnmarshalXDRFrom(u)
-	_ExtraSize := int(u.UnmarshalUint32())
-	if _ExtraSize < 0 {
-		return xdr.ElementSizeExceeded("Extra", _ExtraSize, 16)
-	} else if _ExtraSize == 0 {
-		o.Extra = nil
-	} else {
-		if _ExtraSize > 16 {
-			return xdr.ElementSizeExceeded("Extra", _ExtraSize, 16)
-		}
-		if _ExtraSize <= len(o.Extra) {
-			o.Extra = o.Extra[:_ExtraSize]
-		} else {
-			o.Extra = make([]Device, _ExtraSize)
-		}
-		for i := range o.Extra {
-			(&o.Extra[i]).UnmarshalXDRFrom(u)
-		}
-	}
-	return u.Error
-}
-
-/*
-
-Device Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                   ID (length + padded data)                   \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                      Number of Addresses                      |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                Zero or more Address Structures                \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct Device {
-	opaque ID<32>;
-	Address Addresses<16>;
-}
-
-*/
-
-func (o Device) XDRSize() int {
-	return 4 + len(o.ID) + xdr.Padding(len(o.ID)) +
-		4 + xdr.SizeOfSlice(o.Addresses)
-}
-
-func (o Device) MarshalXDR() ([]byte, error) {
-	buf := make([]byte, o.XDRSize())
-	m := &xdr.Marshaller{Data: buf}
-	return buf, o.MarshalXDRInto(m)
-}
-
-func (o Device) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o Device) MarshalXDRInto(m *xdr.Marshaller) error {
-	if l := len(o.ID); l > 32 {
-		return xdr.ElementSizeExceeded("ID", l, 32)
-	}
-	m.MarshalBytes(o.ID)
-	if l := len(o.Addresses); l > 16 {
-		return xdr.ElementSizeExceeded("Addresses", l, 16)
-	}
-	m.MarshalUint32(uint32(len(o.Addresses)))
-	for i := range o.Addresses {
-		if err := o.Addresses[i].MarshalXDRInto(m); err != nil {
-			return err
-		}
-	}
-	return m.Error
-}
-
-func (o *Device) UnmarshalXDR(bs []byte) error {
-	u := &xdr.Unmarshaller{Data: bs}
-	return o.UnmarshalXDRFrom(u)
-}
-func (o *Device) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	o.ID = u.UnmarshalBytesMax(32)
-	_AddressesSize := int(u.UnmarshalUint32())
-	if _AddressesSize < 0 {
-		return xdr.ElementSizeExceeded("Addresses", _AddressesSize, 16)
-	} else if _AddressesSize == 0 {
-		o.Addresses = nil
-	} else {
-		if _AddressesSize > 16 {
-			return xdr.ElementSizeExceeded("Addresses", _AddressesSize, 16)
-		}
-		if _AddressesSize <= len(o.Addresses) {
-			o.Addresses = o.Addresses[:_AddressesSize]
-		} else {
-			o.Addresses = make([]Address, _AddressesSize)
-		}
-		for i := range o.Addresses {
-			(&o.Addresses[i]).UnmarshalXDRFrom(u)
-		}
-	}
-	return u.Error
-}
-
-/*
-
-Address Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                  URL (length + padded data)                   \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct Address {
-	string URL<2083>;
-}
-
-*/
-
-func (o Address) XDRSize() int {
-	return 4 + len(o.URL) + xdr.Padding(len(o.URL))
-}
-
-func (o Address) MarshalXDR() ([]byte, error) {
-	buf := make([]byte, o.XDRSize())
-	m := &xdr.Marshaller{Data: buf}
-	return buf, o.MarshalXDRInto(m)
-}
-
-func (o Address) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o Address) MarshalXDRInto(m *xdr.Marshaller) error {
-	if l := len(o.URL); l > 2083 {
-		return xdr.ElementSizeExceeded("URL", l, 2083)
-	}
-	m.MarshalString(o.URL)
-	return m.Error
-}
-
-func (o *Address) UnmarshalXDR(bs []byte) error {
-	u := &xdr.Unmarshaller{Data: bs}
-	return o.UnmarshalXDRFrom(u)
-}
-func (o *Address) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	o.URL = u.UnmarshalStringMax(2083)
-	return u.Error
-}

lib/model/model.go

@@ -8,6 +8,7 @@ package model
 
 import (
 	"bufio"
+	"bytes"
 	"crypto/tls"
 	"encoding/json"
 	"errors"
@@ -40,10 +41,8 @@ import (
 
 // How many files to send in each Index/IndexUpdate message.
 const (
-	indexTargetSize   = 250 * 1024 // Aim for making index messages no larger than 250 KiB (uncompressed)
-	indexPerFileSize  = 250        // Each FileInfo is approximately this big, in bytes, excluding BlockInfos
-	indexPerBlockSize = 40         // Each BlockInfo is approximately this big
-	indexBatchSize    = 1000       // Either way, don't include more files than this
+	indexTargetSize = 250 * 1024 // Aim for making index messages no larger than 250 KiB (uncompressed)
+	indexBatchSize  = 1000       // Either way, don't include more files than this
 )
 
 type service interface {
@@ -95,7 +94,7 @@ type Model struct {
 
 	conn              map[protocol.DeviceID]connections.Connection
 	helloMessages     map[protocol.DeviceID]protocol.HelloResult
-	deviceClusterConf map[protocol.DeviceID]protocol.ClusterConfigMessage
+	deviceClusterConf map[protocol.DeviceID]protocol.ClusterConfig
 	devicePaused      map[protocol.DeviceID]bool
 	deviceDownloads   map[protocol.DeviceID]*deviceDownloadState
 	pmut              sync.RWMutex // protects the above
@@ -149,7 +148,7 @@ func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName,
 		folderStatRefs:     make(map[string]*stats.FolderStatisticsReference),
 		conn:               make(map[protocol.DeviceID]connections.Connection),
 		helloMessages:      make(map[protocol.DeviceID]protocol.HelloResult),
-		deviceClusterConf:  make(map[protocol.DeviceID]protocol.ClusterConfigMessage),
+		deviceClusterConf:  make(map[protocol.DeviceID]protocol.ClusterConfig),
 		devicePaused:       make(map[protocol.DeviceID]bool),
 		deviceDownloads:    make(map[protocol.DeviceID]*deviceDownloadState),
 		fmut:               sync.NewRWMutex(),
@@ -414,7 +413,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
 		// This might might be more than it really is, because some blocks can be of a smaller size.
 		downloaded = int64(counts[ft.Name] * protocol.BlockSize)
 
-		fileNeed = ft.Size() - downloaded
+		fileNeed = ft.Size - downloaded
 		if fileNeed < 0 {
 			fileNeed = 0
 		}
@@ -436,7 +435,7 @@ func sizeOfFile(f db.FileIntf) (files, deleted int, bytes int64) {
 	} else {
 		deleted++
 	}
-	bytes += f.Size()
+	bytes += f.FileSize()
 	return
 }
 
@@ -548,12 +547,7 @@ func (m *Model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
 
 // Index is called when a new device is connected and we receive their full index.
 // Implements the protocol.Model interface.
-func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo, flags uint32, options []protocol.Option) {
-	if flags != 0 {
-		l.Warnln("protocol error: unknown flags 0x%x in Index message", flags)
-		return
-	}
-
+func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) {
 	l.Debugf("IDX(in): %s %q: %d files", deviceID, folder, len(fs))
 
 	if !m.folderSharedWith(folder, deviceID) {
@@ -595,12 +589,7 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F
 
 // IndexUpdate is called for incremental updates to connected devices' indexes.
 // Implements the protocol.Model interface.
-func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo, flags uint32, options []protocol.Option) {
-	if flags != 0 {
-		l.Warnln("protocol error: unknown flags 0x%x in IndexUpdate message", flags)
-		return
-	}
-
+func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) {
 	l.Debugf("%v IDXUP(in): %s / %q: %d files", m, deviceID, folder, len(fs))
 
 	if !m.folderSharedWith(folder, deviceID) {
@@ -651,7 +640,7 @@ func (m *Model) folderSharedWithUnlocked(folder string, deviceID protocol.Device
 	return false
 }
 
-func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterConfigMessage) {
+func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterConfig) {
 	// Check the peer device's announced folders against our own. Emits events
 	// for folders that we don't expect (unknown or not shared).
 	// Also, collect a list of folders we do share, and if he's interested in
@@ -659,19 +648,13 @@ func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterCon
 
 	tempIndexFolders := make([]string, 0, len(cm.Folders))
 
-	for _, folder := range cm.Folders {
-		if folder.Flags&^protocol.FlagFolderAll != 0 {
-			// There are flags set that we don't know what they mean. Fatal!
-			l.Warnf("Device %v: unknown flags for folder %s", deviceID, folder.ID)
-			m.fmut.Unlock()
-			m.Close(deviceID, fmt.Errorf("Unknown folder flags from device %v", deviceID))
-			return
-		}
+	m.pmut.RLock()
+	conn := m.conn[deviceID]
+	m.pmut.RUnlock()
 
-		m.fmut.Lock()
-		shared := m.folderSharedWithUnlocked(folder.ID, deviceID)
-		m.fmut.Unlock()
-		if !shared {
+	m.fmut.Lock()
+	for _, folder := range cm.Folders {
+		if !m.folderSharedWithUnlocked(folder.ID, deviceID) {
 			events.Default.Log(events.FolderRejected, map[string]string{
 				"folder":      folder.ID,
 				"folderLabel": folder.Label,
@@ -680,10 +663,76 @@ func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterCon
 			l.Infof("Unexpected folder ID %q sent from device %q; ensure that the folder exists and that this device is selected under \"Share With\" in the folder configuration.", folder.ID, deviceID)
 			continue
 		}
-		if folder.Flags&protocol.FlagFolderDisabledTempIndexes == 0 {
+		if !folder.DisableTempIndexes {
 			tempIndexFolders = append(tempIndexFolders, folder.ID)
 		}
+
+		fs := m.folderFiles[folder.ID]
+		myIndexID := fs.IndexID(protocol.LocalDeviceID)
+		myLocalVersion := fs.LocalVersion(protocol.LocalDeviceID)
+		var startLocalVersion int64
+
+		for _, dev := range folder.Devices {
+			if bytes.Equal(dev.ID, m.id[:]) {
+				// This is the other side's description of what it knows
+				// about us. Lets check to see if we can start sending index
+				// updates directly or need to send the index from start...
+
+				if dev.IndexID == myIndexID {
+					// They say they've seen our index ID before, so we can
+					// send a delta update only.
+
+					if dev.MaxLocalVersion > myLocalVersion {
+						// Safety check. They claim to have more or newer
+						// index data than we have - either we have lost
+						// index data, or reset the index without resetting
+						// the IndexID, or something else weird has
+						// happened. We send a full index to reset the
+						// situation.
+						l.Infof("Device %v folder %q is delta index compatible, but seems out of sync with reality", deviceID, folder.ID)
+						startLocalVersion = 0
+						continue
+					}
+
+					l.Infof("Device %v folder %q is delta index compatible (mlv=%d)", deviceID, folder.ID, dev.MaxLocalVersion)
+					startLocalVersion = dev.MaxLocalVersion
+				} else if dev.IndexID != 0 {
+					// They say they've seen an index ID from us, but it's
+					// not the right one. Either they are confused or we
+					// must have reset our database since last talking to
+					// them. We'll start with a full index transfer.
+					l.Infof("Device %v folder %q has mismatching index ID for us (%v != %v)", deviceID, folder.ID, dev.IndexID, myIndexID)
+					startLocalVersion = 0
+				}
+			} else if bytes.Equal(dev.ID, deviceID[:]) && dev.IndexID != 0 {
+				// This is the other side's description of themselves. We
+				// check to see that it matches the IndexID we have on file,
+				// otherwise we drop our old index data and expect to get a
+				// completely new set.
+
+				theirIndexID := fs.IndexID(deviceID)
+				if dev.IndexID == 0 {
+					// They're not announcing an index ID. This means they
+					// do not support delta indexes and we should clear any
+					// information we have from them before accepting their
+					// index, which will presumably be a full index.
+					fs.Replace(deviceID, nil)
+				} else if dev.IndexID != theirIndexID {
+					// The index ID we have on file is not what they're
+					// announcing. They must have reset their database and
+					// will probably send us a full index. We drop any
+					// information we have and remember this new index ID
+					// instead.
+					l.Infof("Device %v folder %q has a new index ID (%v)", deviceID, folder.ID, dev.IndexID)
+					fs.Replace(deviceID, nil)
+					fs.SetIndexID(deviceID, dev.IndexID)
+				}
+			}
+		}
+
+		go sendIndexes(conn, folder.ID, fs, m.folderIgnores[folder.ID], startLocalVersion)
 	}
+	m.fmut.Unlock()
 
 	// This breaks if we send multiple CM messages during the same connection.
 	if len(tempIndexFolders) > 0 {
@@ -733,7 +782,7 @@ func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterCon
 					}
 
 					// The introducers' introducers are also our introducers.
-					if device.Flags&protocol.FlagIntroducer != 0 {
+					if device.Introducer {
 						l.Infof("Device %v is now also an introducer", id)
 						newDeviceCfg.Introducer = true
 					}
@@ -784,12 +833,6 @@ func (m *Model) Close(device protocol.DeviceID, err error) {
 	})
 
 	m.pmut.Lock()
-	m.fmut.RLock()
-	for _, folder := range m.deviceFolders[device] {
-		m.folderFiles[folder].Replace(device, nil)
-	}
-	m.fmut.RUnlock()
-
 	conn, ok := m.conn[device]
 	if ok {
 		m.progressEmitter.temporaryIndexUnsubscribe(conn)
@@ -804,7 +847,7 @@ func (m *Model) Close(device protocol.DeviceID, err error) {
 
 // Request returns the specified data segment by reading it from local disk.
 // Implements the protocol.Model interface.
-func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, flags uint32, options []protocol.Option, buf []byte) error {
+func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
 	if offset < 0 {
 		return protocol.ErrInvalid
 	}
@@ -813,14 +856,8 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 		l.Warnf("Request from %s for file %s in unshared folder %q", deviceID, name, folder)
 		return protocol.ErrNoSuchFile
 	}
-
-	if flags != 0 && flags != protocol.FlagFromTemporary {
-		// We currently support only no flags, or FromTemporary flag.
-		return fmt.Errorf("protocol error: unknown flags 0x%x in Request message", flags)
-	}
-
 	if deviceID != protocol.LocalDeviceID {
-		l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d f=%d", m, deviceID, folder, name, offset, len(buf), flags)
+		l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d t=%v", m, deviceID, folder, name, offset, len(buf), fromTemporary)
 	}
 	m.fmut.RLock()
 	folderCfg := m.folderCfgs[folder]
@@ -880,7 +917,7 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 
 	// Only check temp files if the flag is set, and if we are set to advertise
 	// the temp indexes.
-	if flags&protocol.FlagFromTemporary != 0 && !folderCfg.DisableTempIndexes {
+	if fromTemporary && !folderCfg.DisableTempIndexes {
 		tempFn := filepath.Join(folderPath, defTempNamer.TempName(name))
 		if err := readOffsetIntoBuf(tempFn, offset, buf); err == nil {
 			return nil
@@ -1027,8 +1064,8 @@ func (m *Model) OnHello(remoteID protocol.DeviceID, addr net.Addr, hello protoco
 }
 
 // GetHello is called when we are about to connect to some remote device.
-func (m *Model) GetHello(protocol.DeviceID) protocol.Version13HelloMessage {
-	return protocol.Version13HelloMessage{
+func (m *Model) GetHello(protocol.DeviceID) protocol.HelloIntf {
+	return &protocol.Hello{
 		DeviceName:    m.deviceName,
 		ClientName:    m.clientName,
 		ClientVersion: m.clientVersion,
@@ -1071,13 +1108,6 @@ func (m *Model) AddConnection(conn connections.Connection, hello protocol.HelloR
 
 	cm := m.generateClusterConfig(deviceID)
 	conn.ClusterConfig(cm)
-
-	m.fmut.RLock()
-	for _, folder := range m.deviceFolders[deviceID] {
-		fs := m.folderFiles[folder]
-		go sendIndexes(conn, folder, fs, m.folderIgnores[folder])
-	}
-	m.fmut.RUnlock()
 	m.pmut.Unlock()
 
 	device, ok := m.cfg.Devices()[deviceID]
@@ -1101,7 +1131,7 @@ func (m *Model) PauseDevice(device protocol.DeviceID) {
 	events.Default.Log(events.DevicePaused, map[string]string{"device": device.String()})
 }
 
-func (m *Model) DownloadProgress(device protocol.DeviceID, folder string, updates []protocol.FileDownloadProgressUpdate, flags uint32, options []protocol.Option) {
+func (m *Model) DownloadProgress(device protocol.DeviceID, folder string, updates []protocol.FileDownloadProgressUpdate) {
 	if !m.folderSharedWith(folder, device) {
 		return
 	}
@@ -1173,15 +1203,15 @@ func (m *Model) receivedFile(folder string, file protocol.FileInfo) {
 	m.folderStatRef(folder).ReceivedFile(file.Name, file.IsDeleted())
 }
 
-func sendIndexes(conn protocol.Connection, folder string, fs *db.FileSet, ignores *ignore.Matcher) {
+func sendIndexes(conn protocol.Connection, folder string, fs *db.FileSet, ignores *ignore.Matcher, startLocalVersion int64) {
 	deviceID := conn.ID()
 	name := conn.Name()
 	var err error
 
-	l.Debugf("sendIndexes for %s-%s/%q starting", deviceID, name, folder)
+	l.Debugf("sendIndexes for %s-%s/%q starting (slv=%d)", deviceID, name, folder, startLocalVersion)
 	defer l.Debugf("sendIndexes for %s-%s/%q exiting: %v", deviceID, name, folder, err)
 
-	minLocalVer, err := sendIndexTo(true, 0, conn, folder, fs, ignores)
+	minLocalVer, err := sendIndexTo(startLocalVersion, conn, folder, fs, ignores)
 
 	// Subscribe to LocalIndexUpdated (we have new information to send) and
 	// DeviceDisconnected (it might be us who disconnected, so we should
@@ -1204,7 +1234,7 @@ func sendIndexes(conn protocol.Connection, folder string, fs *db.FileSet, ignore
 			continue
 		}
 
-		minLocalVer, err = sendIndexTo(false, minLocalVer, conn, folder, fs, ignores)
+		minLocalVer, err = sendIndexTo(minLocalVer, conn, folder, fs, ignores)
 
 		// Wait a short amount of time before entering the next loop. If there
 		// are continuous changes happening to the local index, this gives us
@@ -1213,14 +1243,18 @@ func sendIndexes(conn protocol.Connection, folder string, fs *db.FileSet, ignore
 	}
 }
 
-func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, folder string, fs *db.FileSet, ignores *ignore.Matcher) (int64, error) {
+func sendIndexTo(minLocalVer int64, conn protocol.Connection, folder string, fs *db.FileSet, ignores *ignore.Matcher) (int64, error) {
 	deviceID := conn.ID()
 	name := conn.Name()
 	batch := make([]protocol.FileInfo, 0, indexBatchSize)
 	currentBatchSize := 0
-	maxLocalVer := int64(0)
+	initial := minLocalVer == 0
+	maxLocalVer := minLocalVer
 	var err error
 
+	sorter := NewIndexSorter()
+	defer sorter.Close()
+
 	fs.WithHave(protocol.LocalDeviceID, func(fi db.FileIntf) bool {
 		f := fi.(protocol.FileInfo)
 		if f.LocalVersion <= minLocalVer {
@@ -1236,15 +1270,20 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold
 			return true
 		}
 
+		sorter.Append(f)
+		return true
+	})
+
+	sorter.Sorted(func(f protocol.FileInfo) bool {
 		if len(batch) == indexBatchSize || currentBatchSize > indexTargetSize {
 			if initial {
-				if err = conn.Index(folder, batch, 0, nil); err != nil {
+				if err = conn.Index(folder, batch); err != nil {
 					return false
 				}
 				l.Debugf("sendIndexes for %s-%s/%q: %d files (<%d bytes) (initial index)", deviceID, name, folder, len(batch), currentBatchSize)
 				initial = false
 			} else {
-				if err = conn.IndexUpdate(folder, batch, 0, nil); err != nil {
+				if err = conn.IndexUpdate(folder, batch); err != nil {
 					return false
 				}
 				l.Debugf("sendIndexes for %s-%s/%q: %d files (<%d bytes) (batched update)", deviceID, name, folder, len(batch), currentBatchSize)
@@ -1255,17 +1294,17 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold
 		}
 
 		batch = append(batch, f)
-		currentBatchSize += indexPerFileSize + len(f.Blocks)*indexPerBlockSize
+		currentBatchSize += f.ProtoSize()
 		return true
 	})
 
 	if initial && err == nil {
-		err = conn.Index(folder, batch, 0, nil)
+		err = conn.Index(folder, batch)
 		if err == nil {
 			l.Debugf("sendIndexes for %s-%s/%q: %d files (small initial index)", deviceID, name, folder, len(batch))
 		}
 	} else if len(batch) > 0 && err == nil {
-		err = conn.IndexUpdate(folder, batch, 0, nil)
+		err = conn.IndexUpdate(folder, batch)
 		if err == nil {
 			l.Debugf("sendIndexes for %s-%s/%q: %d files (last batch)", deviceID, name, folder, len(batch))
 		}
@@ -1320,11 +1359,14 @@ func (m *Model) localChangeDetected(folder, path string, files []protocol.FileIn
 		objType := "file"
 		action := "modified"
 
-		// If our local vector is verison 1 AND it is the only version vector so far seen for this file then
-		// it is a new file.  Else if it is > 1 it's not new, and if it is 1 but another shortId version vector
-		// exists then it is new for us but created elsewhere so the file is still not new but modified by us.
-		// Only if it is truly new do we change this to 'added', else we leave it as 'modified'.
-		if len(file.Version) == 1 && file.Version[0].Value == 1 {
+		// If our local vector is version 1 AND it is the only version
+		// vector so far seen for this file then it is a new file.  Else if
+		// it is > 1 it's not new, and if it is 1 but another shortId
+		// version vector exists then it is new for us but created elsewhere
+		// so the file is still not new but modified by us. Only if it is
+		// truly new do we change this to 'added', else we leave it as
+		// 'modified'.
+		if len(file.Version.Counters) == 1 && file.Version.Counters[0].Value == 1 {
 			action = "added"
 		}
 
@@ -1335,11 +1377,8 @@ func (m *Model) localChangeDetected(folder, path string, files []protocol.FileIn
 			action = "deleted"
 		}
 
-		// If the file is a level or more deep then the forward slash seperator is embedded
-		// in the filename and makes the path look wierd on windows, so lets fix it
-		filename := filepath.FromSlash(file.Name)
-		// And append it to the filepath
-		path := filepath.Join(path, filename)
+		// The full file path, adjusted to the local path separator character.
+		path := filepath.Join(path, filepath.FromSlash(file.Name))
 
 		events.Default.Log(events.LocalChangeDetected, map[string]string{
 			"folder": folder,
@@ -1359,7 +1398,7 @@ func (m *Model) requestGlobal(deviceID protocol.DeviceID, folder, name string, o
 		return nil, fmt.Errorf("requestGlobal: no such device: %s", deviceID)
 	}
 
-	l.Debugf("%v REQ(out): %s: %q / %q o=%d s=%d h=%x ft=%t op=%s", m, deviceID, folder, name, offset, size, hash, fromTemporary)
+	l.Debugf("%v REQ(out): %s: %q / %q o=%d s=%d h=%x ft=%t", m, deviceID, folder, name, offset, size, hash, fromTemporary)
 
 	return nc.Request(folder, name, offset, size, hash, fromTemporary)
 }
@@ -1579,10 +1618,14 @@ func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error
 					// File has been ignored or an unsupported symlink. Set invalid bit.
 					l.Debugln("setting invalid bit on ignored", f)
 					nf := protocol.FileInfo{
-						Name:     f.Name,
-						Flags:    f.Flags | protocol.FlagInvalid,
-						Modified: f.Modified,
-						Version:  f.Version, // The file is still the same, so don't bump version
+						Name:          f.Name,
+						Type:          f.Type,
+						Size:          f.Size,
+						Modified:      f.Modified,
+						Permissions:   f.Permissions,
+						NoPermissions: f.NoPermissions,
+						Invalid:       true,
+						Version:       f.Version, // The file is still the same, so don't bump version
 					}
 					batch = append(batch, nf)
 				} else if _, err := osutil.Lstat(filepath.Join(folderCfg.Path(), f.Name)); err != nil {
@@ -1597,16 +1640,13 @@ func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error
 
 					nf := protocol.FileInfo{
 						Name:     f.Name,
-						Flags:    f.Flags | protocol.FlagDeleted,
+						Type:     f.Type,
+						Size:     f.Size,
 						Modified: f.Modified,
+						Deleted:  true,
 						Version:  f.Version.Update(m.shortID),
 					}
 
-					// The deleted file might have been ignored at some
-					// point, but it currently isn't so we make sure to
-					// clear the invalid bit.
-					nf.Flags &^= protocol.FlagInvalid
-
 					batch = append(batch, nf)
 				}
 			}
@@ -1672,30 +1712,23 @@ func (m *Model) numHashers(folder string) int {
 
 // generateClusterConfig returns a ClusterConfigMessage that is correct for
 // the given peer device
-func (m *Model) generateClusterConfig(device protocol.DeviceID) protocol.ClusterConfigMessage {
-	var message protocol.ClusterConfigMessage
+func (m *Model) generateClusterConfig(device protocol.DeviceID) protocol.ClusterConfig {
+	var message protocol.ClusterConfig
 
 	m.fmut.RLock()
 	for _, folder := range m.deviceFolders[device] {
 		folderCfg := m.cfg.Folders()[folder]
+		fs := m.folderFiles[folder]
+
 		protocolFolder := protocol.Folder{
-			ID:    folder,
-			Label: folderCfg.Label,
+			ID:                 folder,
+			Label:              folderCfg.Label,
+			ReadOnly:           folderCfg.Type == config.FolderTypeReadOnly,
+			IgnorePermissions:  folderCfg.IgnorePerms,
+			IgnoreDelete:       folderCfg.IgnoreDelete,
+			DisableTempIndexes: folderCfg.DisableTempIndexes,
 		}
-		var flags uint32
-		if folderCfg.Type == config.FolderTypeReadOnly {
-			flags |= protocol.FlagFolderReadOnly
-		}
-		if folderCfg.IgnorePerms {
-			flags |= protocol.FlagFolderIgnorePerms
-		}
-		if folderCfg.IgnoreDelete {
-			flags |= protocol.FlagFolderIgnoreDelete
-		}
-		if folderCfg.DisableTempIndexes {
-			flags |= protocol.FlagFolderDisabledTempIndexes
-		}
-		protocolFolder.Flags = flags
+
 		for _, device := range m.folderDevices[folder] {
 			// DeviceID is a value type, but with an underlying array. Copy it
 			// so we don't grab aliases to the same array later on in device[:]
@@ -1703,18 +1736,28 @@ func (m *Model) generateClusterConfig(device protocol.DeviceID) protocol.Cluster
 			// TODO: Set read only bit when relevant, and when we have per device
 			// access controls.
 			deviceCfg := m.cfg.Devices()[device]
-			protocolDevice := protocol.Device{
-				ID:          device[:],
-				Name:        deviceCfg.Name,
-				Addresses:   deviceCfg.Addresses,
-				Compression: uint32(deviceCfg.Compression),
-				CertName:    deviceCfg.CertName,
-				Flags:       protocol.FlagShareTrusted,
+
+			var indexID protocol.IndexID
+			var maxLocalVersion int64
+			if device == m.id {
+				indexID = fs.IndexID(protocol.LocalDeviceID)
+				maxLocalVersion = fs.LocalVersion(protocol.LocalDeviceID)
+			} else {
+				indexID = fs.IndexID(device)
+				maxLocalVersion = fs.LocalVersion(device)
 			}
 
-			if deviceCfg.Introducer {
-				protocolDevice.Flags |= protocol.FlagIntroducer
+			protocolDevice := protocol.Device{
+				ID:              device[:],
+				Name:            deviceCfg.Name,
+				Addresses:       deviceCfg.Addresses,
+				Compression:     deviceCfg.Compression,
+				CertName:        deviceCfg.CertName,
+				Introducer:      deviceCfg.Introducer,
+				IndexID:         indexID,
+				MaxLocalVersion: maxLocalVersion,
 			}
+
 			protocolFolder.Devices = append(protocolFolder.Devices, protocolDevice)
 		}
 		message.Folders = append(message.Folders, protocolFolder)
@@ -1759,7 +1802,7 @@ func (m *Model) Override(folder string) {
 		have, ok := fs.Get(protocol.LocalDeviceID, need.Name)
 		if !ok || have.Name != need.Name {
 			// We are missing the file
-			need.Flags |= protocol.FlagDeleted
+			need.Deleted = true
 			need.Blocks = nil
 			need.Version = need.Version.Update(m.shortID)
 		} else {
@@ -1868,7 +1911,7 @@ func (m *Model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly
 
 		if !dirsonly && base != "" {
 			last[base] = []interface{}{
-				time.Unix(f.Modified, 0), f.Size(),
+				time.Unix(f.Modified, 0), f.FileSize(),
 			}
 		}
 
@@ -2181,11 +2224,7 @@ func mapDeviceCfgs(devices []config.DeviceConfiguration) map[protocol.DeviceID]s
 
 func filterIndex(folder string, fs []protocol.FileInfo, dropDeletes bool, ignores *ignore.Matcher) []protocol.FileInfo {
 	for i := 0; i < len(fs); {
-		if fs[i].Flags&^protocol.FlagsAll != 0 {
-			l.Debugln("dropping update for file with unknown bits set", fs[i])
-			fs[i] = fs[len(fs)-1]
-			fs = fs[:len(fs)-1]
-		} else if fs[i].IsDeleted() && dropDeletes {
+		if fs[i].IsDeleted() && dropDeletes {
 			l.Debugln("dropping update for undesired delete", fs[i])
 			fs[i] = fs[len(fs)-1]
 			fs = fs[:len(fs)-1]

lib/model/model_test.go

@@ -55,19 +55,19 @@ func init() {
 var testDataExpected = map[string]protocol.FileInfo{
 	"foo": {
 		Name:     "foo",
-		Flags:    0,
+		Type:     protocol.FileInfoTypeFile,
 		Modified: 0,
 		Blocks:   []protocol.BlockInfo{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
 	},
 	"empty": {
 		Name:     "empty",
-		Flags:    0,
+		Type:     protocol.FileInfoTypeFile,
 		Modified: 0,
 		Blocks:   []protocol.BlockInfo{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
 	},
 	"bar": {
 		Name:     "bar",
-		Flags:    0,
+		Type:     protocol.FileInfoTypeFile,
 		Modified: 0,
 		Blocks:   []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
 	},
@@ -77,8 +77,9 @@ func init() {
 	// Fix expected test data to match reality
 	for n, f := range testDataExpected {
 		fi, _ := os.Stat("testdata/" + n)
-		f.Flags = uint32(fi.Mode())
+		f.Permissions = uint32(fi.Mode())
 		f.Modified = fi.ModTime().Unix()
+		f.Size = fi.Size()
 		testDataExpected[n] = f
 	}
 }
@@ -98,7 +99,7 @@ func TestRequest(t *testing.T) {
 
 	// Existing, shared file
 	bs = bs[:6]
-	err := m.Request(device1, "default", "foo", 0, nil, 0, nil, bs)
+	err := m.Request(device1, "default", "foo", 0, nil, false, bs)
 	if err != nil {
 		t.Error(err)
 	}
@@ -107,32 +108,32 @@ func TestRequest(t *testing.T) {
 	}
 
 	// Existing, nonshared file
-	err = m.Request(device2, "default", "foo", 0, nil, 0, nil, bs)
+	err = m.Request(device2, "default", "foo", 0, nil, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Nonexistent file
-	err = m.Request(device1, "default", "nonexistent", 0, nil, 0, nil, bs)
+	err = m.Request(device1, "default", "nonexistent", 0, nil, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Shared folder, but disallowed file name
-	err = m.Request(device1, "default", "../walk.go", 0, nil, 0, nil, bs)
+	err = m.Request(device1, "default", "../walk.go", 0, nil, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Negative offset
-	err = m.Request(device1, "default", "foo", -4, nil, 0, nil, bs[:0])
+	err = m.Request(device1, "default", "foo", -4, nil, false, bs[:0])
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Larger block than available
 	bs = bs[:42]
-	err = m.Request(device1, "default", "foo", 0, nil, 0, nil, bs)
+	err = m.Request(device1, "default", "foo", 0, nil, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
@@ -168,11 +169,11 @@ func benchmarkIndex(b *testing.B, nfiles int) {
 	m.ServeBackground()
 
 	files := genFiles(nfiles)
-	m.Index(device1, "default", files, 0, nil)
+	m.Index(device1, "default", files)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		m.Index(device1, "default", files, 0, nil)
+		m.Index(device1, "default", files)
 	}
 	b.ReportAllocs()
 }
@@ -199,11 +200,11 @@ func benchmarkIndexUpdate(b *testing.B, nfiles, nufiles int) {
 	files := genFiles(nfiles)
 	ufiles := genFiles(nufiles)
 
-	m.Index(device1, "default", files, 0, nil)
+	m.Index(device1, "default", files)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		m.IndexUpdate(device1, "default", ufiles, 0, nil)
+		m.IndexUpdate(device1, "default", ufiles)
 	}
 	b.ReportAllocs()
 }
@@ -211,8 +212,6 @@ func benchmarkIndexUpdate(b *testing.B, nfiles, nufiles int) {
 type downloadProgressMessage struct {
 	folder  string
 	updates []protocol.FileDownloadProgressUpdate
-	flags   uint32
-	options []protocol.Option
 }
 
 type FakeConnection struct {
@@ -240,11 +239,11 @@ func (f FakeConnection) Option(string) string {
 	return ""
 }
 
-func (FakeConnection) Index(string, []protocol.FileInfo, uint32, []protocol.Option) error {
+func (FakeConnection) Index(string, []protocol.FileInfo) error {
 	return nil
 }
 
-func (FakeConnection) IndexUpdate(string, []protocol.FileInfo, uint32, []protocol.Option) error {
+func (FakeConnection) IndexUpdate(string, []protocol.FileInfo) error {
 	return nil
 }
 
@@ -252,7 +251,7 @@ func (f FakeConnection) Request(folder, name string, offset int64, size int, has
 	return f.requestData, nil
 }
 
-func (FakeConnection) ClusterConfig(protocol.ClusterConfigMessage) {}
+func (FakeConnection) ClusterConfig(protocol.ClusterConfig) {}
 
 func (FakeConnection) Ping() bool {
 	return true
@@ -266,12 +265,10 @@ func (FakeConnection) Statistics() protocol.Statistics {
 	return protocol.Statistics{}
 }
 
-func (f *FakeConnection) DownloadProgress(folder string, updates []protocol.FileDownloadProgressUpdate, flags uint32, options []protocol.Option) {
+func (f *FakeConnection) DownloadProgress(folder string, updates []protocol.FileDownloadProgressUpdate) {
 	f.downloadProgressMessages = append(f.downloadProgressMessages, downloadProgressMessage{
 		folder:  folder,
 		updates: updates,
-		flags:   flags,
-		options: options,
 	})
 }
 
@@ -305,7 +302,7 @@ func BenchmarkRequest(b *testing.B) {
 		},
 		Connection: fc,
 	}, protocol.HelloResult{})
-	m.Index(device1, "default", files, 0, nil)
+	m.Index(device1, "default", files)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -451,13 +448,13 @@ func TestClusterConfig(t *testing.T) {
 	if id := r.Devices[0].ID; !bytes.Equal(id, device1[:]) {
 		t.Errorf("Incorrect device ID %x != %x", id, device1)
 	}
-	if r.Devices[0].Flags&protocol.FlagIntroducer == 0 {
+	if !r.Devices[0].Introducer {
 		t.Error("Device1 should be flagged as Introducer")
 	}
 	if id := r.Devices[1].ID; !bytes.Equal(id, device2[:]) {
 		t.Errorf("Incorrect device ID %x != %x", id, device2)
 	}
-	if r.Devices[1].Flags&protocol.FlagIntroducer != 0 {
+	if r.Devices[1].Introducer {
 		t.Error("Device2 should not be flagged as Introducer")
 	}
 
@@ -471,13 +468,13 @@ func TestClusterConfig(t *testing.T) {
 	if id := r.Devices[0].ID; !bytes.Equal(id, device1[:]) {
 		t.Errorf("Incorrect device ID %x != %x", id, device1)
 	}
-	if r.Devices[0].Flags&protocol.FlagIntroducer == 0 {
+	if !r.Devices[0].Introducer {
 		t.Error("Device1 should be flagged as Introducer")
 	}
 	if id := r.Devices[1].ID; !bytes.Equal(id, device2[:]) {
 		t.Errorf("Incorrect device ID %x != %x", id, device2)
 	}
-	if r.Devices[1].Flags&protocol.FlagIntroducer != 0 {
+	if r.Devices[1].Introducer {
 		t.Error("Device2 should not be flagged as Introducer")
 	}
 }
@@ -566,50 +563,12 @@ func TestIgnores(t *testing.T) {
 
 	// Invalid path, marker should be missing, hence returns an error.
 	m.AddFolder(config.FolderConfiguration{ID: "fresh", RawPath: "XXX"})
-	ignores, _, err = m.GetIgnores("fresh")
+	_, _, err = m.GetIgnores("fresh")
 	if err == nil {
 		t.Error("No error")
 	}
 }
 
-func TestRefuseUnknownBits(t *testing.T) {
-	db := db.OpenMemory()
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
-	m.AddFolder(defaultFolderConfig)
-	m.ServeBackground()
-
-	m.ScanFolder("default")
-	m.Index(device1, "default", []protocol.FileInfo{
-		{
-			Name:  "invalid1",
-			Flags: (protocol.FlagsAll + 1) &^ protocol.FlagInvalid,
-		},
-		{
-			Name:  "invalid2",
-			Flags: (protocol.FlagsAll + 2) &^ protocol.FlagInvalid,
-		},
-		{
-			Name:  "invalid3",
-			Flags: (1 << 31) &^ protocol.FlagInvalid,
-		},
-		{
-			Name:  "valid",
-			Flags: protocol.FlagsAll &^ (protocol.FlagInvalid | protocol.FlagSymlink),
-		},
-	}, 0, nil)
-
-	for _, name := range []string{"invalid1", "invalid2", "invalid3"} {
-		f, ok := m.CurrentGlobalFile("default", name)
-		if ok || f.Name == name {
-			t.Error("Invalid file found or name match")
-		}
-	}
-	f, ok := m.CurrentGlobalFile("default", "valid")
-	if !ok || f.Name != "valid" {
-		t.Error("Valid file not found or name mismatch", ok, f)
-	}
-}
-
 func TestROScanRecovery(t *testing.T) {
 	ldb := db.OpenMemory()
 	set := db.NewFileSet("default", ldb)
@@ -789,17 +748,18 @@ func TestGlobalDirectoryTree(t *testing.T) {
 	m.ServeBackground()
 
 	b := func(isfile bool, path ...string) protocol.FileInfo {
-		flags := uint32(protocol.FlagDirectory)
+		typ := protocol.FileInfoTypeDirectory
 		blocks := []protocol.BlockInfo{}
 		if isfile {
-			flags = 0
+			typ = protocol.FileInfoTypeFile
 			blocks = []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}}
 		}
 		return protocol.FileInfo{
 			Name:     filepath.Join(path...),
-			Flags:    flags,
+			Type:     typ,
 			Modified: 0x666,
 			Blocks:   blocks,
+			Size:     0xa,
 		}
 	}
 
@@ -871,7 +831,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
 		return string(bytes)
 	}
 
-	m.Index(device1, "default", testdata, 0, nil)
+	m.Index(device1, "default", testdata)
 
 	result := m.GlobalDirectoryTree("default", "", -1, false)
 
@@ -1039,17 +999,18 @@ func TestGlobalDirectorySelfFixing(t *testing.T) {
 	m.ServeBackground()
 
 	b := func(isfile bool, path ...string) protocol.FileInfo {
-		flags := uint32(protocol.FlagDirectory)
+		typ := protocol.FileInfoTypeDirectory
 		blocks := []protocol.BlockInfo{}
 		if isfile {
-			flags = 0
+			typ = protocol.FileInfoTypeFile
 			blocks = []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}}
 		}
 		return protocol.FileInfo{
 			Name:     filepath.Join(path...),
-			Flags:    flags,
+			Type:     typ,
 			Modified: 0x666,
 			Blocks:   blocks,
+			Size:     0xa,
 		}
 	}
 
@@ -1130,7 +1091,7 @@ func TestGlobalDirectorySelfFixing(t *testing.T) {
 		return string(bytes)
 	}
 
-	m.Index(device1, "default", testdata, 0, nil)
+	m.Index(device1, "default", testdata)
 
 	result := m.GlobalDirectoryTree("default", "", -1, false)
 
@@ -1215,7 +1176,7 @@ func benchmarkTree(b *testing.B, n1, n2 int) {
 	m.ScanFolder("default")
 	files := genDeepFiles(n1, n2)
 
-	m.Index(device1, "default", files, 0, nil)
+	m.Index(device1, "default", files)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -1244,12 +1205,12 @@ func TestIgnoreDelete(t *testing.T) {
 	}
 
 	// Mark it for deletion
-	f.Flags = protocol.FlagDeleted
+	f.Deleted = true
 	f.Version = f.Version.Update(142) // arbitrary short remote ID
 	f.Blocks = nil
 
 	// Send the index
-	m.Index(device1, "default", []protocol.FileInfo{f}, 0, nil)
+	m.Index(device1, "default", []protocol.FileInfo{f})
 
 	// Make sure we ignored it
 	f, ok = m.CurrentGlobalFile("default", "foo")

lib/model/progressemitter.go

@@ -137,7 +137,7 @@ func (t *ProgressEmitter) sendDownloadProgressMessages() {
 			updates := state.update(folder, activePullers)
 
 			if len(updates) > 0 {
-				conn.DownloadProgress(folder, updates, 0, nil)
+				conn.DownloadProgress(folder, updates)
 			}
 		}
 	}

lib/model/progressemitter_test.go

@@ -100,7 +100,6 @@ func TestProgressEmitter(t *testing.T) {
 }
 
 func TestSendDownloadProgressMessages(t *testing.T) {
-
 	c := config.Wrap("/tmp/test", config.Configuration{})
 	c.SetOptions(config.OptionsConfiguration{
 		ProgressUpdateIntervalS: 0,
@@ -112,7 +111,7 @@ func TestSendDownloadProgressMessages(t *testing.T) {
 	p := NewProgressEmitter(c)
 	p.temporaryIndexSubscribe(fc, []string{"folder", "folder2"})
 
-	expect := func(updateIdx int, state *sharedPullerState, updateType uint32, version protocol.Vector, blocks []int32, remove bool) {
+	expect := func(updateIdx int, state *sharedPullerState, updateType protocol.FileDownloadProgressUpdateType, version protocol.Vector, blocks []int32, remove bool) {
 		messageIdx := -1
 		for i, msg := range fc.downloadProgressMessages {
 			if msg.folder == state.folder {
@@ -346,7 +345,7 @@ func TestSendDownloadProgressMessages(t *testing.T) {
 		file: protocol.FileInfo{
 			Name:    "state5",
 			Version: v1,
-			Flags:   protocol.FlagDirectory,
+			Type:    protocol.FileInfoTypeDirectory,
 			Blocks:  blocks,
 		},
 		mut:              sync.NewRWMutex(),
@@ -359,7 +358,7 @@ func TestSendDownloadProgressMessages(t *testing.T) {
 		file: protocol.FileInfo{
 			Name:    "state6",
 			Version: v1,
-			Flags:   protocol.FlagSymlink,
+			Type:    protocol.FileInfoTypeSymlinkUnknown,
 		},
 		mut:              sync.NewRWMutex(),
 		available:        []int32{1, 2, 3},

lib/model/rwfolder.go

@@ -163,7 +163,7 @@ func (f *rwFolder) configureCopiersAndPullers(config config.FolderConfiguration)
 // set on the local host or the FlagNoPermBits has been set on the file/dir
 // which is being pulled.
 func (f *rwFolder) ignorePermissions(file protocol.FileInfo) bool {
-	return f.ignorePerms || file.Flags&protocol.FlagNoPermBits != 0
+	return f.ignorePerms || file.NoPermissions
 }
 
 // Serve will run scans and pulls. It will return when Stop()ed or on a
@@ -433,12 +433,21 @@ func (f *rwFolder) pullerIteration(ignores *ignore.Matcher) int {
 		l.Debugln(f, "handling", file.Name)
 
 		if !handleFile(file) {
-			// A new or changed file or symlink. This is the only case where we
-			// do stuff concurrently in the background
-			f.queue.Push(file.Name, file.Size(), file.Modified)
+			// A new or changed file or symlink. This is the only case where
+			// we do stuff concurrently in the background. We only queue
+			// files where we are connected to at least one device that has
+			// the file.
+
+			devices := folderFiles.Availability(file.Name)
+			for _, dev := range devices {
+				if f.model.ConnectedTo(dev) {
+					f.queue.Push(file.Name, file.Size, file.Modified)
+					changed++
+					break
+				}
+			}
 		}
 
-		changed++
 		return true
 	})
 
@@ -569,7 +578,7 @@ func (f *rwFolder) handleDir(file protocol.FileInfo) {
 	}()
 
 	realName := filepath.Join(f.dir, file.Name)
-	mode := os.FileMode(file.Flags & 0777)
+	mode := os.FileMode(file.Permissions & 0777)
 	if f.ignorePermissions(file) {
 		mode = 0777
 	}
@@ -909,7 +918,7 @@ func (f *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
 		// touching the file. If we can't stat the file we'll just pull it.
 		if info, err := osutil.Lstat(realName); err == nil {
 			mtime := f.virtualMtimeRepo.GetMtime(file.Name, info.ModTime())
-			if mtime.Unix() != curFile.Modified || info.Size() != curFile.Size() {
+			if mtime.Unix() != curFile.Modified || info.Size() != curFile.Size {
 				l.Debugln("file modified but not rescanned; not pulling:", realName)
 				// Scan() is synchronous (i.e. blocks until the scan is
 				// completed and returns an error), but a scan can't happen
@@ -932,7 +941,7 @@ func (f *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
 
 	// Check for an old temporary file which might have some blocks we could
 	// reuse.
-	tempBlocks, err := scanner.HashFile(tempName, protocol.BlockSize, 0, nil)
+	tempBlocks, err := scanner.HashFile(tempName, protocol.BlockSize, nil)
 	if err == nil {
 		// Check for any reusable blocks in the temp file
 		tempCopyBlocks, _ := scanner.BlockDiff(tempBlocks, file.Blocks)
@@ -965,7 +974,7 @@ func (f *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
 	} else {
 		// Copy the blocks, as we don't want to shuffle them on the FileInfo
 		blocks = append(blocks, file.Blocks...)
-		blocksSize = file.Size()
+		blocksSize = file.Size
 	}
 
 	if f.checkFreeSpace {
@@ -1021,7 +1030,7 @@ func (f *rwFolder) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocks
 func (f *rwFolder) shortcutFile(file protocol.FileInfo) error {
 	realName := filepath.Join(f.dir, file.Name)
 	if !f.ignorePermissions(file) {
-		if err := os.Chmod(realName, os.FileMode(file.Flags&0777)); err != nil {
+		if err := os.Chmod(realName, os.FileMode(file.Permissions&0777)); err != nil {
 			l.Infof("Puller (folder %q, file %q): shortcut: chmod: %v", f.folderID, file.Name, err)
 			f.newError(file.Name, err)
 			return err
@@ -1235,7 +1244,7 @@ func (f *rwFolder) pullerRoutine(in <-chan pullBlockState, out chan<- *sharedPul
 func (f *rwFolder) performFinish(state *sharedPullerState) error {
 	// Set the correct permission bits on the new file
 	if !f.ignorePermissions(state.file) {
-		if err := os.Chmod(state.tempName, os.FileMode(state.file.Flags&0777)); err != nil {
+		if err := os.Chmod(state.tempName, os.FileMode(state.file.Permissions&0777)); err != nil {
 			return err
 		}
 	}

lib/model/rwfolder_test.go

@@ -50,10 +50,8 @@ func setUpFile(filename string, blockNumbers []int) protocol.FileInfo {
 	}
 
 	return protocol.FileInfo{
-		Name:     filename,
-		Flags:    0,
-		Modified: 0,
-		Blocks:   existingBlocks,
+		Name:   filename,
+		Blocks: existingBlocks,
 	}
 }
 
@@ -225,7 +223,7 @@ func TestCopierFinder(t *testing.T) {
 	}
 
 	// Verify that the fetched blocks have actually been written to the temp file
-	blks, err := scanner.HashFile(tempFile, protocol.BlockSize, 0, nil)
+	blks, err := scanner.HashFile(tempFile, protocol.BlockSize, nil)
 	if err != nil {
 		t.Log(err)
 	}

lib/model/sharedpullerstate.go

@@ -12,7 +12,6 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
 )
@@ -122,24 +121,40 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
 		}
 	}
 
+	// The permissions to use for the temporary file should be those of the
+	// final file, except we need user read & write at minimum. The
+	// permissions will be set to the final value later, but in the meantime
+	// we don't want to have a temporary file with looser permissions than
+	// the final outcome.
+	mode := os.FileMode(s.file.Permissions) | 0600
+	if s.ignorePerms {
+		// When ignorePerms is set we use a very permissive mode and let the
+		// system umask filter it.
+		mode = 0666
+	}
+
 	// Attempt to create the temp file
 	// RDWR because of issue #2994.
 	flags := os.O_RDWR
 	if s.reused == 0 {
 		flags |= os.O_CREATE | os.O_EXCL
-	} else {
+	} else if !s.ignorePerms {
 		// With sufficiently bad luck when exiting or crashing, we may have
 		// had time to chmod the temp file to read only state but not yet
 		// moved it to it's final name. This leaves us with a read only temp
 		// file that we're going to try to reuse. To handle that, we need to
 		// make sure we have write permissions on the file before opening it.
-		err := os.Chmod(s.tempName, 0644)
-		if !s.ignorePerms && err != nil {
+		//
+		// When ignorePerms is set we trust that the permissions are fine
+		// already and make no modification, as we would otherwise override
+		// what the umask dictates.
+
+		if err := os.Chmod(s.tempName, mode); err != nil {
 			s.failLocked("dst create chmod", err)
 			return nil, err
 		}
 	}
-	fd, err := os.OpenFile(s.tempName, flags, 0666)
+	fd, err := os.OpenFile(s.tempName, flags, mode)
 	if err != nil {
 		s.failLocked("dst create", err)
 		return nil, err
@@ -150,7 +165,7 @@ func (s *sharedPullerState) tempFile() (io.WriterAt, error) {
 	if s.sparse && !s.file.IsSymlink() {
 		// Truncate sets the size of the file. This creates a sparse file or a
 		// space reservation, depending on the underlying filesystem.
-		if err := fd.Truncate(s.file.Size()); err != nil {
+		if err := fd.Truncate(s.file.Size); err != nil {
 			s.failLocked("dst truncate", err)
 			return nil, err
 		}
@@ -293,8 +308,8 @@ func (s *sharedPullerState) Progress() *pullerProgress {
 		CopiedFromElsewhere: s.copyTotal - s.copyNeeded - s.copyOrigin,
 		Pulled:              s.pullTotal - s.pullNeeded,
 		Pulling:             s.pullNeeded,
-		BytesTotal:          db.BlocksToSize(total),
-		BytesDone:           db.BlocksToSize(done),
+		BytesTotal:          blocksToSize(total),
+		BytesDone:           blocksToSize(done),
 	}
 }
 
@@ -321,3 +336,10 @@ func (s *sharedPullerState) Available() []int32 {
 	s.mut.RUnlock()
 	return blocks
 }
+
+func blocksToSize(num int) int64 {
+	if num < 2 {
+		return protocol.BlockSize / 2
+	}
+	return int64(num-1)*protocol.BlockSize + protocol.BlockSize/2
+}

lib/model/sharedpullerstate_test.go

@@ -29,6 +29,9 @@ func TestSourceFileOK(t *testing.T) {
 
 	bs := make([]byte, 6)
 	n, err := fd.Read(bs)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	if n != len(bs) {
 		t.Fatalf("Wrong read length %d != %d", n, len(bs))

lib/model/sorter.go

@@ -0,0 +1,197 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"encoding/binary"
+	"io/ioutil"
+	"sort"
+
+	"github.com/syncthing/syncthing/lib/osutil"
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+)
+
+const (
+	maxBytesInMemory = 512 << 10
+)
+
+// The IndexSorter sorts FileInfos based on their LocalVersion. You use it
+// by first Append()ing all entries to be sorted, then calling Sorted()
+// which will iterate over all the items in correctly sorted order.
+type IndexSorter interface {
+	Append(f protocol.FileInfo)
+	Sorted(fn func(f protocol.FileInfo) bool)
+	Close()
+}
+
+type internalIndexSorter interface {
+	IndexSorter
+	full() bool
+	copyTo(to IndexSorter)
+}
+
+// NewIndexSorter returns a new IndexSorter that will start out in memory
+// for efficiency but switch to on disk storage once the amount of data
+// becomes large.
+func NewIndexSorter() IndexSorter {
+	return &autoSwitchingIndexSorter{
+		internalIndexSorter: newInMemoryIndexSorter(),
+	}
+}
+
+// An autoSwitchingSorter starts out as an inMemorySorter but becomes an
+// onDiskSorter when the in memory sorter is full().
+type autoSwitchingIndexSorter struct {
+	internalIndexSorter
+}
+
+func (s *autoSwitchingIndexSorter) Append(f protocol.FileInfo) {
+	if s.internalIndexSorter.full() {
+		// We spill before adding a file instead of after, to handle the
+		// case where we're over max size but won't add any more files, in
+		// which case we *don't* need to spill. An example of this would be
+		// an index containing just a single large file.
+		l.Debugf("sorter %p spills to disk", s)
+		next := newOnDiskIndexSorter()
+		s.internalIndexSorter.copyTo(next)
+		s.internalIndexSorter = next
+	}
+	s.internalIndexSorter.Append(f)
+}
+
+// An inMemoryIndexSorter is simply a slice of FileInfos. The full() method
+// returns true when the number of files exceeds maxFiles or the total
+// number of blocks exceeds maxBlocks.
+type inMemoryIndexSorter struct {
+	files    []protocol.FileInfo
+	bytes    int
+	maxBytes int
+}
+
+func newInMemoryIndexSorter() *inMemoryIndexSorter {
+	return &inMemoryIndexSorter{
+		maxBytes: maxBytesInMemory,
+	}
+}
+
+func (s *inMemoryIndexSorter) Append(f protocol.FileInfo) {
+	s.files = append(s.files, f)
+	s.bytes += f.ProtoSize()
+}
+
+func (s *inMemoryIndexSorter) Sorted(fn func(protocol.FileInfo) bool) {
+	sort.Sort(byLocalVersion(s.files))
+	for _, f := range s.files {
+		if !fn(f) {
+			break
+		}
+	}
+}
+
+func (s *inMemoryIndexSorter) Close() {
+}
+
+func (s *inMemoryIndexSorter) full() bool {
+	return s.bytes >= s.maxBytes
+}
+
+func (s *inMemoryIndexSorter) copyTo(dst IndexSorter) {
+	for _, f := range s.files {
+		dst.Append(f)
+	}
+}
+
+// byLocalVersion sorts FileInfos by LocalVersion
+type byLocalVersion []protocol.FileInfo
+
+func (l byLocalVersion) Len() int {
+	return len(l)
+}
+func (l byLocalVersion) Swap(a, b int) {
+	l[a], l[b] = l[b], l[a]
+}
+func (l byLocalVersion) Less(a, b int) bool {
+	return l[a].LocalVersion < l[b].LocalVersion
+}
+
+// An onDiskIndexSorter is backed by a LevelDB database in the temporary
+// directory. It relies on the fact that iterating over the database is done
+// in key order and uses the LocalVersion as key. When done with an
+// onDiskIndexSorter you must call Close() to remove the temporary database.
+type onDiskIndexSorter struct {
+	db  *leveldb.DB
+	dir string
+}
+
+func newOnDiskIndexSorter() *onDiskIndexSorter {
+	// Set options to minimize resource usage.
+	opts := &opt.Options{
+		OpenFilesCacheCapacity: 10,
+		WriteBuffer:            512 << 10,
+	}
+
+	// Use a temporary database directory.
+	tmp, err := ioutil.TempDir("", "syncthing-db.")
+	if err != nil {
+		panic("creating temporary directory: " + err.Error())
+	}
+	db, err := leveldb.OpenFile(tmp, opts)
+	if err != nil {
+		panic("creating temporary database: " + err.Error())
+	}
+
+	s := &onDiskIndexSorter{
+		db:  db,
+		dir: tmp,
+	}
+	l.Debugf("onDiskIndexSorter %p created at %s", s, tmp)
+	return s
+}
+
+func (s *onDiskIndexSorter) Append(f protocol.FileInfo) {
+	key := make([]byte, 8)
+	binary.BigEndian.PutUint64(key[:], uint64(f.LocalVersion))
+	data, err := f.Marshal()
+	if err != nil {
+		panic("bug: marshalling FileInfo should never fail: " + err.Error())
+	}
+	err = s.db.Put(key, data, nil)
+	if err != nil {
+		panic("writing to temporary database: " + err.Error())
+	}
+}
+
+func (s *onDiskIndexSorter) Sorted(fn func(protocol.FileInfo) bool) {
+	it := s.db.NewIterator(nil, nil)
+	defer it.Release()
+	for it.Next() {
+		var f protocol.FileInfo
+		if err := f.Unmarshal(it.Value()); err != nil {
+			panic("unmarshal failed: " + err.Error())
+		}
+		if !fn(f) {
+			break
+		}
+	}
+}
+
+func (s *onDiskIndexSorter) Close() {
+	l.Debugf("onDiskIndexSorter %p closes", s)
+	s.db.Close()
+	osutil.RemoveAll(s.dir)
+}
+
+func (s *onDiskIndexSorter) full() bool {
+	return false
+}
+
+func (s *onDiskIndexSorter) copyTo(dst IndexSorter) {
+	// Just wrap Sorted() if we need to support this in the future.
+	panic("unsupported")
+}

lib/model/sorter_test.go

@@ -0,0 +1,156 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/rand"
+)
+
+func TestInMemoryIndexSorter(t *testing.T) {
+	// An inMemorySorter should be able to absorb a few files in unsorted
+	// order, and return them sorted.
+
+	s := newInMemoryIndexSorter()
+	addFiles(50, s)
+	verifySorted(t, s, 50)
+	verifyBreak(t, s, 50)
+	s.Close()
+}
+
+func TestOnDiskIndexSorter(t *testing.T) {
+	// An onDiskSorter should be able to absorb a few files in unsorted
+	// order, and return them sorted.
+
+	s := newOnDiskIndexSorter()
+	addFiles(50, s)
+	verifySorted(t, s, 50)
+	verifyBreak(t, s, 50)
+
+	// The temporary database should exist on disk. When Close()d, it should
+	// be removed.
+
+	info, err := os.Stat(s.dir)
+	if err != nil {
+		t.Fatal("temp database should exist on disk:", err)
+	}
+	if !info.IsDir() {
+		t.Fatal("temp database should be a directory")
+	}
+
+	s.Close()
+
+	_, err = os.Stat(s.dir)
+	if !os.IsNotExist(err) {
+		t.Fatal("temp database should have been removed")
+	}
+}
+
+func TestIndexSorter(t *testing.T) {
+	// An default IndexSorter should be able to absorb files, have them in
+	// memory, and at some point switch to an on disk database.
+
+	s := NewIndexSorter()
+	defer s.Close()
+
+	// We should start out as an in memory store.
+
+	nFiles := 1
+	addFiles(1, s)
+	verifySorted(t, s, nFiles)
+
+	as := s.(*autoSwitchingIndexSorter)
+	if _, ok := as.internalIndexSorter.(*inMemoryIndexSorter); !ok {
+		t.Fatalf("the sorter should be in memory after only one file")
+	}
+
+	// At some point, for sure with less than maxBytesInMemory files, we
+	// should switch over to an on disk sorter.
+	for i := 0; i < maxBytesInMemory; i++ {
+		addFiles(1, s)
+		nFiles++
+		if _, ok := as.internalIndexSorter.(*onDiskIndexSorter); ok {
+			break
+		}
+	}
+
+	if _, ok := as.internalIndexSorter.(*onDiskIndexSorter); !ok {
+		t.Fatalf("the sorter should be on disk after %d files", nFiles)
+	}
+
+	verifySorted(t, s, nFiles)
+
+	// For test coverage, as some methods are called on the onDiskSorter
+	// only after switching to it.
+
+	addFiles(1, s)
+	verifySorted(t, s, nFiles+1)
+}
+
+// addFiles adds files with random LocalVersion to the Sorter.
+func addFiles(n int, s IndexSorter) {
+	for i := 0; i < n; i++ {
+		rnd := rand.Int63()
+		f := protocol.FileInfo{
+			Name:         fmt.Sprintf("file-%d", rnd),
+			Size:         rand.Int63(),
+			Permissions:  uint32(rand.Intn(0777)),
+			Modified:     rand.Int63(),
+			LocalVersion: rnd,
+			Version:      protocol.Vector{Counters: []protocol.Counter{{ID: 42, Value: uint64(rand.Int63())}}},
+			Blocks: []protocol.BlockInfo{{
+				Size: int32(rand.Intn(128 << 10)),
+				Hash: []byte(rand.String(32)),
+			}},
+		}
+		s.Append(f)
+	}
+}
+
+// verifySorted checks that the files are returned sorted by LocalVersion.
+func verifySorted(t *testing.T, s IndexSorter, expected int) {
+	prevLocalVer := int64(-1)
+	seen := 0
+	s.Sorted(func(f protocol.FileInfo) bool {
+		if f.LocalVersion <= prevLocalVer {
+			t.Fatalf("Unsorted LocalVer, %d <= %d", f.LocalVersion, prevLocalVer)
+		}
+		prevLocalVer = f.LocalVersion
+		seen++
+		return true
+	})
+	if seen != expected {
+		t.Fatalf("expected %d files returned, got %d", expected, seen)
+	}
+}
+
+// verifyBreak checks that the Sorter stops iteration once we return false.
+func verifyBreak(t *testing.T, s IndexSorter, expected int) {
+	prevLocalVer := int64(-1)
+	seen := 0
+	s.Sorted(func(f protocol.FileInfo) bool {
+		if f.LocalVersion <= prevLocalVer {
+			t.Fatalf("Unsorted LocalVer, %d <= %d", f.LocalVersion, prevLocalVer)
+		}
+		if len(f.Blocks) != 1 {
+			t.Fatalf("incorrect number of blocks %d != 1", len(f.Blocks))
+		}
+		if len(f.Version.Counters) != 1 {
+			t.Fatalf("incorrect number of version counters %d != 1", len(f.Version.Counters))
+		}
+		prevLocalVer = f.LocalVersion
+		seen++
+		return seen < expected/2
+	})
+	if seen != expected/2 {
+		t.Fatalf("expected %d files iterated over, got %d", expected, seen)
+	}
+}

lib/protocol/benchmark_test.go

@@ -63,8 +63,8 @@ func benchmarkRequestsConnPair(b *testing.B, conn0, conn1 net.Conn) {
 	c1.Start()
 
 	// Satisfy the assertions in the protocol by sending an initial cluster config
-	c0.ClusterConfig(ClusterConfigMessage{})
-	c1.ClusterConfig(ClusterConfigMessage{})
+	c0.ClusterConfig(ClusterConfig{})
+	c1.ClusterConfig(ClusterConfig{})
 
 	// Report some useful stats and reset the timer for the actual test
 	b.ReportAllocs()
@@ -164,13 +164,13 @@ func negotiateTLS(cert tls.Certificate, conn0, conn1 net.Conn) (net.Conn, net.Co
 
 type fakeModel struct{}
 
-func (m *fakeModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+func (m *fakeModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
 }
 
-func (m *fakeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+func (m *fakeModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
 }
 
-func (m *fakeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
+func (m *fakeModel) Request(deviceID DeviceID, folder string, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
 	// We write the offset to the end of the buffer, so the receiver
 	// can verify that it did in fact get some data back over the
 	// connection.
@@ -178,11 +178,11 @@ func (m *fakeModel) Request(deviceID DeviceID, folder string, name string, offse
 	return nil
 }
 
-func (m *fakeModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
+func (m *fakeModel) ClusterConfig(deviceID DeviceID, config ClusterConfig) {
 }
 
 func (m *fakeModel) Close(deviceID DeviceID, err error) {
 }
 
-func (m *fakeModel) DownloadProgress(deviceID DeviceID, folder string, updates []FileDownloadProgressUpdate, flags uint32, options []Option) {
+func (m *fakeModel) DownloadProgress(deviceID DeviceID, folder string, updates []FileDownloadProgressUpdate) {
 }

lib/protocol/bep.proto

@@ -0,0 +1,190 @@
+// protoc --proto_path=../../../../../:../../../../gogo/protobuf/protobuf:. --gogofast_out=. message.proto
+
+syntax = "proto3";
+
+package protocol;
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.sizer_all) = false;
+option (gogoproto.protosizer_all) = true;
+option (gogoproto.goproto_enum_stringer_all) = true;
+option (gogoproto.goproto_enum_prefix_all) = false;
+
+// --- Pre-auth ---
+
+message Hello {
+    string device_name    = 1;
+    string client_name    = 2;
+    string client_version = 3;
+}
+
+// --- Header ---
+
+message Header {
+    MessageType        type        = 1;
+    MessageCompression compression = 2;
+}
+
+enum MessageType {
+    CLUSTER_CONFIG    = 0 [(gogoproto.enumvalue_customname) = "messageTypeClusterConfig"];
+    INDEX             = 1 [(gogoproto.enumvalue_customname) = "messageTypeIndex"];
+    INDEX_UPDATE      = 2 [(gogoproto.enumvalue_customname) = "messageTypeIndexUpdate"];
+    REQUEST           = 3 [(gogoproto.enumvalue_customname) = "messageTypeRequest"];
+    RESPONSE          = 4 [(gogoproto.enumvalue_customname) = "messageTypeResponse"];
+    DOWNLOAD_PROGRESS = 5 [(gogoproto.enumvalue_customname) = "messageTypeDownloadProgress"];
+    PING              = 6 [(gogoproto.enumvalue_customname) = "messageTypePing"];
+    CLOSE             = 7 [(gogoproto.enumvalue_customname) = "messageTypeClose"];
+}
+
+enum MessageCompression {
+    NONE = 0 [(gogoproto.enumvalue_customname) = "MessageCompressionNone"];
+    LZ4  = 1 [(gogoproto.enumvalue_customname) = "MessageCompressionLZ4"];
+}
+
+// --- Actual messages ---
+
+// Cluster Config
+
+message ClusterConfig {
+    repeated Folder folders = 1 [(gogoproto.nullable) = false];
+}
+
+message Folder {
+    string id                   = 1 [(gogoproto.customname) = "ID"];
+    string label                = 2;
+    bool   read_only            = 3;
+    bool   ignore_permissions   = 4;
+    bool   ignore_delete        = 5;
+    bool   disable_temp_indexes = 6;
+
+    repeated Device devices = 16 [(gogoproto.nullable) = false];
+}
+
+message Device {
+    bytes           id                = 1 [(gogoproto.customname) = "ID"];
+    string          name              = 2;
+    repeated string addresses         = 3;
+    Compression     compression       = 4;
+    string          cert_name         = 5;
+    int64           max_local_version = 6;
+    bool            introducer        = 7;
+    uint64          index_id          = 8 [(gogoproto.customname) = "IndexID", (gogoproto.customtype) = "IndexID", (gogoproto.nullable) = false];
+}
+
+enum Compression {
+    METADATA = 0 [(gogoproto.enumvalue_customname) = "CompressMetadata"];
+    NEVER    = 1 [(gogoproto.enumvalue_customname) = "CompressNever"];
+    ALWAYS   = 2 [(gogoproto.enumvalue_customname) = "CompressAlways"];
+}
+
+// Index and Index Update
+
+message Index {
+    string            folder = 1;
+    repeated FileInfo files  = 2 [(gogoproto.nullable) = false];
+}
+
+message IndexUpdate {
+    string            folder = 1;
+    repeated FileInfo files  = 2 [(gogoproto.nullable) = false];
+}
+
+message FileInfo {
+    option (gogoproto.goproto_stringer) = false;
+    string       name           = 1;
+    FileInfoType type           = 2;
+    int64        size           = 3;
+    uint32       permissions    = 4;
+    int64        modified       = 5;
+    bool         deleted        = 6;
+    bool         invalid        = 7;
+    bool         no_permissions = 8;
+    Vector       version        = 9 [(gogoproto.nullable) = false];
+    int64        local_version  = 10;
+
+    repeated BlockInfo Blocks = 16 [(gogoproto.nullable) = false];
+}
+
+enum FileInfoType {
+    FILE              = 0 [(gogoproto.enumvalue_customname) = "FileInfoTypeFile"];
+    DIRECTORY         = 1 [(gogoproto.enumvalue_customname) = "FileInfoTypeDirectory"];
+    SYMLINK_FILE      = 2 [(gogoproto.enumvalue_customname) = "FileInfoTypeSymlinkFile"];
+    SYMLINK_DIRECTORY = 3 [(gogoproto.enumvalue_customname) = "FileInfoTypeSymlinkDirectory"];
+    SYMLINK_UNKNOWN   = 4 [(gogoproto.enumvalue_customname) = "FileInfoTypeSymlinkUnknown"];
+}
+
+message BlockInfo {
+    option (gogoproto.goproto_stringer) = false;
+    int64 offset = 1;
+    int32 size   = 2;
+    bytes hash   = 3;
+}
+
+message Vector {
+    repeated Counter counters = 1 [(gogoproto.nullable) = false];
+}
+
+message Counter {
+    uint64 id    = 1 [(gogoproto.customname) = "ID", (gogoproto.customtype) = "ShortID", (gogoproto.nullable) = false];
+    uint64 value = 2;
+}
+
+// Request
+
+message Request {
+    int32  id             = 1 [(gogoproto.customname) = "ID"];
+    string folder         = 2;
+    string name           = 3;
+    int64  offset         = 4;
+    int32  size           = 5;
+    bytes  hash           = 6;
+    bool   from_temporary = 7;
+}
+
+// Response
+
+message Response {
+    int32     id   = 1 [(gogoproto.customname) = "ID"];
+    bytes     data = 2;
+    ErrorCode code = 3;
+}
+
+enum ErrorCode {
+    NO_ERROR     = 0 [(gogoproto.enumvalue_customname) = "ErrorCodeNoError"];
+    GENERIC      = 1 [(gogoproto.enumvalue_customname) = "ErrorCodeGeneric"];
+    NO_SUCH_FILE = 2 [(gogoproto.enumvalue_customname) = "ErrorCodeNoSuchFile"];
+    INVALID_FILE = 3 [(gogoproto.enumvalue_customname) = "ErrorCodeInvalidFile"];
+}
+
+// DownloadProgress
+
+message DownloadProgress {
+    string                              folder  = 1;
+    repeated FileDownloadProgressUpdate updates = 2 [(gogoproto.nullable) = false];
+}
+
+message FileDownloadProgressUpdate {
+    FileDownloadProgressUpdateType update_type   = 1;
+    string                         name          = 2;
+    Vector                         version       = 3 [(gogoproto.nullable) = false];
+    repeated int32                 block_indexes = 4;
+}
+
+enum FileDownloadProgressUpdateType {
+    APPEND = 0 [(gogoproto.enumvalue_customname) = "UpdateTypeAppend"];
+    FORGET = 1 [(gogoproto.enumvalue_customname) = "UpdateTypeForget"];
+}
+
+// Ping
+
+message Ping {
+}
+
+// Close
+
+message Close {
+    string reason = 1;
+}
+

lib/protocol/bep_extensions.go

@@ -0,0 +1,124 @@
+// Copyright (C) 2014 The Protocol Authors.
+
+//go:generate go run ../../script/protofmt.go bep.proto
+//go:generate protoc --proto_path=../../../../../:../../../../gogo/protobuf/protobuf:. --gogofast_out=. bep.proto
+
+package protocol
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/binary"
+	"errors"
+	"fmt"
+
+	"github.com/syncthing/syncthing/lib/rand"
+)
+
+var (
+	sha256OfEmptyBlock = sha256.Sum256(make([]byte, BlockSize))
+	HelloMessageMagic  = uint32(0x2EA7D90B)
+)
+
+func (m Hello) Magic() uint32 {
+	return HelloMessageMagic
+}
+
+func (f FileInfo) String() string {
+	return fmt.Sprintf("File{Name:%q, Type:%v, LocalVersion:%d, Permissions:0%o, Modified:%d, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v, Blocks:%v}",
+		f.Name, f.Type, f.LocalVersion, f.Permissions, f.Modified, f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions, f.Blocks)
+}
+
+func (f FileInfo) IsDeleted() bool {
+	return f.Deleted
+}
+
+func (f FileInfo) IsInvalid() bool {
+	return f.Invalid
+}
+
+func (f FileInfo) IsDirectory() bool {
+	return f.Type == FileInfoTypeDirectory
+}
+
+func (f FileInfo) IsSymlink() bool {
+	switch f.Type {
+	case FileInfoTypeSymlinkDirectory, FileInfoTypeSymlinkFile, FileInfoTypeSymlinkUnknown:
+		return true
+	default:
+		return false
+	}
+}
+
+func (f FileInfo) HasPermissionBits() bool {
+	return !f.NoPermissions
+}
+
+func (f FileInfo) FileSize() int64 {
+	if f.IsDirectory() || f.IsDeleted() {
+		return 128
+	}
+	return f.Size
+}
+
+func (f FileInfo) FileName() string {
+	return f.Name
+}
+
+// WinsConflict returns true if "f" is the one to choose when it is in
+// conflict with "other".
+func (f FileInfo) WinsConflict(other FileInfo) bool {
+	// If a modification is in conflict with a delete, we pick the
+	// modification.
+	if !f.IsDeleted() && other.IsDeleted() {
+		return true
+	}
+	if f.IsDeleted() && !other.IsDeleted() {
+		return false
+	}
+
+	// The one with the newer modification time wins.
+	if f.Modified > other.Modified {
+		return true
+	}
+	if f.Modified < other.Modified {
+		return false
+	}
+
+	// The modification times were equal. Use the device ID in the version
+	// vector as tie breaker.
+	return f.Version.Compare(other.Version) == ConcurrentGreater
+}
+
+func (b BlockInfo) String() string {
+	return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash)
+}
+
+// IsEmpty returns true if the block is a full block of zeroes.
+func (b BlockInfo) IsEmpty() bool {
+	return b.Size == BlockSize && bytes.Equal(b.Hash, sha256OfEmptyBlock[:])
+}
+
+type IndexID uint64
+
+func (i IndexID) String() string {
+	return fmt.Sprintf("0x%16X", uint64(i))
+}
+
+func (i IndexID) Marshal() ([]byte, error) {
+	bs := make([]byte, 8)
+	binary.BigEndian.PutUint64(bs, uint64(i))
+	return bs, nil
+}
+
+func (i *IndexID) Unmarshal(bs []byte) error {
+	if len(bs) != 8 {
+		return errors.New("incorrect IndexID length")
+	}
+	*i = IndexID(binary.BigEndian.Uint64(bs))
+	return nil
+}
+
+func NewIndexID() IndexID {
+	return IndexID(rand.Int64())
+}

lib/protocol/bufferpool.go

@@ -0,0 +1,59 @@
+// Copyright (C) 2016 The Protocol Authors.
+
+package protocol
+
+import "sync"
+
+type bufferPool struct {
+	minSize int
+	pool    sync.Pool
+}
+
+// get returns a new buffer of the requested size
+func (p *bufferPool) get(size int) []byte {
+	intf := p.pool.Get()
+	if intf == nil {
+		// Pool is empty, must allocate.
+		return p.new(size)
+	}
+
+	bs := intf.([]byte)
+	if cap(bs) < size {
+		// Buffer was too small, leave it for someone else and allocate.
+		p.put(bs)
+		return p.new(size)
+	}
+
+	return bs[:size]
+}
+
+// upgrade grows the buffer to the requested size, while attempting to reuse
+// it if possible.
+func (p *bufferPool) upgrade(bs []byte, size int) []byte {
+	if cap(bs) >= size {
+		// Reslicing is enough, lets go!
+		return bs[:size]
+	}
+
+	// It was too small. But it pack into the pool and try to get another
+	// buffer.
+	p.put(bs)
+	return p.get(size)
+}
+
+// put returns the buffer to the pool
+func (p *bufferPool) put(bs []byte) {
+	p.pool.Put(bs)
+}
+
+// new creates a new buffer of the requested size, taking the minimum
+// allocation count into account. For internal use only.
+func (p *bufferPool) new(size int) []byte {
+	allocSize := size
+	if allocSize < p.minSize {
+		// Avoid allocating tiny buffers that we won't be able to reuse for
+		// anything useful.
+		allocSize = p.minSize
+	}
+	return make([]byte, allocSize)[:size]
+}

lib/protocol/common_test.go

@@ -5,16 +5,15 @@ package protocol
 import "time"
 
 type TestModel struct {
-	data      []byte
-	folder    string
-	name      string
-	offset    int64
-	size      int
-	hash      []byte
-	flags     uint32
-	options   []Option
-	closedCh  chan struct{}
-	closedErr error
+	data          []byte
+	folder        string
+	name          string
+	offset        int64
+	size          int
+	hash          []byte
+	fromTemporary bool
+	closedCh      chan struct{}
+	closedErr     error
 }
 
 func newTestModel() *TestModel {
@@ -23,20 +22,19 @@ func newTestModel() *TestModel {
 	}
 }
 
-func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+func (t *TestModel) Index(deviceID DeviceID, folder string, files []FileInfo) {
 }
 
-func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo, flags uint32, options []Option) {
+func (t *TestModel) IndexUpdate(deviceID DeviceID, folder string, files []FileInfo) {
 }
 
-func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, hash []byte, flags uint32, options []Option, buf []byte) error {
+func (t *TestModel) Request(deviceID DeviceID, folder, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
 	t.folder = folder
 	t.name = name
 	t.offset = offset
 	t.size = len(buf)
 	t.hash = hash
-	t.flags = flags
-	t.options = options
+	t.fromTemporary = fromTemporary
 	copy(buf, t.data)
 	return nil
 }
@@ -46,10 +44,10 @@ func (t *TestModel) Close(deviceID DeviceID, err error) {
 	close(t.closedCh)
 }
 
-func (t *TestModel) ClusterConfig(deviceID DeviceID, config ClusterConfigMessage) {
+func (t *TestModel) ClusterConfig(deviceID DeviceID, config ClusterConfig) {
 }
 
-func (t *TestModel) DownloadProgress(DeviceID, string, []FileDownloadProgressUpdate, uint32, []Option) {
+func (t *TestModel) DownloadProgress(DeviceID, string, []FileDownloadProgressUpdate) {
 }
 
 func (t *TestModel) closedError() error {

lib/protocol/compression.go

@@ -4,13 +4,7 @@ package protocol
 
 import "fmt"
 
-type Compression int
-
 const (
-	CompressMetadata Compression = iota // zero value is the default, default should be "metadata"
-	CompressNever
-	CompressAlways
-
 	compressionThreshold = 128 // don't bother compressing messages smaller than this many bytes
 )
 
@@ -31,14 +25,6 @@ var compressionUnmarshal = map[string]Compression{
 	"always":   CompressAlways,
 }
 
-func (c Compression) String() string {
-	s, ok := compressionMarshal[c]
-	if !ok {
-		return fmt.Sprintf("unknown:%d", c)
-	}
-	return s
-}
-
 func (c Compression) GoString() string {
 	return fmt.Sprintf("%q", c.String())
 }

lib/protocol/conflict_test.go

@@ -8,8 +8,8 @@ func TestWinsConflict(t *testing.T) {
 	testcases := [][2]FileInfo{
 		// The first should always win over the second
 		{{Modified: 42}, {Modified: 41}},
-		{{Modified: 41}, {Modified: 42, Flags: FlagDeleted}},
-		{{Modified: 41, Version: Vector{{42, 2}, {43, 1}}}, {Modified: 41, Version: Vector{{42, 1}, {43, 2}}}},
+		{{Modified: 41}, {Modified: 42, Deleted: true}},
+		{{Modified: 41, Version: Vector{[]Counter{{42, 2}, {43, 1}}}}, {Modified: 41, Version: Vector{[]Counter{{42, 1}, {43, 2}}}}},
 	}
 
 	for _, tc := range testcases {

lib/protocol/debug.go

@@ -16,7 +16,3 @@ var (
 func init() {
 	l.SetDebug("protocol", strings.Contains(os.Getenv("STTRACE"), "protocol") || os.Getenv("STTRACE") == "all")
 }
-
-func shouldDebug() bool {
-	return l.ShouldDebug("protocol")
-}

lib/protocol/errors.go

@@ -6,13 +6,6 @@ import (
 	"errors"
 )
 
-const (
-	ecNoError int32 = iota
-	ecGeneric
-	ecNoSuchFile
-	ecInvalid
-)
-
 var (
 	ErrNoError    error
 	ErrGeneric    = errors.New("generic error")
@@ -20,32 +13,32 @@ var (
 	ErrInvalid    = errors.New("file is invalid")
 )
 
-var lookupError = map[int32]error{
-	ecNoError:    ErrNoError,
-	ecGeneric:    ErrGeneric,
-	ecNoSuchFile: ErrNoSuchFile,
-	ecInvalid:    ErrInvalid,
+var lookupError = map[ErrorCode]error{
+	ErrorCodeNoError:     ErrNoError,
+	ErrorCodeGeneric:     ErrGeneric,
+	ErrorCodeNoSuchFile:  ErrNoSuchFile,
+	ErrorCodeInvalidFile: ErrInvalid,
 }
 
-var lookupCode = map[error]int32{
-	ErrNoError:    ecNoError,
-	ErrGeneric:    ecGeneric,
-	ErrNoSuchFile: ecNoSuchFile,
-	ErrInvalid:    ecInvalid,
+var lookupCode = map[error]ErrorCode{
+	ErrNoError:    ErrorCodeNoError,
+	ErrGeneric:    ErrorCodeGeneric,
+	ErrNoSuchFile: ErrorCodeNoSuchFile,
+	ErrInvalid:    ErrorCodeInvalidFile,
 }
 
-func codeToError(errcode int32) error {
-	err, ok := lookupError[errcode]
+func codeToError(code ErrorCode) error {
+	err, ok := lookupError[code]
 	if !ok {
 		return ErrGeneric
 	}
 	return err
 }
 
-func errorToCode(err error) int32 {
+func errorToCode(err error) ErrorCode {
 	code, ok := lookupCode[err]
 	if !ok {
-		return ecGeneric
+		return ErrorCodeGeneric
 	}
 	return code
 }

lib/protocol/fuzz.go

@@ -1,70 +0,0 @@
-// Copyright (C) 2015 The Protocol Authors.
-
-// +build gofuzz
-
-package protocol
-
-import (
-	"bytes"
-	"encoding/binary"
-	"encoding/hex"
-	"fmt"
-	"reflect"
-	"sync"
-)
-
-func Fuzz(data []byte) int {
-	// Regenerate the length, or we'll most commonly exit quickly due to an
-	// unexpected eof which is unintestering.
-	if len(data) > 8 {
-		binary.BigEndian.PutUint32(data[4:], uint32(len(data))-8)
-	}
-
-	// Setup a rawConnection we'll use to parse the message.
-	c := rawConnection{
-		cr:     &countingReader{Reader: bytes.NewReader(data)},
-		closed: make(chan struct{}),
-		pool: sync.Pool{
-			New: func() interface{} {
-				return make([]byte, BlockSize)
-			},
-		},
-	}
-
-	// Attempt to parse the message.
-	hdr, msg, err := c.readMessage()
-	if err != nil {
-		return 0
-	}
-
-	// If parsing worked, attempt to encode it again.
-	newBs, err := msg.AppendXDR(nil)
-	if err != nil {
-		panic("not encodable")
-	}
-
-	// Create an appriate header for the re-encoding.
-	newMsg := make([]byte, 8)
-	binary.BigEndian.PutUint32(newMsg, encodeHeader(hdr))
-	binary.BigEndian.PutUint32(newMsg[4:], uint32(len(newBs)))
-	newMsg = append(newMsg, newBs...)
-
-	// Use the rawConnection to parse the re-encoding.
-	c.cr = &countingReader{Reader: bytes.NewReader(newMsg)}
-	hdr2, msg2, err := c.readMessage()
-	if err != nil {
-		fmt.Println("Initial:\n" + hex.Dump(data))
-		fmt.Println("New:\n" + hex.Dump(newMsg))
-		panic("not parseable after re-encode: " + err.Error())
-	}
-
-	// Make sure the data is the same as it was before.
-	if hdr != hdr2 {
-		panic("headers differ")
-	}
-	if !reflect.DeepEqual(msg, msg2) {
-		panic("contents differ")
-	}
-
-	return 1
-}

lib/protocol/fuzz_test.go

@@ -1,89 +0,0 @@
-// Copyright (C) 2015 The Protocol Authors.
-
-// +build gofuzz
-
-package protocol
-
-import (
-	"encoding/binary"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"strings"
-	"testing"
-	"testing/quick"
-)
-
-// This can be used to generate a corpus of valid messages as a starting point
-// for the fuzzer.
-func TestGenerateCorpus(t *testing.T) {
-	t.Skip("Use to generate initial corpus only")
-
-	n := 0
-	check := func(idx IndexMessage) bool {
-		for i := range idx.Options {
-			if len(idx.Options[i].Key) > 64 {
-				idx.Options[i].Key = idx.Options[i].Key[:64]
-			}
-		}
-		hdr := header{
-			version:     0,
-			msgID:       42,
-			msgType:     messageTypeIndex,
-			compression: false,
-		}
-
-		msgBs := idx.MustMarshalXDR()
-
-		buf := make([]byte, 8)
-		binary.BigEndian.PutUint32(buf, encodeHeader(hdr))
-		binary.BigEndian.PutUint32(buf[4:], uint32(len(msgBs)))
-		buf = append(buf, msgBs...)
-
-		ioutil.WriteFile(fmt.Sprintf("testdata/corpus/test-%03d.xdr", n), buf, 0644)
-		n++
-		return true
-	}
-
-	if err := quick.Check(check, &quick.Config{MaxCount: 1000}); err != nil {
-		t.Fatal(err)
-	}
-}
-
-// Tests any crashers found by the fuzzer, for closer investigation.
-func TestCrashers(t *testing.T) {
-	testFiles(t, "testdata/crashers")
-}
-
-// Tests the entire corpus, which should PASS before the fuzzer starts
-// fuzzing.
-func TestCorpus(t *testing.T) {
-	testFiles(t, "testdata/corpus")
-}
-
-func testFiles(t *testing.T, dir string) {
-	fd, err := os.Open(dir)
-	if err != nil {
-		t.Fatal(err)
-	}
-	crashers, err := fd.Readdirnames(-1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	for _, name := range crashers {
-		if strings.HasSuffix(name, ".output") {
-			continue
-		}
-		if strings.HasSuffix(name, ".quoted") {
-			continue
-		}
-
-		t.Log(name)
-		crasher, err := ioutil.ReadFile(dir + "/" + name)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		Fuzz(crasher)
-	}
-}

lib/protocol/header.go

@@ -1,44 +0,0 @@
-// Copyright (C) 2014 The Protocol Authors.
-
-package protocol
-
-import "github.com/calmh/xdr"
-
-type header struct {
-	version     int
-	msgID       int
-	msgType     int
-	compression bool
-}
-
-func (h header) MarshalXDRInto(m *xdr.Marshaller) error {
-	v := encodeHeader(h)
-	m.MarshalUint32(v)
-	return m.Error
-}
-
-func (h *header) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	v := u.UnmarshalUint32()
-	*h = decodeHeader(v)
-	return u.Error
-}
-
-func encodeHeader(h header) uint32 {
-	var isComp uint32
-	if h.compression {
-		isComp = 1 << 0 // the zeroth bit is the compression bit
-	}
-	return uint32(h.version&0xf)<<28 +
-		uint32(h.msgID&0xfff)<<16 +
-		uint32(h.msgType&0xff)<<8 +
-		isComp
-}
-
-func decodeHeader(u uint32) header {
-	return header{
-		version:     int(u>>28) & 0xf,
-		msgID:       int(u>>16) & 0xfff,
-		msgType:     int(u>>8) & 0xff,
-		compression: u&1 == 1,
-	}
-}

lib/protocol/hello.go

@@ -9,10 +9,10 @@ import (
 	"io"
 )
 
-// The HelloMessage interface is implemented by the version specific hello
+// The HelloIntf interface is implemented by the version specific hello
 // message. It knows its magic number and how to serialize itself to a byte
 // buffer.
-type HelloMessage interface {
+type HelloIntf interface {
 	Magic() uint32
 	Marshal() ([]byte, error)
 }
@@ -29,12 +29,15 @@ var (
 	// ErrTooOldVersion12 is returned by ExchangeHello when the other side
 	// speaks the older, incompatible version 0.12 of the protocol.
 	ErrTooOldVersion12 = errors.New("the remote device speaks an older version of the protocol (v0.12) not compatible with this version")
+	// ErrTooOldVersion13 is returned by ExchangeHello when the other side
+	// speaks the older, incompatible version 0.12 of the protocol.
+	ErrTooOldVersion13 = errors.New("the remote device speaks an older version of the protocol (v0.13) not compatible with this version")
 	// ErrUnknownMagic is returned by ExchangeHellow when the other side
 	// speaks something entirely unknown.
 	ErrUnknownMagic = errors.New("the remote device speaks an unknown (newer?) version of the protocol")
 )
 
-func ExchangeHello(c io.ReadWriter, h HelloMessage) (HelloResult, error) {
+func ExchangeHello(c io.ReadWriter, h HelloIntf) (HelloResult, error) {
 	if err := writeHello(c, h); err != nil {
 		return HelloResult{}, err
 	}
@@ -45,7 +48,7 @@ func ExchangeHello(c io.ReadWriter, h HelloMessage) (HelloResult, error) {
 // version mismatch that we might want to alert the user about.
 func IsVersionMismatch(err error) bool {
 	switch err {
-	case ErrTooOldVersion12, ErrUnknownMagic:
+	case ErrTooOldVersion12, ErrTooOldVersion13, ErrUnknownMagic:
 		return true
 	default:
 		return false
@@ -53,15 +56,43 @@ func IsVersionMismatch(err error) bool {
 }
 
 func readHello(c io.Reader) (HelloResult, error) {
-	header := make([]byte, 8)
+	header := make([]byte, 4)
 	if _, err := io.ReadFull(c, header); err != nil {
 		return HelloResult{}, err
 	}
 
-	switch binary.BigEndian.Uint32(header[:4]) {
+	switch binary.BigEndian.Uint32(header) {
+	case HelloMessageMagic:
+		// This is a v0.14 Hello message in proto format
+		if _, err := io.ReadFull(c, header[:2]); err != nil {
+			return HelloResult{}, err
+		}
+		msgSize := binary.BigEndian.Uint16(header[:2])
+		if msgSize > 32767 {
+			return HelloResult{}, fmt.Errorf("hello message too big")
+		}
+		buf := make([]byte, msgSize)
+		if _, err := io.ReadFull(c, buf); err != nil {
+			return HelloResult{}, err
+		}
+
+		var hello Hello
+		if err := hello.Unmarshal(buf); err != nil {
+			return HelloResult{}, err
+		}
+		res := HelloResult{
+			DeviceName:    hello.DeviceName,
+			ClientName:    hello.ClientName,
+			ClientVersion: hello.ClientVersion,
+		}
+		return res, nil
+
 	case Version13HelloMagic:
 		// This is a v0.13 Hello message in XDR format
-		msgSize := binary.BigEndian.Uint32(header[4:])
+		if _, err := io.ReadFull(c, header[:4]); err != nil {
+			return HelloResult{}, err
+		}
+		msgSize := binary.BigEndian.Uint32(header[:4])
 		if msgSize > 1024 {
 			return HelloResult{}, fmt.Errorf("hello message too big")
 		}
@@ -79,7 +110,7 @@ func readHello(c io.Reader) (HelloResult, error) {
 			ClientName:    hello.ClientName,
 			ClientVersion: hello.ClientVersion,
 		}
-		return res, nil
+		return res, ErrTooOldVersion13
 
 	case 0x00010001, 0x00010000:
 		// This is the first word of a v0.12 cluster config message.
@@ -90,15 +121,19 @@ func readHello(c io.Reader) (HelloResult, error) {
 	return HelloResult{}, ErrUnknownMagic
 }
 
-func writeHello(c io.Writer, h HelloMessage) error {
+func writeHello(c io.Writer, h HelloIntf) error {
 	msg, err := h.Marshal()
 	if err != nil {
 		return err
 	}
+	if len(msg) > 32767 {
+		// The header length must be a positive signed int16
+		panic("bug: attempting to serialize too large hello message")
+	}
 
-	header := make([]byte, 8)
+	header := make([]byte, 6)
 	binary.BigEndian.PutUint32(header[:4], h.Magic())
-	binary.BigEndian.PutUint32(header[4:], uint32(len(msg)))
+	binary.BigEndian.PutUint16(header[4:], uint16(len(msg)))
 
 	_, err = c.Write(append(header, msg...))
 	return err

lib/protocol/hello_test.go

@@ -13,6 +13,53 @@ import (
 
 var spaceRe = regexp.MustCompile(`\s`)
 
+func TestVersion14Hello(t *testing.T) {
+	// Tests that we can send and receive a version 0.14 hello message.
+
+	expected := Hello{
+		DeviceName:    "test device",
+		ClientName:    "syncthing",
+		ClientVersion: "v0.14.5",
+	}
+	msgBuf, err := expected.Marshal()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	hdrBuf := make([]byte, 6)
+	binary.BigEndian.PutUint32(hdrBuf, HelloMessageMagic)
+	binary.BigEndian.PutUint16(hdrBuf[4:], uint16(len(msgBuf)))
+
+	outBuf := new(bytes.Buffer)
+	outBuf.Write(hdrBuf)
+	outBuf.Write(msgBuf)
+
+	inBuf := new(bytes.Buffer)
+
+	conn := &readWriter{outBuf, inBuf}
+
+	send := &Hello{
+		DeviceName:    "this device",
+		ClientName:    "other client",
+		ClientVersion: "v0.14.6",
+	}
+
+	res, err := ExchangeHello(conn, send)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if res.ClientName != expected.ClientName {
+		t.Errorf("incorrect ClientName %q != expected %q", res.ClientName, expected.ClientName)
+	}
+	if res.ClientVersion != expected.ClientVersion {
+		t.Errorf("incorrect ClientVersion %q != expected %q", res.ClientVersion, expected.ClientVersion)
+	}
+	if res.DeviceName != expected.DeviceName {
+		t.Errorf("incorrect DeviceName %q != expected %q", res.DeviceName, expected.DeviceName)
+	}
+}
+
 func TestVersion13Hello(t *testing.T) {
 	// Tests that we can send and receive a version 0.13 hello message.
 
@@ -42,8 +89,8 @@ func TestVersion13Hello(t *testing.T) {
 	}
 
 	res, err := ExchangeHello(conn, send)
-	if err != nil {
-		t.Fatal(err)
+	if err != ErrTooOldVersion13 {
+		t.Errorf("unexpected error %v != ErrTooOldVersion13", err)
 	}
 
 	if res.ClientName != expected.ClientName {
@@ -94,7 +141,7 @@ func TestVersion12Hello(t *testing.T) {
 
 	_, err := ExchangeHello(conn, send)
 	if err != ErrTooOldVersion12 {
-		t.Errorf("unexpected error %v != ErrTooOld", err)
+		t.Errorf("unexpected error %v != ErrTooOldVersion12", err)
 	}
 }
 

lib/protocol/message.go

@@ -1,186 +0,0 @@
-// Copyright (C) 2014 The Protocol Authors.
-
-//go:generate -command genxdr go run ../../vendor/github.com/calmh/xdr/cmd/genxdr/main.go
-//go:generate genxdr -o message_xdr.go message.go
-
-package protocol
-
-import (
-	"bytes"
-	"crypto/sha256"
-	"fmt"
-)
-
-var (
-	sha256OfEmptyBlock = sha256.Sum256(make([]byte, BlockSize))
-)
-
-type IndexMessage struct {
-	Folder  string     // max:256
-	Files   []FileInfo // max:1000000
-	Flags   uint32
-	Options []Option // max:64
-}
-
-type FileInfo struct {
-	Name         string // max:8192
-	Flags        uint32
-	Modified     int64
-	Version      Vector
-	LocalVersion int64
-	CachedSize   int64       // noencode (cache only)
-	Blocks       []BlockInfo // max:10000000
-}
-
-func (f FileInfo) String() string {
-	return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%v, Size:%d, Blocks:%v}",
-		f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.Blocks)
-}
-
-func (f FileInfo) Size() (bytes int64) {
-	if f.IsDeleted() || f.IsDirectory() {
-		return 128
-	}
-	if f.CachedSize > 0 {
-		return f.CachedSize
-	}
-	for _, b := range f.Blocks {
-		bytes += int64(b.Size)
-	}
-	f.CachedSize = bytes
-	return
-}
-
-func (f FileInfo) IsDeleted() bool {
-	return f.Flags&FlagDeleted != 0
-}
-
-func (f FileInfo) IsInvalid() bool {
-	return f.Flags&FlagInvalid != 0
-}
-
-func (f FileInfo) IsDirectory() bool {
-	return f.Flags&FlagDirectory != 0
-}
-
-func (f FileInfo) IsSymlink() bool {
-	return f.Flags&FlagSymlink != 0
-}
-
-func (f FileInfo) HasPermissionBits() bool {
-	return f.Flags&FlagNoPermBits == 0
-}
-
-// WinsConflict returns true if "f" is the one to choose when it is in
-// conflict with "other".
-func (f FileInfo) WinsConflict(other FileInfo) bool {
-	// If a modification is in conflict with a delete, we pick the
-	// modification.
-	if !f.IsDeleted() && other.IsDeleted() {
-		return true
-	}
-	if f.IsDeleted() && !other.IsDeleted() {
-		return false
-	}
-
-	// The one with the newer modification time wins.
-	if f.Modified > other.Modified {
-		return true
-	}
-	if f.Modified < other.Modified {
-		return false
-	}
-
-	// The modification times were equal. Use the device ID in the version
-	// vector as tie breaker.
-	return f.Version.Compare(other.Version) == ConcurrentGreater
-}
-
-type BlockInfo struct {
-	Offset int64 // noencode (cache only)
-	Size   int32
-	Hash   []byte // max:64
-}
-
-func (b BlockInfo) String() string {
-	return fmt.Sprintf("Block{%d/%d/%x}", b.Offset, b.Size, b.Hash)
-}
-
-// IsEmpty returns true if the block is a full block of zeroes.
-func (b BlockInfo) IsEmpty() bool {
-	return b.Size == BlockSize && bytes.Equal(b.Hash, sha256OfEmptyBlock[:])
-}
-
-type RequestMessage struct {
-	Folder  string // max:256
-	Name    string // max:8192
-	Offset  int64
-	Size    int32
-	Hash    []byte // max:64
-	Flags   uint32
-	Options []Option // max:64
-}
-
-type ResponseMessage struct {
-	Data []byte
-	Code int32
-}
-
-type ClusterConfigMessage struct {
-	Folders []Folder // max:1000000
-	Options []Option // max:64
-}
-
-type DownloadProgressMessage struct {
-	Folder  string                       // max:64
-	Updates []FileDownloadProgressUpdate // max:1000000
-	Flags   uint32
-	Options []Option // max:64
-}
-
-func (o *ClusterConfigMessage) GetOption(key string) string {
-	for _, option := range o.Options {
-		if option.Key == key {
-			return option.Value
-		}
-	}
-	return ""
-}
-
-type Folder struct {
-	ID      string   // max:256
-	Label   string   // max:256
-	Devices []Device // max:1000000
-	Flags   uint32
-	Options []Option // max:64
-}
-
-type Device struct {
-	ID              []byte   // max:32
-	Name            string   // max:64
-	Addresses       []string // max:64,2083
-	Compression     uint32
-	CertName        string // max:64
-	MaxLocalVersion int64
-	Flags           uint32
-	Options         []Option // max:64
-}
-
-type FileDownloadProgressUpdate struct {
-	UpdateType   uint32
-	Name         string // max:8192
-	Version      Vector
-	BlockIndexes []int32 // max:1000000
-}
-
-type Option struct {
-	Key   string // max:64
-	Value string // max:1024
-}
-
-type CloseMessage struct {
-	Reason string // max:1024
-	Code   int32
-}
-
-type EmptyMessage struct{}