gui, man: Update docs & translations

It used to be an int32, but that's unnecessary and the spec now says
int16. Also relaxes the size requirement to that which fits in a signed
int16 instead of limiting to 1024 bytes, to allow for future growth.

As reported in
https://forum.syncthing.net/t/difference-between-documented-and-implemented-protocol/7798

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/3406

.gitignore

@@ -1,7 +1,7 @@
 /syncthing
-/discosrv
+/stdiscosrv
 syncthing.exe
-discosrv.exe
+stdiscosrv.exe
 *.tar.gz
 *.zip
 *.asc

README.md

@@ -27,6 +27,11 @@ There are a few examples for keeping Syncthing running in the background
 on your system in [the etc directory][3]. There are also several [GUI
 implementations][11] for Windows, Mac and Linux.
 
+## Vote on features/bugs
+
+We'd like to encourage you to [vote][12] on issues that matter to you.
+This helps the team understand what are the biggest pain points for our users, and could potentially influence what is being worked on next.
+
 ## Getting in Touch
 
 The first and best point of contact is the [Forum][8]. There is also an IRC
@@ -66,3 +71,4 @@ All code is licensed under the [MPLv2 License][7].
 [9]: https://kiwiirc.com/client/irc.freenode.net/#syncthing
 [10]: https://github.com/syncthing/syncthing/issues
 [11]: http://docs.syncthing.net/users/contrib.html#gui-wrappers
+[12]: https://www.bountysource.com/teams/syncthing/issues

build.go

@@ -96,38 +96,55 @@ var targets = map[string]target{
 			{src: "etc/linux-systemd/user/syncthing.service", dst: "deb/usr/lib/systemd/user/syncthing.service", perm: 0644},
 		},
 	},
-	"discosrv": {
-		name:       "discosrv",
-		buildPkg:   "./cmd/discosrv",
-		binaryName: "discosrv", // .exe will be added automatically for Windows builds
+	"stdiscosrv": {
+		name:       "stdiscosrv",
+		buildPkg:   "./cmd/stdiscosrv",
+		binaryName: "stdiscosrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
-			{src: "cmd/discosrv/README.md", dst: "README.txt", perm: 0644},
-			{src: "cmd/discosrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
+			{src: "cmd/stdiscosrv/README.md", dst: "README.txt", perm: 0644},
+			{src: "cmd/stdiscosrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
 		debianFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
-			{src: "cmd/discosrv/README.md", dst: "deb/usr/share/doc/discosrv/README.txt", perm: 0644},
-			{src: "cmd/discosrv/LICENSE", dst: "deb/usr/share/doc/discosrv/LICENSE.txt", perm: 0644},
-			{src: "AUTHORS", dst: "deb/usr/share/doc/discosrv/AUTHORS.txt", perm: 0644},
+			{src: "cmd/stdiscosrv/README.md", dst: "deb/usr/share/doc/stdiscosrv/README.txt", perm: 0644},
+			{src: "cmd/stdiscosrv/LICENSE", dst: "deb/usr/share/doc/stdiscosrv/LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "deb/usr/share/doc/stdiscosrv/AUTHORS.txt", perm: 0644},
 		},
 		tags: []string{"purego"},
 	},
-	"relaysrv": {
-		name:       "relaysrv",
-		buildPkg:   "./cmd/relaysrv",
-		binaryName: "relaysrv", // .exe will be added automatically for Windows builds
+	"strelaysrv": {
+		name:       "strelaysrv",
+		buildPkg:   "./cmd/strelaysrv",
+		binaryName: "strelaysrv", // .exe will be added automatically for Windows builds
 		archiveFiles: []archiveFile{
 			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
-			{src: "cmd/relaysrv/README.md", dst: "README.txt", perm: 0644},
-			{src: "cmd/relaysrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
+			{src: "cmd/strelaysrv/README.md", dst: "README.txt", perm: 0644},
+			{src: "cmd/strelaysrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
 		},
 		debianFiles: []archiveFile{
 			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
-			{src: "cmd/relaysrv/README.md", dst: "deb/usr/share/doc/relaysrv/README.txt", perm: 0644},
-			{src: "cmd/relaysrv/LICENSE", dst: "deb/usr/share/doc/relaysrv/LICENSE.txt", perm: 0644},
+			{src: "cmd/strelaysrv/README.md", dst: "deb/usr/share/doc/strelaysrv/README.txt", perm: 0644},
+			{src: "cmd/strelaysrv/LICENSE", dst: "deb/usr/share/doc/strelaysrv/LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "deb/usr/share/doc/strelaysrv/AUTHORS.txt", perm: 0644},
+		},
+	},
+	"strelaypoolsrv": {
+		name:       "strelaypoolsrv",
+		buildPkg:   "./cmd/strelaypoolsrv",
+		binaryName: "strelaypoolsrv", // .exe will be added automatically for Windows builds
+		archiveFiles: []archiveFile{
+			{src: "{{binary}}", dst: "{{binary}}", perm: 0755},
+			{src: "cmd/strelaypoolsrv/README.md", dst: "README.txt", perm: 0644},
+			{src: "cmd/strelaypoolsrv/LICENSE", dst: "LICENSE.txt", perm: 0644},
+			{src: "AUTHORS", dst: "AUTHORS.txt", perm: 0644},
+		},
+		debianFiles: []archiveFile{
+			{src: "{{binary}}", dst: "deb/usr/bin/{{binary}}", perm: 0755},
+			{src: "cmd/strelaypoolsrv/README.md", dst: "deb/usr/share/doc/relaysrv/README.txt", perm: 0644},
+			{src: "cmd/strelaypoolsrv/LICENSE", dst: "deb/usr/share/doc/relaysrv/LICENSE.txt", perm: 0644},
 			{src: "AUTHORS", dst: "deb/usr/share/doc/relaysrv/AUTHORS.txt", perm: 0644},
 		},
 	},
@@ -238,8 +255,8 @@ func runCommand(cmd string, target target) {
 	case "assets":
 		rebuildAssets()
 
-	case "xdr":
-		xdr()
+	case "proto":
+		proto()
 
 	case "translate":
 		translate()
@@ -271,9 +288,12 @@ func runCommand(cmd string, target target) {
 	case "metalint":
 		if isGometalinterInstalled() {
 			dirs := []string{".", "./cmd/...", "./lib/..."}
-			gometalinter("deadcode", dirs, "test/util.go")
-			gometalinter("structcheck", dirs)
-			gometalinter("varcheck", dirs)
+			ok := gometalinter("deadcode", dirs, "test/util.go")
+			ok = gometalinter("structcheck", dirs) && ok
+			ok = gometalinter("varcheck", dirs) && ok
+			if !ok {
+				os.Exit(1)
+			}
 		}
 
 	default:
@@ -543,16 +563,17 @@ func listFiles(dir string) []string {
 
 func rebuildAssets() {
 	runPipe("lib/auto/gui.files.go", "go", "run", "script/genassets.go", "gui")
+	runPipe("cmd/strelaypoolsrv/auto/gui.go", "go", "run", "script/genassets.go", "cmd/strelaypoolsrv/gui")
 }
 
 func lazyRebuildAssets() {
-	if shouldRebuildAssets() {
+	if shouldRebuildAssets("lib/auto/gui.files.go", "gui") || shouldRebuildAssets("cmd/strelaypoolsrv/auto/gui.go", "cmd/strelaypoolsrv/auto/gui") {
 		rebuildAssets()
 	}
 }
 
-func shouldRebuildAssets() bool {
-	info, err := os.Stat("lib/auto/gui.files.go")
+func shouldRebuildAssets(target, srcdir string) bool {
+	info, err := os.Stat(target)
 	if err != nil {
 		// If the file doesn't exist, we must rebuild it
 		return true
@@ -562,7 +583,7 @@ func shouldRebuildAssets() bool {
 	// so we should rebuild it.
 	currentBuild := info.ModTime()
 	assetsAreNewer := false
-	filepath.Walk("gui", func(path string, info os.FileInfo, err error) error {
+	filepath.Walk(srcdir, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -576,8 +597,8 @@ func shouldRebuildAssets() bool {
 	return assetsAreNewer
 }
 
-func xdr() {
-	runPrint("go", "generate", "./lib/discover", "./lib/db", "./lib/protocol", "./lib/relay/protocol")
+func proto() {
+	runPrint("go", "generate", "./lib/...")
 }
 
 func translate() {
@@ -717,10 +738,18 @@ func getBranchSuffix() string {
 }
 
 func buildStamp() int64 {
+	// If SOURCE_DATE_EPOCH is set, use that.
+	if s, _ := strconv.ParseInt(os.Getenv("SOURCE_DATE_EPOCH"), 10, 64); s > 0 {
+		return s
+	}
+
+	// Try to get the timestamp of the latest commit.
 	bs, err := runError("git", "show", "-s", "--format=%ct")
 	if err != nil {
+		// Fall back to "now".
 		return time.Now().Unix()
 	}
+
 	s, _ := strconv.ParseInt(string(bs), 10, 64)
 	return s
 }
@@ -948,13 +977,17 @@ func lint(pkg string) {
 	}
 
 	analCommentPolicy := regexp.MustCompile(`exported (function|method|const|type|var) [^\s]+ should have comment`)
-	for _, line := range bytes.Split(bs, []byte("\n")) {
-		if analCommentPolicy.Match(line) {
+	for _, line := range strings.Split(string(bs), "\n") {
+		if line == "" {
 			continue
 		}
-		if len(line) > 0 {
-			log.Printf("%s", line)
+		if analCommentPolicy.MatchString(line) {
+			continue
 		}
+		if strings.Contains(line, ".pb.go:") {
+			continue
+		}
+		log.Println(line)
 	}
 }
 
@@ -995,7 +1028,7 @@ func isGometalinterInstalled() bool {
 	return true
 }
 
-func gometalinter(linter string, dirs []string, excludes ...string) {
+func gometalinter(linter string, dirs []string, excludes ...string) bool {
 	params := []string{"--disable-all"}
 	params = append(params, fmt.Sprintf("--deadline=%ds", 60))
 	params = append(params, "--enable="+linter)
@@ -1008,12 +1041,19 @@ func gometalinter(linter string, dirs []string, excludes ...string) {
 		params = append(params, dir)
 	}
 
-	bs, err := runError("gometalinter", params...)
+	bs, _ := runError("gometalinter", params...)
 
-	if len(bs) > 0 {
-		log.Printf("%s", bs)
-	}
-	if err != nil {
-		log.Printf("%v", err)
+	nerr := 0
+	for _, line := range strings.Split(string(bs), "\n") {
+		if line == "" {
+			continue
+		}
+		if strings.Contains(line, ".pb.go:") {
+			continue
+		}
+		log.Println(line)
+		nerr++
 	}
+
+	return nerr == 0
 }

cmd/stdisco/main.go

@@ -9,6 +9,7 @@ package main
 import (
 	"bytes"
 	"crypto/rand"
+	"encoding/binary"
 	"flag"
 	"log"
 	"strings"
@@ -66,24 +67,25 @@ func recv(bc beacon.Interface) {
 	seen := make(map[string]bool)
 	for {
 		data, src := bc.Recv()
-		var ann discover.Announce
-		ann.UnmarshalXDR(data)
+		if m := binary.BigEndian.Uint32(data); m != discover.Magic {
+			log.Printf("Incorrect magic %x in announcement from %v", m, src)
+			continue
+		}
 
-		if bytes.Equal(ann.This.ID, myID) {
+		var ann discover.Announce
+		ann.Unmarshal(data[4:])
+
+		if bytes.Equal(ann.ID, myID) {
 			// This is one of our own fake packets, don't print it.
 			continue
 		}
 
 		// Print announcement details for the first packet from a given
 		// device ID and source address, or if -all was given.
-		key := string(ann.This.ID) + src.String()
+		key := string(ann.ID) + src.String()
 		if all || !seen[key] {
 			log.Printf("Announcement from %v\n", src)
-			log.Printf(" %v at %s\n", protocol.DeviceIDFromBytes(ann.This.ID), strings.Join(addrStrs(ann.This), ", "))
-
-			for _, dev := range ann.Extra {
-				log.Printf(" %v at %s\n", protocol.DeviceIDFromBytes(dev.ID), strings.Join(addrStrs(dev), ", "))
-			}
+			log.Printf(" %v at %s\n", protocol.DeviceIDFromBytes(ann.ID), strings.Join(ann.Addresses, ", "))
 			seen[key] = true
 		}
 	}
@@ -92,15 +94,10 @@ func recv(bc beacon.Interface) {
 // sends fake discovery announcements once every second
 func send(bc beacon.Interface) {
 	ann := discover.Announce{
-		Magic: discover.AnnouncementMagic,
-		This: discover.Device{
-			ID: myID,
-			Addresses: []discover.Address{
-				{URL: "tcp://fake.example.com:12345"},
-			},
-		},
+		ID:        myID,
+		Addresses: []string{"tcp://fake.example.com:12345"},
 	}
-	bs, _ := ann.MarshalXDR()
+	bs, _ := ann.Marshal()
 
 	for {
 		bc.Send(bs)
@@ -108,15 +105,6 @@ func send(bc beacon.Interface) {
 	}
 }
 
-// returns the list of address URLs
-func addrStrs(dev discover.Device) []string {
-	ss := make([]string, len(dev.Addresses))
-	for i, addr := range dev.Addresses {
-		ss[i] = addr.URL
-	}
-	return ss
-}
-
 // returns a random but recognizable device ID
 func randomDeviceID() []byte {
 	var id [32]byte

cmd/stdiscosrv/README.md

@@ -1,12 +1,12 @@
-discosrv
-========
+stdiscosrv
+==========
 
-[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/discosrv.svg?style=flat-square)](http://build.syncthing.net/job/discosrv/lastBuild/)
+[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/stdiscosrv.svg?style=flat-square)](http://build.syncthing.net/job/stdiscosrv/lastBuild/)
 
 This is the global discovery server for the `syncthing` project.
 
-To get it, run `go get github.com/syncthing/discosrv` or download the
-[latest build](http://build.syncthing.net/job/discosrv/lastSuccessfulBuild/artifact/)
+To get it, run `go get github.com/syncthing/stdiscosrv` or download the
+[latest build](http://build.syncthing.net/job/stdiscosrv/lastSuccessfulBuild/artifact/)
 from the build server.
 
 Usage
@@ -19,15 +19,15 @@ By default it will use in-memory `ql` backend. If you wish to persist the
 information on disk between restarts in `ql`, specify a file DSN:
 
 ```bash
-$ discosrv -db-dsn="file:///var/run/discosrv.db"
+$ stdiscosrv -db-dsn="file:///var/run/stdiscosrv.db"
 ```
 
 For `postgres`, you will need to create a database and a user with permissions
-to create tables in it, then start the discosrv as follows:
+to create tables in it, then start the stdiscosrv as follows:
 
 ```bash
-$ export DISCOSRV_DB_DSN="postgres://user:password@localhost/databasename"
-$ discosrv -db-backend="postgres"
+$ export STDISCOSRV_DB_DSN="postgres://user:password@localhost/databasename"
+$ stdiscosrv -db-backend="postgres"
 ```
 
 You can pass the DSN as command line option, but the value what you pass in will
@@ -37,4 +37,4 @@ to other users.
 In all cases, the appropriate tables and indexes will be created at first
 startup. If it doesn't exit with an error, you're fine.
 
-See `discosrv -help` for other options.
+See `stdiscosrv -help` for other options.

cmd/stdiscosrv/main.go

@@ -38,7 +38,7 @@ func init() {
 	BuildDate = time.Unix(int64(stamp), 0)
 
 	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`discosrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
+	LongVersion = fmt.Sprintf(`stdiscosrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
 }
 
 var (
@@ -48,7 +48,7 @@ var (
 	globalStats stats
 	statsFile   string
 	backend     = "ql"
-	dsn         = getEnvDefault("DISCOSRV_DB_DSN", "memory://discosrv")
+	dsn         = getEnvDefault("STDISCOSRV_DB_DSN", "memory://stdiscosrv")
 	certFile    = "cert.pem"
 	keyFile     = "key.pem"
 	debug       = false

cmd/stdiscosrv/psql.go

@@ -28,7 +28,7 @@ func postgresSetup(db *sql.DB) error {
 	}
 
 	row := db.QueryRow(`SELECT 'DevicesDeviceIDIndex'::regclass`)
-	if err := row.Scan(nil); err != nil {
+	if err = row.Scan(nil); err != nil {
 		_, err = db.Exec(`CREATE INDEX DevicesDeviceIDIndex ON Devices (DeviceID)`)
 	}
 	if err != nil {
@@ -36,7 +36,7 @@ func postgresSetup(db *sql.DB) error {
 	}
 
 	row = db.QueryRow(`SELECT 'DevicesSeenIndex'::regclass`)
-	if err := row.Scan(nil); err != nil {
+	if err = row.Scan(nil); err != nil {
 		_, err = db.Exec(`CREATE INDEX DevicesSeenIndex ON Devices (Seen)`)
 	}
 	if err != nil {
@@ -53,7 +53,7 @@ func postgresSetup(db *sql.DB) error {
 	}
 
 	row = db.QueryRow(`SELECT 'AddressesDeviceIDSeenIndex'::regclass`)
-	if err := row.Scan(nil); err != nil {
+	if err = row.Scan(nil); err != nil {
 		_, err = db.Exec(`CREATE INDEX AddressesDeviceIDSeenIndex ON Addresses (DeviceID, Seen)`)
 	}
 	if err != nil {
@@ -61,7 +61,7 @@ func postgresSetup(db *sql.DB) error {
 	}
 
 	row = db.QueryRow(`SELECT 'AddressesDeviceIDAddressIndex'::regclass`)
-	if err := row.Scan(nil); err != nil {
+	if err = row.Scan(nil); err != nil {
 		_, err = db.Exec(`CREATE INDEX AddressesDeviceIDAddressIndex ON Addresses (DeviceID, Address)`)
 	}
 	if err != nil {

cmd/stdiscosrv/querysrv.go

@@ -330,6 +330,16 @@ func (s *querysrv) handleAnnounce(ctx context.Context, remote net.IP, deviceID p
 
 		ip := net.ParseIP(host)
 		if host == "" || ip.IsUnspecified() {
+			// Do not use IPv6 remote address if requested scheme is tcp4
+			if uri.Scheme == "tcp4" && remote.To4() == nil {
+				continue
+			}
+
+			// Do not use IPv4 remote address if requested scheme is tcp6
+			if uri.Scheme == "tcp6" && remote.To4() != nil {
+				continue
+			}
+
 			host = remote.String()
 		}
 

cmd/stindex/dump.go

@@ -13,50 +13,61 @@ import (
 
 	"github.com/syncthing/syncthing/lib/db"
 	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syndtr/goleveldb/leveldb"
 )
 
-func dump(ldb *leveldb.DB) {
+func dump(ldb *db.Instance) {
 	it := ldb.NewIterator(nil, nil)
-	var dev protocol.DeviceID
 	for it.Next() {
 		key := it.Key()
 		switch key[0] {
 		case db.KeyTypeDevice:
-			folder := nulString(key[1 : 1+64])
-			devBytes := key[1+64 : 1+64+32]
-			name := nulString(key[1+64+32:])
-			copy(dev[:], devBytes)
-			fmt.Printf("[device] F:%q N:%q D:%v\n", folder, name, dev)
+			folder := binary.BigEndian.Uint32(key[1:])
+			device := binary.BigEndian.Uint32(key[1+4:])
+			name := nulString(key[1+4+4:])
+			fmt.Printf("[device] F:%d D:%d N:%q", folder, device, name)
 
 			var f protocol.FileInfo
-			err := f.UnmarshalXDR(it.Value())
+			err := f.Unmarshal(it.Value())
 			if err != nil {
 				log.Fatal(err)
 			}
-			fmt.Printf("  N:%q\n  F:%#o\n  M:%d\n  V:%v\n  S:%d\n  B:%d\n", f.Name, f.Flags, f.Modified, f.Version, f.Size(), len(f.Blocks))
+			fmt.Printf(" V:%v\n", f)
 
 		case db.KeyTypeGlobal:
-			folder := nulString(key[1 : 1+64])
-			name := nulString(key[1+64:])
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
 			var flv db.VersionList
-			flv.UnmarshalXDR(it.Value())
-			fmt.Printf("[global] F:%q N:%q V: %s\n", folder, name, flv)
+			flv.Unmarshal(it.Value())
+			fmt.Printf("[global] F:%d N:%q V:%s\n", folder, name, flv)
 
 		case db.KeyTypeBlock:
-			folder := nulString(key[1 : 1+64])
-			hash := key[1+64 : 1+64+32]
-			name := nulString(key[1+64+32:])
-			fmt.Printf("[block] F:%q H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
+			folder := binary.BigEndian.Uint32(key[1:])
+			hash := key[1+4 : 1+4+32]
+			name := nulString(key[1+4+32:])
+			fmt.Printf("[block] F:%d H:%x N:%q I:%d\n", folder, hash, name, binary.BigEndian.Uint32(it.Value()))
 
 		case db.KeyTypeDeviceStatistic:
-			fmt.Printf("[dstat]\n  %x\n  %x\n", it.Key(), it.Value())
+			fmt.Printf("[dstat] K:%x V:%x\n", it.Key(), it.Value())
 
 		case db.KeyTypeFolderStatistic:
-			fmt.Printf("[fstat]\n  %x\n  %x\n", it.Key(), it.Value())
+			fmt.Printf("[fstat] K:%x V:%x\n", it.Key(), it.Value())
 
 		case db.KeyTypeVirtualMtime:
-			fmt.Printf("[mtime]\n  %x\n  %x\n", it.Key(), it.Value())
+			fmt.Printf("[mtime] K:%x V:%x\n", it.Key(), it.Value())
+
+		case db.KeyTypeFolderIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			fmt.Printf("[folderidx] K:%d V:%q\n", key, it.Value())
+
+		case db.KeyTypeDeviceIdx:
+			key := binary.BigEndian.Uint32(it.Key()[1:])
+			val := it.Value()
+			if len(val) == 0 {
+				fmt.Printf("[deviceidx] K:%d V:<nil>\n", key)
+			} else {
+				dev := protocol.DeviceIDFromBytes(val)
+				fmt.Printf("[deviceidx] K:%d V:%s\n", key, dev)
+			}
 
 		default:
 			fmt.Printf("[???]\n  %x\n  %x\n", it.Key(), it.Value())

cmd/stindex/dumpsize.go

@@ -8,11 +8,10 @@ package main
 
 import (
 	"container/heap"
+	"encoding/binary"
 	"fmt"
 
 	"github.com/syncthing/syncthing/lib/db"
-	"github.com/syncthing/syncthing/lib/protocol"
-	"github.com/syndtr/goleveldb/leveldb"
 )
 
 type SizedElement struct {
@@ -38,33 +37,31 @@ func (h *ElementHeap) Pop() interface{} {
 	return x
 }
 
-func dumpsize(ldb *leveldb.DB) {
+func dumpsize(ldb *db.Instance) {
 	h := &ElementHeap{}
 	heap.Init(h)
 
 	it := ldb.NewIterator(nil, nil)
-	var dev protocol.DeviceID
 	var ele SizedElement
 	for it.Next() {
 		key := it.Key()
 		switch key[0] {
 		case db.KeyTypeDevice:
-			folder := nulString(key[1 : 1+64])
-			devBytes := key[1+64 : 1+64+32]
-			name := nulString(key[1+64+32:])
-			copy(dev[:], devBytes)
-			ele.key = fmt.Sprintf("DEVICE:%s:%s:%s", dev, folder, name)
+			folder := binary.BigEndian.Uint32(key[1:])
+			device := binary.BigEndian.Uint32(key[1+4:])
+			name := nulString(key[1+4+4:])
+			ele.key = fmt.Sprintf("DEVICE:%d:%d:%s", folder, device, name)
 
 		case db.KeyTypeGlobal:
-			folder := nulString(key[1 : 1+64])
-			name := nulString(key[1+64:])
-			ele.key = fmt.Sprintf("GLOBAL:%s:%s", folder, name)
+			folder := binary.BigEndian.Uint32(key[1:])
+			name := nulString(key[1+4:])
+			ele.key = fmt.Sprintf("GLOBAL:%d:%s", folder, name)
 
 		case db.KeyTypeBlock:
-			folder := nulString(key[1 : 1+64])
-			hash := key[1+64 : 1+64+32]
-			name := nulString(key[1+64+32:])
-			ele.key = fmt.Sprintf("BLOCK:%s:%x:%s", folder, hash, name)
+			folder := binary.BigEndian.Uint32(key[1:])
+			hash := key[1+4 : 1+4+32]
+			name := nulString(key[1+4+32:])
+			ele.key = fmt.Sprintf("BLOCK:%d:%x:%s", folder, hash, name)
 
 		case db.KeyTypeDeviceStatistic:
 			ele.key = fmt.Sprintf("DEVICESTATS:%s", key[1:])
@@ -75,6 +72,14 @@ func dumpsize(ldb *leveldb.DB) {
 		case db.KeyTypeVirtualMtime:
 			ele.key = fmt.Sprintf("MTIME:%s", key[1:])
 
+		case db.KeyTypeFolderIdx:
+			id := binary.BigEndian.Uint32(key[1:])
+			ele.key = fmt.Sprintf("FOLDERIDX:%d", id)
+
+		case db.KeyTypeDeviceIdx:
+			id := binary.BigEndian.Uint32(key[1:])
+			ele.key = fmt.Sprintf("DEVICEIDX:%d", id)
+
 		default:
 			ele.key = fmt.Sprintf("UNKNOWN:%x", key)
 		}

cmd/stindex/main.go

@@ -13,8 +13,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/syndtr/goleveldb/leveldb"
-	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syncthing/syncthing/lib/db"
 )
 
 func main() {
@@ -28,16 +27,12 @@ func main() {
 
 	path := flag.Arg(0)
 	if path == "" {
-		path = filepath.Join(defaultConfigDir(), "index-v0.11.0.db")
+		path = filepath.Join(defaultConfigDir(), "index-v0.14.0.db")
 	}
 
 	fmt.Println("Path:", path)
 
-	ldb, err := leveldb.OpenFile(path, &opt.Options{
-		ErrorIfMissing:         true,
-		Strict:                 opt.StrictAll,
-		OpenFilesCacheCapacity: 100,
-	})
+	ldb, err := db.Open(path)
 	if err != nil {
 		log.Fatal(err)
 	}

cmd/strelaypoolsrv/README.md

@@ -0,0 +1,15 @@
+# relaypoolsrv
+
+[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/relaypoolsrv.svg?style=flat-square)](http://build.syncthing.net/job/relaypoolsrv/lastBuild/)
+
+This is the relay pool server for the `syncthing` project, which allows community hosted [relaysrv](https://github.com/syncthing/relaysrv)'s to join the public pool.
+
+Servers that join the pool are then advertised to users of `syncthing` as potential connection points for those who are unable to connect directly due to NAT or firewall issues.
+
+There is very little reason why you'd want to run this yourself, as `relaypoolsrv` is just used for announcement and lookup of public relay servers. If you are looking to setup a private or a public relay, please check the documentation for [relaysrv](https://github.com/syncthing/relaysrv), which also explains how to join the default public pool.
+
+If you still want to run it, you can run `go get github.com/syncthing/relaypoolsrv` download it or download the
+[latest build](http://build.syncthing.net/job/relaypoolsrv/lastSuccessfulBuild/artifact/)
+from the build server.
+
+See `relaypoolsrv -help` for configuration options.

cmd/strelaypoolsrv/auto/.gitignore

@@ -0,0 +1 @@
+gui.go

cmd/strelaypoolsrv/gui/index.html

@@ -0,0 +1,395 @@
+<!DOCTYPE html>
+
+<html lang="en" ng-app="syncthing" ng-controller="relayDataController">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta name="description" content="">
+    <meta name="author" content="">
+
+    <title>Relay stats</title>
+    <link href="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/css/bootstrap.min.css" rel="stylesheet">
+    <link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/font-awesome/4.6.1/css/font-awesome.min.css">
+
+    <style>
+      #map {
+        height: 600px;
+      }
+      .ng-cloak {
+        display: none;
+      }
+      table {
+        font-size: 11px !important;
+        width: 100%;
+        border: 1px;
+
+      }
+      td {
+        padding: 0px !important;
+      }
+      tfoot td {
+        font-weight: bold;
+      }
+    </style>
+  </head>
+
+  <body class="ng-cloak">
+    <div class="container">
+      <h1>Relay Pool Data</h2>
+      <div ng-if="relays === undefined" class="text-center">
+        <img src="//cdnjs.cloudflare.com/ajax/libs/galleriffic/2.0.1/css/loader.gif"/>
+        <p>Please wait while we gather data</p>
+      </div>
+      <div>
+        <div ng-show="relays !== undefined" class="ng-hide">
+          <p>
+            Currently {{ relays.length }} relays online ({{ totals.goMaxProcs }} cores in total).
+          </p>
+        </div>
+        <div id="map"></div> <!-- Can't hide the map, otherwise it freaks out -->
+        <p>The circle size represents how much bytes the relay transfered relative to other relays</p>
+      </div>
+      <div>
+        <table class="table table-striped table-condensed table">
+          <thead>
+            <tr>
+              <th rowspan="2">Address</td>
+              <th rowspan="2">
+                <a ng-click="sortType = 'status.numActiveSessions || -1'; sortReverse = !sortReverse">
+                  Sessions
+                  <span ng-show="sortType == 'status.numActiveSessions || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.numActiveSessions || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+              <th rowspan="2">
+                <a ng-click="sortType = 'status.numConnections || -1'; sortReverse = !sortReverse">
+                  Connections
+                  <span ng-show="sortType == 'status.numConnections || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.numConnections || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+              <th rowspan="2">
+                <a ng-click="sortType = 'status.bytesProxied || -1'; sortReverse = !sortReverse">
+                  Data relayed
+                  <span ng-show="sortType == 'status.bytesProxied || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.bytesProxied || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+              <th colspan="6" class="text-center">Transfer rate in the last period</th>
+              <th rowspan="2">
+                <a ng-click="sortType = 'status.uptimeSeconds || -1'; sortReverse = !sortReverse">
+                  Uptime hours
+                  <span ng-show="sortType == 'status.uptimeSeconds || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.uptimeSeconds || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+              <th rowspan="2">
+                <a ng-click="sortType = 'status.options[\'provided-by\'] || \'\''; sortReverse = !sortReverse">
+                  Provided by
+                  <span ng-show="sortType == 'status.options[\'provided-by\'] || \'\'' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.options[\'provided-by\'] || \'\'' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+            </tr>
+            <tr>
+              <th>
+                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[0] || -1'; sortReverse = !sortReverse">
+                  10s
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[0] || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[0] || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+              <th>
+                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[1] || -1'; sortReverse = !sortReverse">
+                  1m
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[1] || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[1] || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+              <th>
+                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[2] || -1'; sortReverse = !sortReverse">
+                  5m
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[2] || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[2] || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+              <th>
+                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[3] || -1'; sortReverse = !sortReverse">
+                  15m
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[3] || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[3] || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+              <th>
+                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[4] || -1'; sortReverse = !sortReverse">
+                  30m
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[4] || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[4] || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+              <th>
+                <a ng-click="sortType = 'status.kbps10s1m5m15m30m60m[5] || -1'; sortReverse = !sortReverse">
+                  60m
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[5] || -1' && !sortReverse" class="fa fa-caret-down"></span>
+                  <span ng-show="sortType == 'status.kbps10s1m5m15m30m60m[5] || -1' && sortReverse" class="fa fa-caret-up"></span>
+                </a>
+              </th>
+            </tr>
+          </thead>
+          <tbody>
+            <tr ng-repeat="relay in relays | orderBy:sortType:sortReverse ">
+              <td>{{ relay.address }}</td>
+              <td ng-if="relay.status === undefined" colspan="11" class="text-center">Looking up...</td>
+              <td ng-if-start="relay.status !== undefined">{{ relay.status.numActiveSessions }}</td>
+              <td>{{ relay.status.numConnections }}</td>
+              <td>{{ relay.status.bytesProxied | bytes }}</td>
+              <td>{{ relay.status.kbps10s1m5m15m30m60m[0] * 128 | bytes }}/s</td>
+              <td>{{ relay.status.kbps10s1m5m15m30m60m[1] * 128 | bytes }}/s</td>
+              <td>{{ relay.status.kbps10s1m5m15m30m60m[2] * 128 | bytes }}/s</td>
+              <td>{{ relay.status.kbps10s1m5m15m30m60m[3] * 128 | bytes }}/s</td>
+              <td>{{ relay.status.kbps10s1m5m15m30m60m[4] * 128 | bytes }}/s</td>
+              <td>{{ relay.status.kbps10s1m5m15m30m60m[5] * 128 | bytes }}/s</td>
+              <td ng-if="relay.status.uptimeSeconds != undefined">{{ relay.status.uptimeSeconds/60/60 | number:0 }}</td>
+              <td ng-if="relay.status.uptimeSeconds == undefined"></td>
+              <td title="{{ relay.status.options['provided-by'] || '' }}" ng-if-end>
+                {{ relay.status.options['provided-by'] || '' | limitTo:50 }}
+                <span ng-if="(relay.status.options['provided-by'] || '').length > 50">&hellip;
+              </td>
+            </tr>
+          </tbody>
+          <tfoot>
+            <tr>
+              <td>Totals</td>
+              <td>{{ totals.numActiveSessions }}</td>
+              <td>{{ totals.numConnections }}</td>
+              <td>{{ totals.bytesProxied | bytes }}</td>
+              <td>{{ totals.kbps10s1m5m15m30m60m[0] * 128 | bytes }}/s</td>
+              <td>{{ totals.kbps10s1m5m15m30m60m[1] * 128 | bytes }}/s</td>
+              <td>{{ totals.kbps10s1m5m15m30m60m[2] * 128 | bytes }}/s</td>
+              <td>{{ totals.kbps10s1m5m15m30m60m[3] * 128 | bytes }}/s</td>
+              <td>{{ totals.kbps10s1m5m15m30m60m[4] * 128 | bytes }}/s</td>
+              <td>{{ totals.kbps10s1m5m15m30m60m[5] * 128 | bytes }}/s</td>
+              <td>{{ totals.uptimeSeconds/60/60 | number:0 }} hours</td>
+              <td>{{ relays.length }} relays</td>
+            </tr>
+          </tfoor>
+        </table>
+      </div>
+      <hr>
+      <p>
+        This product includes GeoLite2 data created by MaxMind, available from
+        <a href="http://www.maxmind.com">http://www.maxmind.com</a>.
+      </p>
+    </div>
+
+
+    <script src="//code.jquery.com/jquery-2.1.4.min.js"></script>
+    <script src="//cdnjs.cloudflare.com/ajax/libs/angular.js/1.4.7/angular.min.js"></script>
+    <script src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.5/js/bootstrap.min.js"></script>
+    <script src="//maps.googleapis.com/maps/api/js"></script>
+  </body>
+
+  <script>
+  angular.module('syncthing', [
+  ])
+  .config(function($httpProvider) {
+    $httpProvider.defaults.timeout = 5000;
+  })
+  .filter('bytes', function() {
+    return function(bytes, precision) {
+      if (isNaN(parseFloat(bytes)) || !isFinite(bytes)) return '-';
+      if (typeof precision === 'undefined') precision = 1;
+
+      var units = ['bytes', 'kB', 'MB', 'GB', 'TB', 'PB'],
+          number = Math.floor(Math.log(bytes) / Math.log(1024));
+
+      var value = (bytes / Math.pow(1000, Math.floor(number)));
+      if (!isFinite(value)) {
+        value = 0;
+        precision = 0;
+      }
+      if (!isFinite(number)) {
+        units = 'bytes';
+      } else {
+        units = units[number];
+      }
+      return value.toFixed(precision) + ' ' + units;
+    }
+  })
+  .controller('relayDataController', ['$scope', '$rootScope', '$http', '$q', '$compile', '$timeout', function($scope, $rootScope, $http, $q, $compile, $timeout) {
+    $scope.totals = {
+      bytesProxied: 0,
+      goMaxProcs: 0,
+      kbps10s1m5m15m30m60m: [0, 0, 0, 0, 0, 0],
+      numActiveSessions: 0,
+      numConnections: 0,
+      numPendingSessionKeys: 0,
+      numProxies: 0,
+      uptimeSeconds: 0,
+    };
+    $scope.map = new google.maps.Map(document.getElementById('map'), {
+      zoom: 1,
+      mapTypeId: google.maps.MapTypeId.ROADMAP
+    });
+    $scope.mapBounds = new google.maps.LatLngBounds();
+    $scope.tooltipTemplate = $('#infoTemplate').html();
+    $scope.usedLocations = {};
+    $scope.sortType = 'status.numActiveSessions || -1';
+    $scope.sortReverse = true;
+
+    $http.get("/endpoint").then(function(response) {
+      $scope.relays = response.data.relays;
+      var promises = [];
+      angular.forEach($scope.relays, function(relay) {
+
+        relay.uri = constructURI(relay.url);
+        relay.address = relay.url.split('/')[2];
+
+        addMarkerToMap(relay);
+
+        promises.push(getRelayStatus(relay));
+      });
+
+      // Can only add circles once we know the totals for transfers, which means
+      // we need to resolve all statuses.
+      $q.all(promises).then(function() {
+        angular.forEach($scope.relays, function(relay) {
+          if (relay.status) {
+            addCircleToMap(relay);
+          }
+        });
+      });
+
+      $scope.map.fitBounds($scope.mapBounds);
+      if ($scope.relays.length == 1) {
+        $scope.map.setZoom(13);
+      }
+    });
+
+    function addMarkerToMap(relay) {
+        var loc = relay.location.latitude + "," + relay.location.longitude;
+
+        // Deal with overlapping markers
+        while (loc in $scope.usedLocations) {
+          var locParts = loc.split(',');
+          locParts = [parseFloat(locParts[0]), parseFloat(locParts[1])];
+          locParts[Math.round(Math.random())] += 0.5 * (Math.random() >= 0.5 ? 1 : -1);
+          loc = locParts.join(',');
+        }
+
+        $scope.usedLocations[loc] = true;
+
+        var locParts = loc.split(',');
+
+        relay.marker = new google.maps.Marker({
+          map: $scope.map,
+          position: new google.maps.LatLng(locParts[0], locParts[1]),
+          title: relay.url,
+        });
+
+        var scope = $rootScope.$new(true);
+        scope.relay = relay;
+
+        relay.marker.info = new google.maps.InfoWindow({
+          content: $compile($scope.tooltipTemplate)(scope)[0],
+        });
+
+        relay.marker.addListener('mouseover', function() {
+          relay.marker.info.open($scope.map, relay.marker);
+        });
+
+        relay.marker.addListener('mouseout', function() {
+          relay.marker.info.close();
+        });
+
+        $scope.mapBounds.extend(relay.marker.position);
+    }
+
+    function addCircleToMap(relay) {
+      relay.marker.circle = new google.maps.Circle({
+        strokeColor: '#FF0000',
+        strokeOpacity: 0.8,
+        strokeWeight: 2,
+        fillColor: '#FF0000',
+        fillOpacity: 0.35,
+        map: $scope.map,
+        center: relay.marker.position,
+        radius: ((relay.status.bytesProxied * 100) / $scope.totals.bytesProxied) * 10000
+      });
+    }
+
+    function getRelayStatus(relay) {
+      // Normal timeout doesn't deal with relays which accept the TCP connection
+      // but don't respond (some firewalls do that), so deal with it this way.
+      var timeoutRequest = $q.defer();
+      var resolveStatus = $q.defer();
+
+      $http.get("http://" + relay.uri.hostname + (relay.uri.args.statusAddr || ":22070") + "/status", { timeout: timeoutRequest.promise }).then(function (response) {
+        relay.status = response.data;
+        resolveStatus.resolve();
+        angular.forEach($scope.totals, function(value, key) {
+          if (typeof $scope.totals[key] == 'number') {
+            $scope.totals[key] += response.data[key];
+          } else if (typeof $scope.totals[key] == 'object' && $scope.totals[key] instanceof Array) {
+            angular.forEach($scope.totals[key], function(value, index) {
+              $scope.totals[key][index] += response.data[key][index];
+            });
+          }
+        });
+      }, function() {
+        relay.status = null;
+        resolveStatus.resolve();
+      });
+
+      $timeout(function() {
+        timeoutRequest.resolve();
+      }, 5000);
+
+      return resolveStatus.promise;
+    }
+
+    function constructURI(url) {
+      var uri = document.createElement('a');
+
+      // HAX, otherwise doesn't work
+      uri.href = url.replace('relay://', 'http://');
+
+      // Convert query string to object
+      uri.args = {};
+      angular.forEach(uri.search.replace(/^\?/, '').split('&'), function(query) {
+          var split = query.split('=');
+          uri.args[split[0]] = split[1];
+      });
+
+      return uri;
+    }
+  }]);
+  </script>
+
+  <script type="text/template" id="infoTemplate">
+    <div>
+      <p><b>{{ relay.uri.hostname }}</b> <span ng-if="relay.status.options['provided-by']">provided by <u>{{ relay.status.options['provided-by'] }}</u></span></p>
+      <div ng-if="relay.status">
+        <span ng-if="relay.status.startTime">Start time: {{ relay.status.startTime | date:"medium" }}</br></span>
+        <span ng-if="relay.status.bytesProxied != undefined">Proxied: {{ relay.status.bytesProxied | bytes }}</br></span>
+        <span ng-if="relay.status.numActiveSessions != undefined">Sessions: {{ relay.status.numActiveSessions }}</br></span>
+        <span ng-if="relay.status.numConnections != undefined">Clients: {{ relay.status.numConnections }}</br></span>
+        <span ng-if="relay.status.options.pools">Pools: {{ relay.status.options.pools.join(', ') }}</br></span>
+        <span ng-if="relay.status.options['global-rate'] != undefined">
+          <span ng-if="relay.status.options['global-rate'] > 0">Global rate limit: {{ relay.status.options['global-rate'] | bytes }}/s</span>
+          <span ng-if="relay.status.options['global-rate'] == 0">Global rate limit: unlimited</span>
+          </br>
+        </span>
+        <span ng-if="relay.status.options['per-session-rate'] != undefined">
+          <span ng-if="relay.status.options['per-session-rate'] > 0">Session rate limit: {{ relay.status.options['per-session-rate'] | bytes }}/s</span>
+          <span ng-if="relay.status.options['per-session-rate'] == 0">Session rate limit: unlimited</span>
+          </br>
+        </span>
+      </div>
+      <div ng-if="!relay.status">
+        Data unavailable.
+      <div>
+    </div>
+  </script>
+</html>

cmd/strelaypoolsrv/main.go

@@ -0,0 +1,536 @@
+// Copyright (C) 2015 Audrius Butkevicius and Contributors (see the CONTRIBUTORS file).
+
+//go:generate go run genassets.go gui auto/gui.go
+
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"crypto/tls"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"mime"
+	"net"
+	"net/http"
+	"net/url"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/golang/groupcache/lru"
+	"github.com/juju/ratelimit"
+
+	"github.com/oschwald/geoip2-golang"
+
+	"github.com/syncthing/syncthing/cmd/strelaypoolsrv/auto"
+	"github.com/syncthing/syncthing/lib/relay/client"
+	"github.com/syncthing/syncthing/lib/sync"
+	"github.com/syncthing/syncthing/lib/tlsutil"
+)
+
+type location struct {
+	Latitude  float64 `json:"latitude"`
+	Longitude float64 `json:"longitude"`
+}
+
+type relay struct {
+	URL      string   `json:"url"`
+	Location location `json:"location"`
+	uri      *url.URL
+}
+
+func (r relay) String() string {
+	return r.URL
+}
+
+type request struct {
+	relay  relay
+	uri    *url.URL
+	result chan result
+}
+
+type result struct {
+	err      error
+	eviction time.Duration
+}
+
+var (
+	testCert       tls.Certificate
+	listen         = ":80"
+	dir            string
+	evictionTime   = time.Hour
+	debug          bool
+	getLRUSize           = 10 << 10
+	getLimitBurst  int64 = 10
+	getLimitAvg          = 1
+	postLRUSize          = 1 << 10
+	postLimitBurst int64 = 2
+	postLimitAvg         = 1
+	getLimit       time.Duration
+	postLimit      time.Duration
+	permRelaysFile string
+	ipHeader       string
+	geoipPath      string
+
+	getMut      = sync.NewRWMutex()
+	getLRUCache *lru.Cache
+
+	postMut      = sync.NewRWMutex()
+	postLRUCache *lru.Cache
+
+	requests = make(chan request, 10)
+
+	mut             = sync.NewRWMutex()
+	knownRelays     = make([]relay, 0)
+	permanentRelays = make([]relay, 0)
+	evictionTimers  = make(map[string]*time.Timer)
+)
+
+func main() {
+	flag.StringVar(&listen, "listen", listen, "Listen address")
+	flag.StringVar(&dir, "keys", dir, "Directory where http-cert.pem and http-key.pem is stored for TLS listening")
+	flag.BoolVar(&debug, "debug", debug, "Enable debug output")
+	flag.DurationVar(&evictionTime, "eviction", evictionTime, "After how long the relay is evicted")
+	flag.IntVar(&getLRUSize, "get-limit-cache", getLRUSize, "Get request limiter cache size")
+	flag.IntVar(&getLimitAvg, "get-limit-avg", 2, "Allowed average get request rate, per 10 s")
+	flag.Int64Var(&getLimitBurst, "get-limit-burst", getLimitBurst, "Allowed burst get requests")
+	flag.IntVar(&postLRUSize, "post-limit-cache", postLRUSize, "Post request limiter cache size")
+	flag.IntVar(&postLimitAvg, "post-limit-avg", 2, "Allowed average post request rate, per minute")
+	flag.Int64Var(&postLimitBurst, "post-limit-burst", postLimitBurst, "Allowed burst post requests")
+	flag.StringVar(&permRelaysFile, "perm-relays", "", "Path to list of permanent relays")
+	flag.StringVar(&ipHeader, "ip-header", "", "Name of header which holds clients ip:port. Only meaningful when running behind a reverse proxy.")
+	flag.StringVar(&geoipPath, "geoip", "GeoLite2-City.mmdb", "Path to GeoLite2-City database")
+
+	flag.Parse()
+
+	getLimit = 10 * time.Second / time.Duration(getLimitAvg)
+	postLimit = time.Minute / time.Duration(postLimitAvg)
+
+	getLRUCache = lru.New(getLRUSize)
+	postLRUCache = lru.New(postLRUSize)
+
+	var listener net.Listener
+	var err error
+
+	if permRelaysFile != "" {
+		loadPermanentRelays(permRelaysFile)
+	}
+
+	testCert = createTestCertificate()
+
+	go requestProcessor()
+
+	if dir != "" {
+		if debug {
+			log.Println("Starting TLS listener on", listen)
+		}
+		certFile, keyFile := filepath.Join(dir, "http-cert.pem"), filepath.Join(dir, "http-key.pem")
+		var cert tls.Certificate
+		cert, err = tls.LoadX509KeyPair(certFile, keyFile)
+		if err != nil {
+			log.Fatalln("Failed to load HTTP X509 key pair:", err)
+		}
+
+		tlsCfg := &tls.Config{
+			Certificates: []tls.Certificate{cert},
+			MinVersion:   tls.VersionTLS10, // No SSLv3
+			CipherSuites: []uint16{
+				// No RC4
+				tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+				tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
+				tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
+				tls.TLS_RSA_WITH_AES_128_CBC_SHA,
+				tls.TLS_RSA_WITH_AES_256_CBC_SHA,
+				tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,
+				tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA,
+			},
+		}
+
+		listener, err = tls.Listen("tcp", listen, tlsCfg)
+	} else {
+		if debug {
+			log.Println("Starting plain listener on", listen)
+		}
+		listener, err = net.Listen("tcp", listen)
+	}
+
+	if err != nil {
+		log.Fatalln("listen:", err)
+	}
+
+	handler := http.NewServeMux()
+	handler.HandleFunc("/", handleAssets)
+	handler.HandleFunc("/endpoint", handleRequest)
+
+	srv := http.Server{
+		Handler:     handler,
+		ReadTimeout: 10 * time.Second,
+	}
+
+	err = srv.Serve(listener)
+	if err != nil {
+		log.Fatalln("serve:", err)
+	}
+}
+
+func handleAssets(w http.ResponseWriter, r *http.Request) {
+	assets := auto.Assets()
+	path := r.URL.Path[1:]
+	if path == "" {
+		path = "index.html"
+	}
+
+	bs, ok := assets[path]
+	if !ok {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+
+	mtype := mimeTypeForFile(path)
+	if len(mtype) != 0 {
+		w.Header().Set("Content-Type", mtype)
+	}
+
+	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
+		w.Header().Set("Content-Encoding", "gzip")
+	} else {
+		// ungzip if browser not send gzip accepted header
+		var gr *gzip.Reader
+		gr, _ = gzip.NewReader(bytes.NewReader(bs))
+		bs, _ = ioutil.ReadAll(gr)
+		gr.Close()
+	}
+	w.Header().Set("Content-Length", fmt.Sprintf("%d", len(bs)))
+
+	w.Write(bs)
+}
+
+func mimeTypeForFile(file string) string {
+	// We use a built in table of the common types since the system
+	// TypeByExtension might be unreliable. But if we don't know, we delegate
+	// to the system.
+	ext := filepath.Ext(file)
+	switch ext {
+	case ".htm", ".html":
+		return "text/html"
+	case ".css":
+		return "text/css"
+	case ".js":
+		return "application/javascript"
+	case ".json":
+		return "application/json"
+	case ".png":
+		return "image/png"
+	case ".ttf":
+		return "application/x-font-ttf"
+	case ".woff":
+		return "application/x-font-woff"
+	case ".svg":
+		return "image/svg+xml"
+	default:
+		return mime.TypeByExtension(ext)
+	}
+}
+
+func handleRequest(w http.ResponseWriter, r *http.Request) {
+	if ipHeader != "" {
+		r.RemoteAddr = r.Header.Get(ipHeader)
+	}
+	w.Header().Set("Access-Control-Allow-Origin", "*")
+	switch r.Method {
+	case "GET":
+		if limit(r.RemoteAddr, getLRUCache, getMut, getLimit, int64(getLimitBurst)) {
+			w.WriteHeader(429)
+			return
+		}
+		handleGetRequest(w, r)
+	case "POST":
+		if limit(r.RemoteAddr, postLRUCache, postMut, postLimit, int64(postLimitBurst)) {
+			w.WriteHeader(429)
+			return
+		}
+		handlePostRequest(w, r)
+	default:
+		if debug {
+			log.Println("Unhandled HTTP method", r.Method)
+		}
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+	}
+}
+
+func handleGetRequest(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	mut.RLock()
+	relays := append(permanentRelays, knownRelays...)
+	mut.RUnlock()
+
+	// Shuffle
+	for i := range relays {
+		j := rand.Intn(i + 1)
+		relays[i], relays[j] = relays[j], relays[i]
+	}
+
+	json.NewEncoder(w).Encode(map[string][]relay{
+		"relays": relays,
+	})
+}
+
+func handlePostRequest(w http.ResponseWriter, r *http.Request) {
+	var newRelay relay
+	err := json.NewDecoder(r.Body).Decode(&newRelay)
+	r.Body.Close()
+
+	if err != nil {
+		if debug {
+			log.Println("Failed to parse payload")
+		}
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	uri, err := url.Parse(newRelay.URL)
+	if err != nil {
+		if debug {
+			log.Println("Failed to parse URI", newRelay.URL)
+		}
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	host, port, err := net.SplitHostPort(uri.Host)
+	if err != nil {
+		if debug {
+			log.Println("Failed to split URI", newRelay.URL)
+		}
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	// Get the IP address of the client
+	rhost, _, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		if debug {
+			log.Println("Failed to split remote address", r.RemoteAddr)
+		}
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	// The client did not provide an IP address, use the IP address of the client.
+	if host == "" {
+		uri.Host = net.JoinHostPort(rhost, port)
+		newRelay.URL = uri.String()
+	} else if host != rhost {
+		if debug {
+			log.Println("IP address advertised does not match client IP address", r.RemoteAddr, uri)
+		}
+		http.Error(w, "IP address does not match client IP", http.StatusUnauthorized)
+		return
+	}
+	newRelay.uri = uri
+	newRelay.Location = getLocation(uri.Host)
+
+	for _, current := range permanentRelays {
+		if current.uri.Host == newRelay.uri.Host {
+			if debug {
+				log.Println("Asked to add a relay", newRelay, "which exists in permanent list")
+			}
+			http.Error(w, "Invalid request", 500)
+			return
+		}
+	}
+
+	reschan := make(chan result)
+
+	select {
+	case requests <- request{newRelay, uri, reschan}:
+		result := <-reschan
+		if result.err != nil {
+			http.Error(w, result.err.Error(), 500)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		json.NewEncoder(w).Encode(map[string]time.Duration{
+			"evictionIn": result.eviction,
+		})
+
+	default:
+		if debug {
+			log.Println("Dropping request")
+		}
+		w.WriteHeader(429)
+	}
+}
+
+func requestProcessor() {
+	for request := range requests {
+		if debug {
+			log.Println("Request for", request.relay)
+		}
+		if !client.TestRelay(request.uri, []tls.Certificate{testCert}, time.Second, 2*time.Second, 3) {
+			if debug {
+				log.Println("Test for relay", request.relay, "failed")
+			}
+			request.result <- result{fmt.Errorf("test failed"), 0}
+			continue
+		}
+
+		mut.Lock()
+		timer, ok := evictionTimers[request.relay.uri.Host]
+		if ok {
+			if debug {
+				log.Println("Stopping existing timer for", request.relay)
+			}
+			timer.Stop()
+		}
+
+		for i, current := range knownRelays {
+			if current.uri.Host == request.relay.uri.Host {
+				if debug {
+					log.Println("Relay", request.relay, "already exists")
+				}
+
+				// Evict the old entry anyway, as configuration might have changed.
+				last := len(knownRelays) - 1
+				knownRelays[i] = knownRelays[last]
+				knownRelays = knownRelays[:last]
+
+				goto found
+			}
+		}
+
+		if debug {
+			log.Println("Adding new relay", request.relay)
+		}
+
+	found:
+
+		knownRelays = append(knownRelays, request.relay)
+
+		evictionTimers[request.relay.uri.Host] = time.AfterFunc(evictionTime, evict(request.relay))
+		mut.Unlock()
+		request.result <- result{nil, evictionTime}
+	}
+
+}
+
+func evict(relay relay) func() {
+	return func() {
+		mut.Lock()
+		defer mut.Unlock()
+		if debug {
+			log.Println("Evicting", relay)
+		}
+		for i, current := range knownRelays {
+			if current.uri.Host == relay.uri.Host {
+				if debug {
+					log.Println("Evicted", relay)
+				}
+				last := len(knownRelays) - 1
+				knownRelays[i] = knownRelays[last]
+				knownRelays = knownRelays[:last]
+			}
+		}
+		delete(evictionTimers, relay.uri.Host)
+	}
+}
+
+func limit(addr string, cache *lru.Cache, lock sync.RWMutex, rate time.Duration, burst int64) bool {
+	host, _, err := net.SplitHostPort(addr)
+	if err != nil {
+		return false
+	}
+
+	lock.RLock()
+	bkt, ok := cache.Get(host)
+	lock.RUnlock()
+	if ok {
+		bkt := bkt.(*ratelimit.Bucket)
+		if bkt.TakeAvailable(1) != 1 {
+			// Rate limit
+			return true
+		}
+	} else {
+		lock.Lock()
+		cache.Add(host, ratelimit.NewBucket(rate, burst))
+		lock.Unlock()
+	}
+	return false
+}
+
+func loadPermanentRelays(file string) {
+	content, err := ioutil.ReadFile(file)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, line := range strings.Split(string(content), "\n") {
+		if len(line) == 0 {
+			continue
+		}
+
+		uri, err := url.Parse(line)
+		if err != nil {
+			if debug {
+				log.Println("Skipping permanent relay", line, "due to parse error", err)
+			}
+			continue
+
+		}
+
+		permanentRelays = append(permanentRelays, relay{
+			URL:      line,
+			Location: getLocation(uri.Host),
+			uri:      uri,
+		})
+		if debug {
+			log.Println("Adding permanent relay", line)
+		}
+	}
+}
+
+func createTestCertificate() tls.Certificate {
+	tmpDir, err := ioutil.TempDir("", "relaypoolsrv")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	certFile, keyFile := filepath.Join(tmpDir, "cert.pem"), filepath.Join(tmpDir, "key.pem")
+	cert, err := tlsutil.NewCertificate(certFile, keyFile, "relaypoolsrv", 3072)
+	if err != nil {
+		log.Fatalln("Failed to create test X509 key pair:", err)
+	}
+
+	return cert
+}
+
+func getLocation(host string) location {
+	db, err := geoip2.Open(geoipPath)
+	if err != nil {
+		return location{}
+	}
+	defer db.Close()
+
+	addr, err := net.ResolveTCPAddr("tcp", host)
+	if err != nil {
+		return location{}
+	}
+
+	city, err := db.City(addr.IP)
+	if err != nil {
+		return location{}
+	}
+
+	return location{
+		Latitude:  city.Location.Latitude,
+		Longitude: city.Location.Longitude,
+	}
+}

cmd/strelaysrv/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 The Syncthing Project
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

cmd/strelaysrv/README.md

@@ -1,12 +1,12 @@
-relaysrv
-========
+strelaysrv
+==========
 
-[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/relaysrv.svg?style=flat-square)](http://build.syncthing.net/job/relaysrv/lastBuild/)
+[![Latest Build](http://img.shields.io/jenkins/s/http/build.syncthing.net/strelaysrv.svg?style=flat-square)](http://build.syncthing.net/job/strelaysrv/lastBuild/)
 
 This is the relay server for the `syncthing` project.
 
-To get it, run `go get github.com/syncthing/relaysrv` or download the
-[latest build](http://build.syncthing.net/job/relaysrv/lastSuccessfulBuild/artifact/)
+To get it, run `go get github.com/syncthing/strelaysrv` or download the
+[latest build](http://build.syncthing.net/job/strelaysrv/lastSuccessfulBuild/artifact/)
 from the build server.
 
 :exclamation:Warnings:exclamation: - Read or regret
@@ -16,13 +16,13 @@ By default, all relay servers will join the default public relay pool, which mea
 
 If you wish to disable this behaviour, please specify `-pools=""` argument.
 
-Please note that `relaysrv` is only usable by `syncthing` **version v0.12 and onwards**.
+Please note that `strelaysrv` is only usable by `syncthing` **version v0.12 and onwards**.
 
-To run `relaysrv` you need to have port 22067 available to the internet, which means you might need to allow it through your firewall if you **have a public IP, or setup a port-forwarding** (22067 to 22067) if you are behind a router.
+To run `strelaysrv` you need to have port 22067 available to the internet, which means you might need to allow it through your firewall if you **have a public IP, or setup a port-forwarding** (22067 to 22067) if you are behind a router.
 
-Furthermore, **by default relaysrv will also expose a /status HTTP endpoint on port 22070**, which is used by the pool servers to peek at metrics of the relaysrv, such as what are the current transfer rates, how many clients are connected, etc, etc. If you wish this information to be available, similarlly you might want to allow it through your firewall, or port-forward it (22070 to 22070) on your NAT device.
+Furthermore, **by default strelaysrv will also expose a /status HTTP endpoint on port 22070**, which is used by the pool servers to peek at metrics of the strelaysrv, such as what are the current transfer rates, how many clients are connected, etc, etc. If you wish this information to be available, similarlly you might want to allow it through your firewall, or port-forward it (22070 to 22070) on your NAT device.
 
-This is **not mandatory** for the relaysrv to function, and is used only to gather metrics and present them in the overview page of the pool server, displaying stats about the specific relay.
+This is **not mandatory** for the strelaysrv to function, and is used only to gather metrics and present them in the overview page of the pool server, displaying stats about the specific relay.
 
 At the point of writing the endpoint output looks as follows:
 
@@ -62,31 +62,31 @@ At the point of writing the endpoint output looks as follows:
 }
 ```
 
-If you wish to disable the /status endpoint, provide `-status-srv=""` as one of the arguments when starting the relaysrv.
+If you wish to disable the /status endpoint, provide `-status-srv=""` as one of the arguments when starting the strelaysrv.
 
 Running for public use
 ----
 Make sure you have a public IP with port 22067 open, or make sure you have port-forwarding (22067 to 22067) if you are behind a router.
 
-Run the `relaysrv` with no arguments (or `-debug` if you want more output), and that should be enough for the server to join the public relay pool.
+Run the `strelaysrv` with no arguments (or `-debug` if you want more output), and that should be enough for the server to join the public relay pool.
 You should see a message saying:
 ```
 2015/09/21 22:45:46 pool.go:60: Joined https://relays.syncthing.net/endpoint rejoining in 48m0s
 ```
 
-See `relaysrv -help` for other options, such as rate limits, timeout intervals, etc.
+See `strelaysrv -help` for other options, such as rate limits, timeout intervals, etc.
 
 Running for private use
 -----
 
-Once you've started the `relaysrv`, it will generate a key pair and print an URI:
+Once you've started the `strelaysrv`, it will generate a key pair and print an URI:
 ```bash
 relay://:22067/?id=EZQOIDM-6DDD4ZI-DJ65NSM-4OQWRAT-EIKSMJO-OZ552BO-WQZEGYY-STS5RQM&pingInterval=1m0s&networkTimeout=2m0s&sessionLimitBps=0&globalLimitBps=0&statusAddr=:22070
 ```
 
 This URI contains partial address of the relay server, as well as it's options which in the future may be taken into account when choosing the best suitable relay out of multiple available.
 
-Because `-listen` option was not used, the `relaysrv` does not know it's external IP, therefore you should replace the host part of the URI with your public IP address on which the `relaysrv` will be available:
+Because `-listen` option was not used, the `strelaysrv` does not know it's external IP, therefore you should replace the host part of the URI with your public IP address on which the `strelaysrv` will be available:
 
 ```bash
 relay://123.123.123.123:22067/?id=EZQOIDM-6DDD4ZI-DJ65NSM-4OQWRAT-EIKSMJO-OZ552BO-WQZEGYY-STS5RQM&pingInterval=1m0s&networkTimeout=2m0s&sessionLimitBps=0&globalLimitBps=0&statusAddr=:22070
@@ -100,7 +100,7 @@ relay://123.123.123.123:22067
 
 This URI can then be used in `syncthing` as one of the relay servers.
 
-See `relaysrv -help` for other options, such as rate limits, timeout intervals, etc.
+See `strelaysrv -help` for other options, such as rate limits, timeout intervals, etc.
 
 Other items available in this repo
 ----

cmd/strelaysrv/etc/linux-systemd/strelaysrv.service

@@ -3,10 +3,10 @@ Description=Syncthing relay server
 After=network.target
 
 [Service]
-User=syncthing-relaysrv
-Group=syncthing-relaysrv
-ExecStart=/usr/bin/relaysrv
-WorkingDirectory=/var/lib/syncthing-relaysrv
+User=strelaysrv
+Group=strelaysrv
+ExecStart=/usr/bin/strelaysrv
+WorkingDirectory=/var/lib/strelaysrv
 
 PrivateTmp=true
 ProtectSystem=full

cmd/strelaysrv/main.go

@@ -42,7 +42,7 @@ func init() {
 	BuildDate = time.Unix(int64(stamp), 0)
 
 	date := BuildDate.UTC().Format("2006-01-02 15:04:05 MST")
-	LongVersion = fmt.Sprintf(`relaysrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
+	LongVersion = fmt.Sprintf(`strelaysrv %s (%s %s-%s) %s@%s %s`, Version, runtime.Version(), runtime.GOOS, runtime.GOARCH, BuildUser, BuildHost, date)
 }
 
 var (
@@ -88,7 +88,7 @@ func main() {
 	flag.StringVar(&statusAddr, "status-srv", ":22070", "Listen address for status service (blank to disable)")
 	flag.StringVar(&poolAddrs, "pools", defaultPoolAddrs, "Comma separated list of relay pool addresses to join")
 	flag.StringVar(&providedBy, "provided-by", "", "An optional description about who provides the relay")
-	flag.StringVar(&extAddress, "ext-address", "", "An optional address to advertising as being available on.\n\tAllows listening on an unprivileged port with port forwarding from e.g. 443, and be connected to on port 443.")
+	flag.StringVar(&extAddress, "ext-address", "", "An optional address to advertise as being available on.\n\tAllows listening on an unprivileged port with port forwarding from e.g. 443, and be connected to on port 443.")
 
 	flag.Parse()
 
@@ -111,7 +111,7 @@ func main() {
 
 		go monitorLimits()
 	} else if err != nil && runtime.GOOS != "windows" {
-		log.Println("Assuming no connection limit, due to error retrievign rlimits:", err)
+		log.Println("Assuming no connection limit, due to error retrieving rlimits:", err)
 	}
 
 	sessionAddress = addr.IP[:]
@@ -121,7 +121,7 @@ func main() {
 	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
 		log.Println("Failed to load keypair. Generating one, this might take a while...")
-		cert, err = tlsutil.NewCertificate(certFile, keyFile, "relaysrv", 3072)
+		cert, err = tlsutil.NewCertificate(certFile, keyFile, "strelaysrv", 3072)
 		if err != nil {
 			log.Fatalln("Failed to generate X509 key pair:", err)
 		}

cmd/syncthing/gui.go

@@ -41,8 +41,7 @@ import (
 )
 
 var (
-	configInSync = true
-	startTime    = time.Now()
+	startTime = time.Now()
 )
 
 type apiService struct {
@@ -86,7 +85,7 @@ type modelIntf interface {
 	DelayScan(folder string, next time.Duration)
 	ScanFolder(folder string) error
 	ScanFolders() map[string]error
-	ScanFolderSubs(folder string, subs []string) error
+	ScanFolderSubdirs(folder string, subs []string) error
 	BringToFront(folder, file string)
 	ConnectedTo(deviceID protocol.DeviceID) bool
 	GlobalSize(folder string) (nfiles, deleted int, bytes int64)
@@ -100,12 +99,13 @@ type configIntf interface {
 	GUI() config.GUIConfiguration
 	Raw() config.Configuration
 	Options() config.OptionsConfiguration
-	Replace(cfg config.Configuration) config.CommitResponse
+	Replace(cfg config.Configuration) error
 	Subscribe(c config.Committer)
 	Folders() map[string]config.FolderConfiguration
 	Devices() map[protocol.DeviceID]config.DeviceConfiguration
 	Save() error
 	ListenAddresses() []string
+	RequiresRestart() bool
 }
 
 type connectionsIntf interface {
@@ -577,7 +577,7 @@ func (s *apiService) getDBStatus(w http.ResponseWriter, r *http.Request) {
 func folderSummary(cfg configIntf, m modelIntf, folder string) map[string]interface{} {
 	var res = make(map[string]interface{})
 
-	res["invalid"] = cfg.Folders()[folder].Invalid
+	res["invalid"] = "" // Deprecated, retains external API for now
 
 	globalFiles, globalDeleted, globalBytes := m.GlobalSize(folder)
 	res["globalFiles"], res["globalDeleted"], res["globalBytes"] = globalFiles, globalDeleted, globalBytes
@@ -690,7 +690,7 @@ func (s *apiService) postSystemConfig(w http.ResponseWriter, r *http.Request) {
 	to, err := config.ReadJSON(r.Body, myID)
 	r.Body.Close()
 	if err != nil {
-		l.Warnln("decoding posted config:", err)
+		l.Warnln("Decoding posted config:", err)
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
@@ -722,13 +722,19 @@ func (s *apiService) postSystemConfig(w http.ResponseWriter, r *http.Request) {
 
 	// Activate and save
 
-	resp := s.cfg.Replace(to)
-	configInSync = !resp.RequiresRestart
-	s.cfg.Save()
+	if err := s.cfg.Replace(to); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	if err := s.cfg.Save(); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
 }
 
 func (s *apiService) getSystemConfigInsync(w http.ResponseWriter, r *http.Request) {
-	sendJSON(w, map[string]bool{"configInSync": configInSync})
+	sendJSON(w, map[string]bool{"configInSync": !s.cfg.RequiresRestart()})
 }
 
 func (s *apiService) postSystemRestart(w http.ResponseWriter, r *http.Request) {
@@ -1071,7 +1077,7 @@ func (s *apiService) postDBScan(w http.ResponseWriter, r *http.Request) {
 		}
 
 		subs := qs["sub"]
-		err = s.model.ScanFolderSubs(folder, subs)
+		err = s.model.ScanFolderSubdirs(folder, subs)
 		if err != nil {
 			http.Error(w, err.Error(), 500)
 			return
@@ -1173,13 +1179,17 @@ type jsonFileInfo protocol.FileInfo
 
 func (f jsonFileInfo) MarshalJSON() ([]byte, error) {
 	return json.Marshal(map[string]interface{}{
-		"name":         f.Name,
-		"size":         protocol.FileInfo(f).Size(),
-		"flags":        fmt.Sprintf("%#o", f.Flags),
-		"modified":     time.Unix(f.Modified, 0),
-		"localVersion": f.LocalVersion,
-		"numBlocks":    len(f.Blocks),
-		"version":      jsonVersionVector(f.Version),
+		"name":          f.Name,
+		"type":          f.Type,
+		"size":          f.Size,
+		"permissions":   fmt.Sprintf("%#o", f.Permissions),
+		"deleted":       f.Deleted,
+		"invalid":       f.Invalid,
+		"noPermissions": f.NoPermissions,
+		"modified":      time.Unix(f.Modified, 0),
+		"localVersion":  f.LocalVersion,
+		"numBlocks":     len(f.Blocks),
+		"version":       jsonVersionVector(f.Version),
 	})
 }
 
@@ -1187,20 +1197,23 @@ type jsonDBFileInfo db.FileInfoTruncated
 
 func (f jsonDBFileInfo) MarshalJSON() ([]byte, error) {
 	return json.Marshal(map[string]interface{}{
-		"name":         f.Name,
-		"size":         db.FileInfoTruncated(f).Size(),
-		"flags":        fmt.Sprintf("%#o", f.Flags),
-		"modified":     time.Unix(f.Modified, 0),
-		"localVersion": f.LocalVersion,
-		"version":      jsonVersionVector(f.Version),
+		"name":          f.Name,
+		"type":          f.Type,
+		"size":          f.Size,
+		"permissions":   fmt.Sprintf("%#o", f.Permissions),
+		"deleted":       f.Deleted,
+		"invalid":       f.Invalid,
+		"noPermissions": f.NoPermissions,
+		"modified":      time.Unix(f.Modified, 0),
+		"localVersion":  f.LocalVersion,
 	})
 }
 
 type jsonVersionVector protocol.Vector
 
 func (v jsonVersionVector) MarshalJSON() ([]byte, error) {
-	res := make([]string, len(v))
-	for i, c := range v {
+	res := make([]string, len(v.Counters))
+	for i, c := range v.Counters {
 		res[i] = fmt.Sprintf("%v:%d", c.ID, c.Value)
 	}
 	return json.Marshal(res)

cmd/syncthing/gui_test.go

@@ -11,6 +11,7 @@ import (
 	"compress/gzip"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -613,3 +614,55 @@ func TestRandomString(t *testing.T) {
 		t.Errorf("Expected 27 random characters, got %q of length %d", res["random"], len(res["random"]))
 	}
 }
+
+func TestConfigPostOK(t *testing.T) {
+	cfg := bytes.NewBuffer([]byte(`{
+		"version": 15,
+		"folders": [
+			{"id": "foo"}
+		]
+	}`))
+
+	resp, err := testConfigPost(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.StatusCode != http.StatusOK {
+		t.Error("Expected 200 OK, not", resp.Status)
+	}
+}
+
+func TestConfigPostDupFolder(t *testing.T) {
+	cfg := bytes.NewBuffer([]byte(`{
+		"version": 15,
+		"folders": [
+			{"id": "foo"},
+			{"id": "foo"}
+		]
+	}`))
+
+	resp, err := testConfigPost(cfg)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if resp.StatusCode != http.StatusBadRequest {
+		t.Error("Expected 400 Bad Request, not", resp.Status)
+	}
+}
+
+func testConfigPost(data io.Reader) (*http.Response, error) {
+	const testAPIKey = "foobarbaz"
+	cfg := new(mockedConfig)
+	cfg.gui.APIKey = testAPIKey
+	baseURL, err := startHTTP(cfg)
+	if err != nil {
+		return nil, err
+	}
+	cli := &http.Client{
+		Timeout: time.Second,
+	}
+
+	req, _ := http.NewRequest("POST", baseURL+"/rest/system/config", data)
+	req.Header.Set("X-API-Key", testAPIKey)
+	return cli.Do(req)
+}

cmd/syncthing/locations.go

@@ -48,7 +48,7 @@ var locations = map[locationEnum]string{
 	locKeyFile:       "${config}/key.pem",
 	locHTTPSCertFile: "${config}/https-cert.pem",
 	locHTTPSKeyFile:  "${config}/https-key.pem",
-	locDatabase:      "${config}/index-v0.13.0.db",
+	locDatabase:      "${config}/index-v0.14.0.db",
 	locLogFile:       "${config}/syncthing.log", // -logfile on Windows
 	locCsrfTokens:    "${config}/csrftokens.txt",
 	locPanicLog:      "${config}/panic-${timestamp}.log",

cmd/syncthing/main.go

@@ -51,7 +51,7 @@ import (
 
 var (
 	Version           = "unknown-dev"
-	Codename          = "Copper Cockroach"
+	Codename          = "Dysprosium Dragonfly"
 	BuildStamp        = "0"
 	BuildDate         time.Time
 	BuildHost         = "unknown"
@@ -539,8 +539,9 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 	errors := logger.NewRecorder(l, logger.LevelWarn, maxSystemErrors, 0)
 	systemLog := logger.NewRecorder(l, logger.LevelDebug, maxSystemLog, initialSystemLog)
 
-	// Event subscription for the API; must start early to catch the early events.  The LocalDiskUpdated
-	// event might overwhelm the event reciever in some situations so we will not subscribe to it here.
+	// Event subscription for the API; must start early to catch the early
+	// events. The LocalChangeDetected event might overwhelm the event
+	// receiver in some situations so we will not subscribe to it here.
 	apiSub := events.NewBufferedSubscription(events.Default.Subscribe(events.AllEvents&^events.LocalChangeDetected), 1000)
 
 	if len(os.Getenv("GOMAXPROCS")) == 0 {
@@ -690,7 +691,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 			if device == myID {
 				continue
 			}
-			m.Index(device, folderCfg.ID, nil, 0, nil)
+			m.Index(device, folderCfg.ID, nil)
 		}
 		m.StartFolder(folderCfg.ID)
 	}
@@ -863,7 +864,6 @@ func loadConfig() (*config.Wrapper, error) {
 	cfg, err := config.Load(cfgFile, myID)
 
 	if err != nil {
-		l.Infoln("Error loading config file; using defaults for now")
 		myName, _ := os.Hostname()
 		newCfg := defaultConfig(myName)
 		cfg = config.Wrap(cfgFile, newCfg)

cmd/syncthing/mocked_config_test.go

@@ -31,8 +31,8 @@ func (c *mockedConfig) Options() config.OptionsConfiguration {
 	return config.OptionsConfiguration{}
 }
 
-func (c *mockedConfig) Replace(cfg config.Configuration) config.CommitResponse {
-	return config.CommitResponse{}
+func (c *mockedConfig) Replace(cfg config.Configuration) error {
+	return nil
 }
 
 func (c *mockedConfig) Subscribe(cm config.Committer) {}
@@ -48,3 +48,7 @@ func (c *mockedConfig) Devices() map[protocol.DeviceID]config.DeviceConfiguratio
 func (c *mockedConfig) Save() error {
 	return nil
 }
+
+func (c *mockedConfig) RequiresRestart() bool {
+	return false
+}

cmd/syncthing/mocked_model_test.go

@@ -85,7 +85,7 @@ func (m *mockedModel) ScanFolders() map[string]error {
 	return nil
 }
 
-func (m *mockedModel) ScanFolderSubs(folder string, subs []string) error {
+func (m *mockedModel) ScanFolderSubdirs(folder string, subs []string) error {
 	return nil
 }
 

gui/default/assets/lang/lang-ko-KR.json

@@ -3,7 +3,7 @@
     "A negative number of days doesn't make sense.": "음수로는 지정할 수 없습니다.",
     "A new major version may not be compatible with previous versions.": "새로운 메이저 버전은 이전 버전과 호환되지 않을 수 있습니다.",
     "API Key": "API 키",
-    "About": " 정보",
+    "About": "정보",
     "Actions": "동작",
     "Add": "추가",
     "Add Device": "기기 추가",

gui/default/assets/lang/lang-pt-BR.json

@@ -60,7 +60,7 @@
     "Enable Relaying": "Habilitar retransmissão",
     "Enable UPnP": "Habilitar UPnP",
     "Enter comma separated  (\"tcp://ip:port\", \"tcp://host:port\") addresses or \"dynamic\" to perform automatic discovery of the address.": "Insira endereços (\"tcp://ip:porta\", \"tcp://host:porta\") separados por vírgula ou \"dynamic\" para executar a descoberta automática do endereço.",
-    "Enter ignore patterns, one per line.": "Insira os padrões de exclusão, um por linha.",
+    "Enter ignore patterns, one per line.": "Insira os filtros, um por linha.",
     "Error": "Erro",
     "External File Versioning": "Versionamento externo de arquivo",
     "Failed Items": "Itens com falha",
@@ -89,7 +89,7 @@
     "Help": "Ajuda",
     "Home page": "Página inicial",
     "Ignore": "Ignorar",
-    "Ignore Patterns": "Padrões de exclusão",
+    "Ignore Patterns": "Filtros",
     "Ignore Permissions": "Ignorar permissões",
     "Incoming Rate Limit (KiB/s)": "Limite de velocidade de recepção (KiB/s)",
     "Incorrect configuration may damage your folder contents and render Syncthing inoperable.": "A configuração incorreta poderá causar danos aos seus dados e tornar o Syncthing inoperante.",

gui/default/assets/lang/lang-sv.json

@@ -1,6 +1,6 @@
 {
-    "A device with that ID is already added.": "En enhet med det ID är redan tillagt.",
-    "A negative number of days doesn't make sense.": "Negativt antal dagar är inte troligt.",
+    "A device with that ID is already added.": "En enhet med det ID:t är redan tillagt.",
+    "A negative number of days doesn't make sense.": "Ett negativt antal dagar är inte rimligt.",
     "A new major version may not be compatible with previous versions.": "En ny huvudversion kan eventuellt vara inkompatibel med tidigare versioner.",
     "API Key": "API-nyckel",
     "About": "Om",
@@ -16,98 +16,98 @@
     "Advanced Configuration": "Avancerad konfiguration",
     "Advanced settings": "Avancerade inställningar",
     "All Data": "All data",
-    "Allow Anonymous Usage Reporting?": "Tillåt anonym användarstatistik?",
+    "Allow Anonymous Usage Reporting?": "Tillåt anonym användarstatistiksrapportering?",
     "Alphabetic": "Alfabetisk",
-    "An external command handles the versioning. It has to remove the file from the synced folder.": "Ett externt kommando sköter versionshanteringen. Det måste ta bort filen från den synkroniserade mappen.",
-    "Anonymous Usage Reporting": "Anonym användarstatistik",
+    "An external command handles the versioning. It has to remove the file from the synced folder.": "Ett externt kommando sköter versionshanteringen. Den behöver ta bort filen från den synkroniserade mappen.",
+    "Anonymous Usage Reporting": "Anonym användarstatistik rapportering",
     "Any devices configured on an introducer device will be added to this device as well.": "Enheter konfigurerade på en introduktörsenhet kommer också att läggas till den här enheten.",
-    "Automatic upgrades": "Automatisk uppgradering",
+    "Automatic upgrades": "Automatiska uppgraderingar",
     "Be careful!": "Var aktsam!",
     "Bugs": "Buggar",
-    "CPU Utilization": "CPU-användning",
-    "Changelog": "Changelog",
+    "CPU Utilization": "CPU användning",
+    "Changelog": "Ändringslogg",
     "Clean out after": "Rensa efteråt",
     "Close": "Stäng",
     "Command": "Kommando",
-    "Comment, when used at the start of a line": "Kommentar, vid början av en rad.",
+    "Comment, when used at the start of a line": "Kommentar, vid användning i början av en rad.",
     "Compression": "Komprimering",
     "Connection Error": "Anslutningsproblem",
     "Connection Type": "Anslutningstyp",
     "Copied from elsewhere": "Kopierat utifrån",
-    "Copied from original": "Oförändrat",
-    "Copyright © 2014-2016 the following Contributors:": "Copyright © 2014-2016 följande bidragande:",
+    "Copied from original": "Kopierat från original",
+    "Copyright © 2014-2016 the following Contributors:": "Copyright © 2014-2016 följande bidragare:",
     "Copyright © 2015 the following Contributors:": "Copyright © 2015 följande medverkande:",
     "Danger!": "Fara!",
-    "Delete": "Radera",
+    "Delete": "Ta bort",
     "Deleted": "Borttaget",
     "Device \"{%name%}\" ({%device%} at {%address%}) wants to connect. Add new device?": "Enhet \"{{name}}\" ({{device}} på {{address}}) vill ansluta. Lägg till ny enhet?",
-    "Device ID": "Enhets-ID",
+    "Device ID": "Enhets ID",
     "Device Identification": "Enhetsidentifikation",
     "Device Name": "Enhetsnamn",
     "Device {%device%} ({%address%}) wants to connect. Add new device?": "Enheten {{device}} ({{address}}) vill ansluta. Lägg till ny enhet?",
     "Devices": "Enheter",
-    "Disconnected": "Ej ansluten",
-    "Discovery": "Uppslagning",
+    "Disconnected": "Frånkopplad",
+    "Discovery": "Upptäckt",
     "Documentation": "Dokumentation",
-    "Download Rate": "Nedladdningshastighet",
-    "Downloaded": "Nerladdat",
-    "Downloading": "Laddar ner",
+    "Download Rate": "Hämtningshastighet",
+    "Downloaded": "Hämtat",
+    "Downloading": "Hämtar",
     "Edit": "Redigera",
     "Edit Device": "Redigera enhet",
     "Edit Folder": "Redigera katalog",
     "Editing": "Redigerar",
     "Enable NAT traversal": "Aktivera NAT traversering",
     "Enable Relaying": "Aktivera reläa",
-    "Enable UPnP": "Använd UPnP",
+    "Enable UPnP": "Aktivera UPnP",
     "Enter comma separated  (\"tcp://ip:port\", \"tcp://host:port\") addresses or \"dynamic\" to perform automatic discovery of the address.": "Ange kommaseparerade (\"tcp://ip:port\", \"tcp://host:port\")-adresser eller ordet \"dynamic\" för att använda automatisk uppslagning.",
-    "Enter ignore patterns, one per line.": "Ange filmönster, ett per rad.",
+    "Enter ignore patterns, one per line.": "Ange ignorera mönster, en per rad.",
     "Error": "Fel",
-    "External File Versioning": "Extern versionshantering",
-    "Failed Items": "Misslyckade filer",
-    "File Pull Order": "Hämtningsprioritering av filer",
-    "File Versioning": "Versionshantering",
+    "External File Versioning": "Extern filversionshantering",
+    "Failed Items": "Misslyckade objekt",
+    "File Pull Order": "Filhämtningsprioritering",
+    "File Versioning": "Filversionshantering",
     "File permission bits are ignored when looking for changes. Use on FAT file systems.": "Filrättigheter ignoreras vid sökning efter förändringar. Används på FAT-filsystem.",
     "Files are moved to .stversions folder when replaced or deleted by Syncthing.": "Filer flyttas till katalogen .stversions om de ersätts eller raderas av Syncthing.",
     "Files are moved to date stamped versions in a .stversions folder when replaced or deleted by Syncthing.": "Filer flyttas till datummärkta versioner i en .stversions-mapp när de ersatts eller raderats av Syncthing.",
     "Files are protected from changes made on other devices, but changes made on this device will be sent to the rest of the cluster.": "Filer skyddas från ändringar gjorda på andra enheter, men ändringar som görs på den här noden skickas till de andra klustermedlemmarna.",
     "Folder": "Katalog",
     "Folder ID": "Katalog-ID",
-    "Folder Label": "Katalog etikett",
-    "Folder Master": "Huvudlagring",
-    "Folder Path": "Sökväg",
+    "Folder Label": "Katalog-etikett",
+    "Folder Master": "Huvudkatalog",
+    "Folder Path": "Katalog-sökväg",
     "Folder Type": "Katalogtyp",
     "Folders": "Kataloger",
-    "GUI": "GUI",
-    "GUI Authentication Password": "GUI-lösenord",
-    "GUI Authentication User": "GUI-användare",
-    "GUI Listen Addresses": "GUI-adress",
-    "Generate": "Skapa",
-    "Global Discovery": "Global uppslagning",
-    "Global Discovery Server": "Global uppslagningsserver",
-    "Global Discovery Servers": "Globala uppslagningsservrar",
+    "GUI": "Grafiskt gränssnitt",
+    "GUI Authentication Password": "Grafiska gränssnittets lösenord",
+    "GUI Authentication User": "Grafiska gränssnittets användare",
+    "GUI Listen Addresses": "Grafiska gränssnittets avlyssningsadresser",
+    "Generate": "Generera",
+    "Global Discovery": "Global upptäckt",
+    "Global Discovery Server": "Global upptäcktsserver",
+    "Global Discovery Servers": "Globala upptäcktsservrar",
     "Global State": "Global status",
     "Help": "Hjälp",
     "Home page": "Hemsida",
     "Ignore": "Ignorera",
-    "Ignore Patterns": "Ignorerade filmönster",
-    "Ignore Permissions": "Ignorera filrättigheter",
-    "Incoming Rate Limit (KiB/s)": "Max nedladdningshastighet (KiB/s)",
+    "Ignore Patterns": "Ignorera mönster",
+    "Ignore Permissions": "Ignorera rättigheter",
+    "Incoming Rate Limit (KiB/s)": "Nedladdningshastighetsgräns (KiB/s)",
     "Incorrect configuration may damage your folder contents and render Syncthing inoperable.": "Inkorrekt konfiguration kan skada innehållet i katalogen and få Syncthing att sluta fungera.",
     "Introducer": "introduktör",
     "Inversion of the given condition (i.e. do not exclude)": "Vänder på villkoret, d.v.s. exkluderar inte.",
     "Keep Versions": "Behåll versioner",
-    "Largest First": "Störst först",
-    "Last File Received": "Senast Mottagna Fil",
-    "Last Scan": "Senast skanning",
-    "Last seen": "Senast online",
+    "Largest First": "Största först",
+    "Last File Received": "Senaste fil mottagen",
+    "Last Scan": "Senaste skanning",
+    "Last seen": "Senast sedd",
     "Later": "Senare",
     "Listeners": "Lyssnare",
     "Local Discovery": "Lokal uppslagning",
     "Local State": "Lokal status",
-    "Local State (Total)": "Lokal status (Total)",
-    "Major Upgrade": "Stor uppgradering",
+    "Local State (Total)": "Lokal status (totalt)",
+    "Major Upgrade": "Större uppgradering",
     "Master": "Huvud",
-    "Maximum Age": "Högsta åldersgräns",
+    "Maximum Age": "Maximum ålder",
     "Metadata Only": "Endast metadata",
     "Minimum Free Disk Space": "Minimum ledigt diskutrymme",
     "Move to top of queue": "Flytta till överst i kön",
@@ -117,7 +117,7 @@
     "New Folder": "Ny katalog",
     "Newest First": "Nyast först",
     "No": "Nej",
-    "No File Versioning": "Ingen versionshantering",
+    "No File Versioning": "Ingen filversionshantering",
     "Normal": "Normal",
     "Notice": "Observera",
     "OK": "OK",
@@ -125,40 +125,40 @@
     "Oldest First": "Äldst först",
     "Optional descriptive label for the folder. Can be different on each device.": "Valfri beskrivande etikett för katalogen. Kan vara olika på varje enhet.",
     "Options": "Alternativ",
-    "Out of Sync": "Osynkad",
-    "Out of Sync Items": "Osynkade poster",
+    "Out of Sync": "Osynkroniserad",
+    "Out of Sync Items": "Osynkroniserade objekt",
     "Outgoing Rate Limit (KiB/s)": "Max uppladdningshastighet (KiB/s)",
-    "Override Changes": "Skriv över ändringar",
+    "Override Changes": "Åsidosätt förändringar",
     "Path to the folder on the local computer. Will be created if it does not exist. The tilde character (~) can be used as a shortcut for": "Sökväg till katalogen på din dator. Kommer att skapas om det inte finns. Tecknet tilde (~) kan användas som en genväg för",
     "Path where versions should be stored (leave empty for the default .stversions folder in the folder).": "Sökväg där versioner sparas (lämna tomt för att använda .stversions i den ordinarie katalogen).",
     "Pause": "Paus",
     "Paused": "Pausad",
     "Please consult the release notes before performing a major upgrade.": "Läs igenom versionsnyheterna innan den stora uppgraderingen.",
-    "Please set a GUI Authentication User and Password in the Settings dialog.": "Ställ in ett grafiskt användarautentisering och lösenord i dialogrutan Inställningar.",
+    "Please set a GUI Authentication User and Password in the Settings dialog.": "Ställ in ett grafiska gränssnittets användarautentisering och lösenord i inställningsdialogrutan.",
     "Please wait": "Var god vänta",
     "Preview": "Förhandsgranska",
     "Preview Usage Report": "Förhandsgranska statistik",
-    "Quick guide to supported patterns": "Snabb guide till filmönster som stöds",
-    "RAM Utilization": "Minnesanvändning",
+    "Quick guide to supported patterns": "Snabb handledning till mönster som stöds",
+    "RAM Utilization": "RAM användning",
     "Random": "Slumpmässig",
     "Relay Servers": "Reläservrar",
     "Relayed via": "Vidarbefordras via",
-    "Relays": "Vidarbefordringar",
-    "Release Notes": "versionsnyheter",
+    "Relays": "Reläservrar",
+    "Release Notes": "Versionsanteckningar",
     "Remote Devices": "Fjärrenheter",
     "Remove": "Ta bort",
     "Required identifier for the folder. Must be the same on all cluster devices.": "Krävs identifierare för katalogen. Måste vara densamma på alla kluster enheter.",
-    "Rescan": "Uppdatera",
-    "Rescan All": "Uppdatera alla",
-    "Rescan Interval": "Uppdateringsintervall",
+    "Rescan": "Skanna om",
+    "Rescan All": "Skanna om alla",
+    "Rescan Interval": "Omskanningsintervall",
     "Restart": "Starta om",
     "Restart Needed": "Omstart behövs",
     "Restarting": "Startar om",
     "Resume": "Återuppta",
-    "Reused": "Återanvänt",
+    "Reused": "Återanvänd",
     "Save": "Spara",
-    "Scan Time Remaining": "Granska återstående tid",
-    "Scanning": "Uppdaterar",
+    "Scan Time Remaining": "Återstående skanningstid",
+    "Scanning": "Skannar",
     "Select the devices to share this folder with.": "Ange enheterna att dela den här katalogen med.",
     "Select the folders to share with this device.": "Välj kataloger att dela med den här enheten.",
     "Settings": "Inställningar",
@@ -166,85 +166,85 @@
     "Share Folder": "Dela katalog",
     "Share Folders With Device": "Dela kataloger med enhet",
     "Share With Devices": "Dela med enheter",
-    "Share this folder?": "Dela den här katalogen?",
+    "Share this folder?": "Dela denna katalog?",
     "Shared With": "Delad med",
     "Short identifier for the folder. Must be the same on all cluster devices.": "Kort identifieringssträng för katalogen. Måste vara samma på alla enheter i klustret.",
     "Show ID": "Visa ID",
     "Show QR": "Visa QR",
-    "Shown instead of Device ID in the cluster status. Will be advertised to other devices as an optional default name.": "Visas i stället för enhets-ID. Skickas till andra enheter som namn på denna enhet.",
-    "Shown instead of Device ID in the cluster status. Will be updated to the name the device advertises if left empty.": "Visas i stället för enhets-ID. Sätts till namnet på den andra enheten vid första anslutning om det lämnas tomt.",
+    "Shown instead of Device ID in the cluster status. Will be advertised to other devices as an optional default name.": "Visas i stället för enhetens ID. Skickas till andra enheter som namn på denna enhet.",
+    "Shown instead of Device ID in the cluster status. Will be updated to the name the device advertises if left empty.": "Visas i stället för enhetens ID. Sätts till namnet på den andra enheten vid första anslutning om det lämnas tomt.",
     "Shutdown": "Stäng av",
     "Shutdown Complete": "Avstängning klar",
-    "Simple File Versioning": "Enkel versionshantering",
+    "Simple File Versioning": "Enkel filversionshantering",
     "Single level wildcard (matches within a directory only)": "Jokertecken som representerar noll eller fler godtyckliga tecken i ett filnamn.",
     "Smallest First": "Minst först",
     "Source Code": "Källkod",
-    "Staggered File Versioning": "Versionshantering i intervall",
-    "Start Browser": "Starta browser",
+    "Staggered File Versioning": "Filversionshantering i intervall",
+    "Start Browser": "Starta webbläsare",
     "Statistics": "Statistik",
     "Stopped": "Stoppad",
     "Support": "Support",
     "Sync Protocol Listen Addresses": "Address för inkommande anslutningar",
     "Syncing": "Synkroniserar",
-    "Syncthing has been shut down.": "Syncthing har stängts ner.",
+    "Syncthing has been shut down.": "Syncthing har stängts.",
     "Syncthing includes the following software or portions thereof:": "Syncthing innehåller följande mjukvarupaket eller delar av dem:",
     "Syncthing is restarting.": "Syncthing startar om.",
     "Syncthing is upgrading.": "Syncthing uppgraderas.",
     "Syncthing seems to be down, or there is a problem with your Internet connection. Retrying…": "Syncthing verkar avstängd, eller finns det problem med din Internetanslutning. Försöker igen...",
-    "Syncthing seems to be experiencing a problem processing your request. Please refresh the page or restart Syncthing if the problem persists.": "Syncthing verkar ha drabbats av ett problem. Uppdatera sidan eller starta om Syncthing om problemet kvarstår.",
+    "Syncthing seems to be experiencing a problem processing your request. Please refresh the page or restart Syncthing if the problem persists.": "Syncthing verkar ha drabbats av ett problem med behandlingen av din begäran. Uppdatera sidan eller starta om Syncthing om problemet kvarstår.",
     "The Syncthing admin interface is configured to allow remote access without a password.": "Syncthing administratör gränssnittet är konfigurerat för att tillåta fjärrtillträde utan ett lösenord.",
     "The aggregated statistics are publicly available at the URL below.": "Den aggregerade statistiken är offentligt tillgängliga på webbadressen nedan.",
     "The aggregated statistics are publicly available at {%url%}.": "Sammanställd statistik finns publikt tillgänglig på {{url}}.",
     "The configuration has been saved but not activated. Syncthing must restart to activate the new configuration.": "Konfigurationen har sparats men inte aktiverats. Syncthing måste startas om för att aktivera den nya konfigurationen.",
-    "The device ID cannot be blank.": "Enhets-ID kan inte vara tomt.",
-    "The device ID to enter here can be found in the \"Actions > Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).": "Enhets-ID som behövs här kan du hitta i \"Redigera > Visa ID\"-dialogen på den andra enheten. Mellanrum och bindestreck är valfria (ignoreras).",
-    "The device ID to enter here can be found in the \"Edit > Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).": "Enhets-ID som behövs här kan du hitta i \"Redigera > Visa ID\"-dialogen på den andra enheten. Mellanrum och bindestreck är valfria (ignoreras).",
+    "The device ID cannot be blank.": "Enhetens ID kan inte vara tomt.",
+    "The device ID to enter here can be found in the \"Actions > Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).": "Enhetens ID som behövs här kan du hitta i \"Redigera > Visa ID\"-dialogen på den andra enheten. Mellanrum och bindestreck är valfria (ignoreras).",
+    "The device ID to enter here can be found in the \"Edit > Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).": "Enhetens ID som behövs här kan du hitta i \"Redigera > Visa ID\"-dialogen på den andra enheten. Mellanrum och bindestreck är valfria (ignoreras).",
     "The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.": "Den krypterade användarstatistiken skickas dagligen. Den används för att spåra vanliga plattformar, katalogstorlekar och versioner. Om datan som rapporteras ändras så kommer du att bli tillfrågad igen.",
-    "The entered device ID does not look valid. It should be a 52 or 56 character string consisting of letters and numbers, with spaces and dashes being optional.": "Det inmatade enhets-ID:t verkar inte korrekt. Det ska vara en 52 eller 56 teckens sträng bestående av siffror och bokstäver, eventuellt med mellanrum och bindestreck.",
+    "The entered device ID does not look valid. It should be a 52 or 56 character string consisting of letters and numbers, with spaces and dashes being optional.": "Det inmatade enhetens ID verkar inte korrekt. Det ska vara en 52 eller 56 teckens sträng bestående av siffror och bokstäver, eventuellt med mellanrum och bindestreck.",
     "The first command line parameter is the folder path and the second parameter is the relative path in the folder.": "Den första kommandoparametern är sökvägen till mappen och den andra parametern är den relativa sökvägen i mappen.",
-    "The folder ID cannot be blank.": "Ange ett enhets-ID.",
-    "The folder ID must be a short identifier (64 characters or less) consisting of letters, numbers and the dot (.), dash (-) and underscode (_) characters only.": "Katalog-ID:t måste vara en kort sträng (64 tecken eller mindre), bestående av endast bokstäver, siffror, punkt (.), bindestreck (-) och understreck (_).",
-    "The folder ID must be unique.": "Katalog-ID:t måste vara unikt.",
-    "The folder path cannot be blank.": "Ange en sökväg.",
+    "The folder ID cannot be blank.": "Katalogens ID får inte vara tomt.",
+    "The folder ID must be a short identifier (64 characters or less) consisting of letters, numbers and the dot (.), dash (-) and underscode (_) characters only.": "Katalogens ID måste vara en kort sträng (64 tecken eller mindre), bestående av endast bokstäver, siffror, punkt (.), bindestreck (-) och understreck (_).",
+    "The folder ID must be unique.": "Katalogens ID måste vara unikt.",
+    "The folder path cannot be blank.": "Katalogsökvägen får inte vara tom.",
     "The following intervals are used: for the first hour a version is kept every 30 seconds, for the first day a version is kept every hour, for the first 30 days a version is kept every day, until the maximum age a version is kept every week.": "De följande intervallen används: varje 30 sekunder under den första timmen; varje timme under den första dagen; varje dag för de första 30 dagarna; varje vecka tills den maximala åldersgränsen uppnås.",
-    "The following items could not be synchronized.": "Följande filer kunde inte synkroniseras.",
+    "The following items could not be synchronized.": "Följande objekt kunde inte synkroniseras.",
     "The maximum age must be a number and cannot be blank.": "Åldersgränsen måste vara ett tal och kan inte lämnas tomt.",
     "The maximum time to keep a version (in days, set to 0 to keep versions forever).": "Den längsta tiden att behålla en version (i dagar, sätt till 0 för att behålla versioner för evigt).",
     "The minimum free disk space percentage must be a non-negative number between 0 and 100 (inclusive).": "Minimum ledigt diskutrymme i procent måste vara en icke negativ siffra mellan 0 och 100 (inklusive).",
     "The number of days must be a number and cannot be blank.": "Antalet dagar måste vara en siffra och får inte vara tomt.",
-    "The number of days to keep files in the trash can. Zero means forever.": "Antal dagar som filer ligger kvar i papperskorgen. Noll betyder för alltid.",
+    "The number of days to keep files in the trash can. Zero means forever.": "Antalet dagar som filer ligger kvar i papperskorgen. Noll betyder för alltid.",
     "The number of old versions to keep, per file.": "Antalet gamla versioner som ska behållas, per fil.",
     "The number of versions must be a number and cannot be blank.": "Antalet versioner måste vara ett nummer och kan inte lämnas tomt.",
-    "The path cannot be blank.": "Ange en sökväg",
+    "The path cannot be blank.": "Sökvägen kan inte vara tom.",
     "The rate limit must be a non-negative number (0: no limit)": "Frekvensgränsen måste vara ett icke-negativt tal (0: ingen gräns)",
     "The rescan interval must be a non-negative number of seconds.": "Förnyelseintervallet måste vara ett positivt antal sekunder",
     "They are retried automatically and will be synced when the error is resolved.": "De omprövas automatiskt och kommer att synkroniseras när felet är löst.",
-    "This Device": "Enheten",
+    "This Device": "Denna enhet",
     "This can easily give hackers access to read and change any files on your computer.": "Detta kan lätt ge hackare tillgång till att läsa och ändra några filer på datorn.",
     "This is a major version upgrade.": "Det här är en stor uppgradering.",
-    "Trash Can File Versioning": "Versionshantering på filer i papperskorgen",
-    "Unknown": "Okänt",
+    "Trash Can File Versioning": "Papperskorgs filversionshantering",
+    "Unknown": "Okänd",
     "Unshared": "Inte delad",
     "Unused": "Oanvänd",
-    "Up to Date": "Helt uppdaterad",
+    "Up to Date": "Uppdaterad",
     "Updated": "Uppdaterad",
     "Upgrade": "Uppgradering",
     "Upgrade To {%version%}": "Uppgradera till {{version}}",
     "Upgrading": "Uppgraderar",
     "Upload Rate": "Uppladdningshastighet",
     "Uptime": "Tid sedan start",
-    "Use HTTPS for GUI": "Använd HTTPS för GUI",
+    "Use HTTPS for GUI": "Använd HTTPS för grafiska gränssnittet",
     "Version": "Version",
-    "Versions Path": "Katalog för versioner",
+    "Versions Path": "Sökväg för versioner",
     "Versions are automatically deleted if they are older than the maximum age or exceed the number of files allowed in an interval.": "Versioner tas bort automatiskt när de är äldre än den maximala åldersgränsen eller överstiger frekvensen i sitt interval.",
     "Warning, this path is a subdirectory of an existing folder \"{%otherFolder%}\".": "Varning, denna sökväg är en underkatalog till en befintlig katalog \"{{otherFolder}}\".",
     "When adding a new device, keep in mind that this device must be added on the other side too.": "När du lägger till en ny enhet, kom ihåg att den här enheten måste läggas till på den andra enheten också.",
-    "When adding a new folder, keep in mind that the Folder ID is used to tie folders together between devices. They are case sensitive and must match exactly between all devices.": "När du lägger till ny katalog, tänk på att katalog-ID:t knyter ihop katalogen mellan olika noder. De måste vara exakt desamma mellan noder och stora eller små bokstäver har betydelse.",
+    "When adding a new folder, keep in mind that the Folder ID is used to tie folders together between devices. They are case sensitive and must match exactly between all devices.": "När du lägger till ny katalog, tänk på att katalogens ID knyter ihop katalogen mellan olika noder. De måste vara exakt desamma mellan noder och stora eller små bokstäver har betydelse.",
     "Yes": "Ja",
     "You must keep at least one version.": "Du måste behålla åtminstone en version.",
     "days": "dagar",
     "full documentation": "fullständig dokumentation",
-    "items": "poster",
+    "items": "objekt",
     "{%device%} wants to share folder \"{%folder%}\".": "{{device}} vill dela katalogen \"{{folder}}\".",
     "{%device%} wants to share folder \"{%folderLabel%}\" ({%folder%}).": "{{device}} vill dela katalogen \"{{folderLabel}}\" ({{folder}}).",
     "{%device%} wants to share folder \"{%folderlabel%}\" ({%folder%}).": "{{device}} vill dela katalogen \"{{folderlabel}}\" ({{folder}})."

gui/default/syncthing/core/syncthingController.js

@@ -1318,7 +1318,7 @@ angular.module('syncthing.core')
             $scope.editingExisting = false;
             $scope.folderEditor.$setPristine();
             $http.get(urlbase + '/svc/random/string?length=10').success(function (data) {
-                $scope.currentFolder.id = data.random.substr(0, 5) + '-' + data.random.substr(5, 5);
+                $scope.currentFolder.id = (data.random.substr(0, 5) + '-' + data.random.substr(5, 5)).toLowerCase();
                 $('#editFolder').modal();
             });
         };
@@ -1453,7 +1453,7 @@ angular.module('syncthing.core')
                 }
             }
 
-            folders.sort();
+            folders.sort(folderCompare);
             return folders;
         };
 

gui/default/syncthing/usagereport/usageReportPreviewModalView.html

@@ -3,9 +3,8 @@
     <p translate>
       The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.
     </p>
-    <p translate translate-value-url="<a href=&quot;https://data.syncthing.net&quot; target=&quot;_blank&quot;>https://data.syncthing.net</a>">
-      The aggregated statistics are publicly available at {%url%}.
-    </p>
+    <p translate>The aggregated statistics are publicly available at the URL below.</p>
+    <p><a href="https://data.syncthing.net/" target="_blank">https://data.syncthing.net/</a></p>
     <form>
       <textarea class="form-control" rows="20">{{reportData | json}}</textarea>
     </form>

lib/config/commit_test.go

@@ -11,12 +11,18 @@ import (
 	"testing"
 )
 
-type requiresRestart struct{}
+type requiresRestart struct {
+	committed chan struct{}
+}
 
 func (requiresRestart) VerifyConfiguration(_, _ Configuration) error {
 	return nil
 }
-func (requiresRestart) CommitConfiguration(_, _ Configuration) bool {
+func (c requiresRestart) CommitConfiguration(_, _ Configuration) bool {
+	select {
+	case c.committed <- struct{}{}:
+	default:
+	}
 	return false
 }
 func (requiresRestart) String() string {
@@ -28,7 +34,7 @@ type validationError struct{}
 func (validationError) VerifyConfiguration(_, _ Configuration) error {
 	return errors.New("some error")
 }
-func (validationError) CommitConfiguration(_, _ Configuration) bool {
+func (c validationError) CommitConfiguration(_, _ Configuration) bool {
 	return true
 }
 func (validationError) String() string {
@@ -44,11 +50,11 @@ func TestReplaceCommit(t *testing.T) {
 	// Replace config. We should get back a clean response and the config
 	// should change.
 
-	resp := w.Replace(Configuration{Version: 1})
-	if resp.ValidationError != nil {
-		t.Fatal("Should not have a validation error")
+	err := w.Replace(Configuration{Version: 1})
+	if err != nil {
+		t.Fatal("Should not have a validation error:", err)
 	}
-	if resp.RequiresRestart {
+	if w.RequiresRestart() {
 		t.Fatal("Should not require restart")
 	}
 	if w.Raw().Version != 1 {
@@ -58,13 +64,16 @@ func TestReplaceCommit(t *testing.T) {
 	// Now with a subscriber requiring restart. We should get a clean response
 	// but with the restart flag set, and the config should change.
 
-	w.Subscribe(requiresRestart{})
+	sub0 := requiresRestart{committed: make(chan struct{}, 1)}
+	w.Subscribe(sub0)
 
-	resp = w.Replace(Configuration{Version: 2})
-	if resp.ValidationError != nil {
-		t.Fatal("Should not have a validation error")
+	err = w.Replace(Configuration{Version: 2})
+	if err != nil {
+		t.Fatal("Should not have a validation error:", err)
 	}
-	if !resp.RequiresRestart {
+
+	<-sub0.committed
+	if !w.RequiresRestart() {
 		t.Fatal("Should require restart")
 	}
 	if w.Raw().Version != 2 {
@@ -76,12 +85,12 @@ func TestReplaceCommit(t *testing.T) {
 
 	w.Subscribe(validationError{})
 
-	resp = w.Replace(Configuration{Version: 3})
-	if resp.ValidationError == nil {
+	err = w.Replace(Configuration{Version: 3})
+	if err == nil {
 		t.Fatal("Should have a validation error")
 	}
-	if resp.RequiresRestart {
-		t.Fatal("Should not require restart")
+	if !w.RequiresRestart() {
+		t.Fatal("Should still require restart")
 	}
 	if w.Raw().Version != 2 {
 		t.Fatal("Config should not have changed")

lib/config/config.go

@@ -10,6 +10,7 @@ package config
 import (
 	"encoding/json"
 	"encoding/xml"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"net/url"
@@ -81,11 +82,15 @@ func ReadXML(r io.Reader, myID protocol.DeviceID) (Configuration, error) {
 	util.SetDefaults(&cfg.Options)
 	util.SetDefaults(&cfg.GUI)
 
-	err := xml.NewDecoder(r).Decode(&cfg)
+	if err := xml.NewDecoder(r).Decode(&cfg); err != nil {
+		return Configuration{}, err
+	}
 	cfg.OriginalVersion = cfg.Version
 
-	cfg.prepare(myID)
-	return cfg, err
+	if err := cfg.prepare(myID); err != nil {
+		return Configuration{}, err
+	}
+	return cfg, nil
 }
 
 func ReadJSON(r io.Reader, myID protocol.DeviceID) (Configuration, error) {
@@ -97,14 +102,18 @@ func ReadJSON(r io.Reader, myID protocol.DeviceID) (Configuration, error) {
 
 	bs, err := ioutil.ReadAll(r)
 	if err != nil {
-		return cfg, err
+		return Configuration{}, err
 	}
 
-	err = json.Unmarshal(bs, &cfg)
+	if err := json.Unmarshal(bs, &cfg); err != nil {
+		return Configuration{}, err
+	}
 	cfg.OriginalVersion = cfg.Version
 
-	cfg.prepare(myID)
-	return cfg, err
+	if err := cfg.prepare(myID); err != nil {
+		return Configuration{}, err
+	}
+	return cfg, nil
 }
 
 type Configuration struct {
@@ -154,7 +163,7 @@ func (cfg *Configuration) WriteXML(w io.Writer) error {
 	return err
 }
 
-func (cfg *Configuration) prepare(myID protocol.DeviceID) {
+func (cfg *Configuration) prepare(myID protocol.DeviceID) error {
 	util.FillNilSlices(&cfg.Options)
 
 	// Initialize any empty slices
@@ -168,19 +177,19 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
 		cfg.Options.AlwaysLocalNets = []string{}
 	}
 
-	// Check for missing, bad or duplicate folder ID:s
-	var seenFolders = map[string]*FolderConfiguration{}
+	// Prepare folders and check for duplicates. Duplicates are bad and
+	// dangerous, can't currently be resolved in the GUI, and shouldn't
+	// happen when configured by the GUI. We return with an error in that
+	// situation.
+	seenFolders := make(map[string]struct{})
 	for i := range cfg.Folders {
 		folder := &cfg.Folders[i]
 		folder.prepare()
 
-		if seen, ok := seenFolders[folder.ID]; ok {
-			l.Warnf("Multiple folders with ID %q; disabling", folder.ID)
-			seen.Invalid = "duplicate folder ID"
-			folder.Invalid = "duplicate folder ID"
-		} else {
-			seenFolders[folder.ID] = folder
+		if _, ok := seenFolders[folder.ID]; ok {
+			return fmt.Errorf("duplicate folder ID %q in configuration", folder.ID)
 		}
+		seenFolders[folder.ID] = struct{}{}
 	}
 
 	cfg.Options.ListenAddresses = util.UniqueStrings(cfg.Options.ListenAddresses)
@@ -257,6 +266,8 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
 	if cfg.GUI.APIKey == "" {
 		cfg.GUI.APIKey = rand.String(32)
 	}
+
+	return nil
 }
 
 func convertV14V15(cfg *Configuration) {

lib/config/config_test.go

@@ -549,6 +549,9 @@ func TestPullOrder(t *testing.T) {
 	t.Logf("%s", buf.Bytes())
 
 	cfg, err = ReadXML(buf, device1)
+	if err != nil {
+		t.Fatal(err)
+	}
 	wrapper = Wrap("testdata/pullorder.xml", cfg)
 	folders = wrapper.Folders()
 
@@ -595,22 +598,14 @@ func TestGUIConfigURL(t *testing.T) {
 	}
 }
 
-func TestRemoveDuplicateDevicesFolders(t *testing.T) {
-	wrapper, err := Load("testdata/duplicates.xml", device1)
+func TestDuplicateDevices(t *testing.T) {
+	// Duplicate devices should be removed
+
+	wrapper, err := Load("testdata/dupdevices.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	// All folders are loaded, but the duplicate ones are disabled.
-	if l := len(wrapper.Raw().Folders); l != 3 {
-		t.Errorf("Incorrect number of folders, %d != 3", l)
-	}
-	for i, f := range wrapper.Raw().Folders {
-		if f.ID == "f1" && f.Invalid == "" {
-			t.Errorf("Folder %d (%q) is not set invalid", i, f.ID)
-		}
-	}
-
 	if l := len(wrapper.Raw().Devices); l != 3 {
 		t.Errorf("Incorrect number of devices, %d != 3", l)
 	}
@@ -621,6 +616,30 @@ func TestRemoveDuplicateDevicesFolders(t *testing.T) {
 	}
 }
 
+func TestDuplicateFolders(t *testing.T) {
+	// Duplicate folders are a loading error
+
+	_, err := Load("testdata/dupfolders.xml", device1)
+	if err == nil || !strings.HasPrefix(err.Error(), "duplicate folder ID") {
+		t.Fatal(`Expected error to mention "duplicate folder ID":`, err)
+	}
+}
+
+func TestEmptyFolderPaths(t *testing.T) {
+	// Empty folder paths are allowed at the loading stage, and should not
+	// get messed up by the prepare steps (e.g., become the current dir or
+	// get a slash added so that it becomes the root directory or similar).
+
+	wrapper, err := Load("testdata/nopath.xml", device1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	folder := wrapper.Folders()["f1"]
+	if folder.Path() != "" {
+		t.Errorf("Expected %q to be empty", folder.Path())
+	}
+}
+
 func TestV14ListenAddressesMigration(t *testing.T) {
 	tcs := [][3][]string{
 

lib/config/folderconfiguration.go

@@ -39,7 +39,6 @@ type FolderConfiguration struct {
 	DisableSparseFiles    bool                        `xml:"disableSparseFiles" json:"disableSparseFiles"`
 	DisableTempIndexes    bool                        `xml:"disableTempIndexes" json:"disableTempIndexes"`
 
-	Invalid    string `xml:"-" json:"invalid"` // Set at runtime when there is an error, not saved
 	cachedPath string
 
 	DeprecatedReadOnly bool `xml:"ro,attr,omitempty" json:"-"`
@@ -70,7 +69,7 @@ func (f FolderConfiguration) Path() string {
 	// This is intentionally not a pointer method, because things like
 	// cfg.Folders["default"].Path() should be valid.
 
-	if f.cachedPath == "" {
+	if f.cachedPath == "" && f.RawPath != "" {
 		l.Infoln("bug: uncached path call (should only happen in tests)")
 		return f.cleanedPath()
 	}
@@ -108,31 +107,24 @@ func (f *FolderConfiguration) DeviceIDs() []protocol.DeviceID {
 }
 
 func (f *FolderConfiguration) prepare() {
-	if len(f.RawPath) == 0 {
-		f.Invalid = "no directory configured"
-		return
-	}
+	if f.RawPath != "" {
+		// The reason it's done like this:
+		// C:          ->  C:\            ->  C:\        (issue that this is trying to fix)
+		// C:\somedir  ->  C:\somedir\    ->  C:\somedir
+		// C:\somedir\ ->  C:\somedir\\   ->  C:\somedir
+		// This way in the tests, we get away without OS specific separators
+		// in the test configs.
+		f.RawPath = filepath.Dir(f.RawPath + string(filepath.Separator))
 
-	// The reason it's done like this:
-	// C:          ->  C:\            ->  C:\        (issue that this is trying to fix)
-	// C:\somedir  ->  C:\somedir\    ->  C:\somedir
-	// C:\somedir\ ->  C:\somedir\\   ->  C:\somedir
-	// This way in the tests, we get away without OS specific separators
-	// in the test configs.
-	f.RawPath = filepath.Dir(f.RawPath + string(filepath.Separator))
-
-	// If we're not on Windows, we want the path to end with a slash to
-	// penetrate symlinks. On Windows, paths must not end with a slash.
-	if runtime.GOOS != "windows" && f.RawPath[len(f.RawPath)-1] != filepath.Separator {
-		f.RawPath = f.RawPath + string(filepath.Separator)
+		// If we're not on Windows, we want the path to end with a slash to
+		// penetrate symlinks. On Windows, paths must not end with a slash.
+		if runtime.GOOS != "windows" && f.RawPath[len(f.RawPath)-1] != filepath.Separator {
+			f.RawPath = f.RawPath + string(filepath.Separator)
+		}
 	}
 
 	f.cachedPath = f.cleanedPath()
 
-	if f.ID == "" {
-		f.ID = "default"
-	}
-
 	if f.RescanIntervalS > MaxRescanIntervalS {
 		f.RescanIntervalS = MaxRescanIntervalS
 	} else if f.RescanIntervalS < 0 {
@@ -145,6 +137,10 @@ func (f *FolderConfiguration) prepare() {
 }
 
 func (f *FolderConfiguration) cleanedPath() string {
+	if f.RawPath == "" {
+		return ""
+	}
+
 	cleaned := f.RawPath
 
 	// Attempt tilde expansion; leave unchanged in case of error

lib/config/testdata/dupdevices.xml

@@ -15,15 +15,6 @@
         <!-- duplicate, will be removed -->
         <address>192.0.2.5</address>
     </device>
-    <folder id="f1" directory="testdata/">
-        <!-- duplicate, will be disabled -->
-        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
-        <device id="GYRZZQBIRNPV4T7TC52WEQYJ3TFDQW6MWDFLMU4SSSU6EMFBK2VA"></device>
-    </folder>
-    <folder id="f1" directory="testdata/">
-        <!-- duplicate, will be disabled -->
-        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
-    </folder>
     <folder id="f2" directory="testdata/">
         <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
         <device id="GYRZZQBIRNPV4T7TC52WEQYJ3TFDQW6MWDFLMU4SSSU6EMFBK2VA"></device>

lib/config/testdata/dupfolders.xml

@@ -0,0 +1,6 @@
+<configuration version="15">
+    <folder id="f1" directory="testdata/">
+    </folder>
+    <folder id="f1" directory="testdata/">
+    </folder>
+</configuration>

lib/config/testdata/nopath.xml

@@ -0,0 +1,4 @@
+<configuration version="15">
+    <folder id="f1">
+    </folder>
+</configuration>

lib/config/wrapper.go

@@ -8,6 +8,7 @@ package config
 
 import (
 	"os"
+	"sync/atomic"
 
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/osutil"
@@ -41,11 +42,6 @@ type Committer interface {
 	String() string
 }
 
-type CommitResponse struct {
-	ValidationError error
-	RequiresRestart bool
-}
-
 // A wrapper around a Configuration that manages loads, saves and published
 // notifications of changes to registered Handlers
 
@@ -58,6 +54,8 @@ type Wrapper struct {
 	replaces  chan Configuration
 	subs      []Committer
 	mut       sync.Mutex
+
+	requiresRestart uint32 // an atomic bool
 }
 
 // Wrap wraps an existing Configuration structure and ties it to a file on
@@ -128,32 +126,21 @@ func (w *Wrapper) Raw() Configuration {
 }
 
 // Replace swaps the current configuration object for the given one.
-func (w *Wrapper) Replace(cfg Configuration) CommitResponse {
+func (w *Wrapper) Replace(cfg Configuration) error {
 	w.mut.Lock()
 	defer w.mut.Unlock()
+
 	return w.replaceLocked(cfg)
 }
 
-func (w *Wrapper) replaceLocked(to Configuration) CommitResponse {
+func (w *Wrapper) replaceLocked(to Configuration) error {
 	from := w.cfg
 
 	for _, sub := range w.subs {
 		l.Debugln(sub, "verifying configuration")
 		if err := sub.VerifyConfiguration(from, to); err != nil {
 			l.Debugln(sub, "rejected config:", err)
-			return CommitResponse{
-				ValidationError: err,
-			}
-		}
-	}
-
-	allOk := true
-	for _, sub := range w.subs {
-		l.Debugln(sub, "committing configuration")
-		ok := sub.CommitConfiguration(from, to)
-		if !ok {
-			l.Debugln(sub, "requires restart")
-			allOk = false
+			return err
 		}
 	}
 
@@ -161,8 +148,22 @@ func (w *Wrapper) replaceLocked(to Configuration) CommitResponse {
 	w.deviceMap = nil
 	w.folderMap = nil
 
-	return CommitResponse{
-		RequiresRestart: !allOk,
+	w.notifyListeners(from, to)
+
+	return nil
+}
+
+func (w *Wrapper) notifyListeners(from, to Configuration) {
+	for _, sub := range w.subs {
+		go w.notifyListener(sub, from, to)
+	}
+}
+
+func (w *Wrapper) notifyListener(sub Committer, from, to Configuration) {
+	l.Debugln(sub, "committing configuration")
+	if !sub.CommitConfiguration(from, to) {
+		l.Debugln(sub, "requires restart")
+		w.setRequiresRestart()
 	}
 }
 
@@ -182,7 +183,7 @@ func (w *Wrapper) Devices() map[protocol.DeviceID]DeviceConfiguration {
 
 // SetDevice adds a new device to the configuration, or overwrites an existing
 // device with the same ID.
-func (w *Wrapper) SetDevice(dev DeviceConfiguration) CommitResponse {
+func (w *Wrapper) SetDevice(dev DeviceConfiguration) error {
 	w.mut.Lock()
 	defer w.mut.Unlock()
 
@@ -218,7 +219,7 @@ func (w *Wrapper) Folders() map[string]FolderConfiguration {
 
 // SetFolder adds a new folder to the configuration, or overwrites an existing
 // folder with the same ID.
-func (w *Wrapper) SetFolder(fld FolderConfiguration) CommitResponse {
+func (w *Wrapper) SetFolder(fld FolderConfiguration) error {
 	w.mut.Lock()
 	defer w.mut.Unlock()
 
@@ -246,7 +247,7 @@ func (w *Wrapper) Options() OptionsConfiguration {
 }
 
 // SetOptions replaces the current options configuration object.
-func (w *Wrapper) SetOptions(opts OptionsConfiguration) CommitResponse {
+func (w *Wrapper) SetOptions(opts OptionsConfiguration) error {
 	w.mut.Lock()
 	defer w.mut.Unlock()
 	newCfg := w.cfg.Copy()
@@ -262,7 +263,7 @@ func (w *Wrapper) GUI() GUIConfiguration {
 }
 
 // SetGUI replaces the current GUI configuration object.
-func (w *Wrapper) SetGUI(gui GUIConfiguration) CommitResponse {
+func (w *Wrapper) SetGUI(gui GUIConfiguration) error {
 	w.mut.Lock()
 	defer w.mut.Unlock()
 	newCfg := w.cfg.Copy()
@@ -332,3 +333,11 @@ func (w *Wrapper) ListenAddresses() []string {
 	}
 	return util.UniqueStrings(addresses)
 }
+
+func (w *Wrapper) RequiresRestart() bool {
+	return atomic.LoadUint32(&w.requiresRestart) != 0
+}
+
+func (w *Wrapper) setRequiresRestart() {
+	atomic.StoreUint32(&w.requiresRestart, 1)
+}

lib/connections/relay_dial.go

@@ -52,7 +52,7 @@ func (d *relayDialer) Dial(id protocol.DeviceID, uri *url.URL) (IntermediateConn
 		tc = tls.Client(conn, d.tlsCfg)
 	}
 
-	err = tc.Handshake()
+	err = tlsTimedHandshake(tc)
 	if err != nil {
 		tc.Close()
 		return IntermediateConnection{}, err

lib/connections/relay_listen.go

@@ -85,7 +85,7 @@ func (t *relayListener) Serve() {
 				tc = tls.Client(conn, t.tlsCfg)
 			}
 
-			err = tc.Handshake()
+			err = tlsTimedHandshake(tc)
 			if err != nil {
 				tc.Close()
 				l.Infoln("TLS handshake (BEP/relay):", err)

lib/connections/service.go

@@ -36,7 +36,10 @@ var (
 	listeners = make(map[string]listenerFactory, 0)
 )
 
-const perDeviceWarningRate = 1.0 / (15 * 60) // Once per 15 minutes
+const (
+	perDeviceWarningRate = 1.0 / (15 * 60) // Once per 15 minutes
+	tlsHandshakeTimeout  = 10 * time.Second
+)
 
 // Service listens and dials all configured unconnected devices, via supported
 // dialers. Successful connections are handed to the model.
@@ -159,8 +162,17 @@ next:
 		if err != nil {
 			if protocol.IsVersionMismatch(err) {
 				// The error will be a relatively user friendly description
-				// of what's wrong with the version compatibility
-				msg := fmt.Sprintf("Connecting to %s (%s): %s", remoteID, c.RemoteAddr(), err)
+				// of what's wrong with the version compatibility. By
+				// default identify the other side by device ID and IP.
+				remote := fmt.Sprintf("%v (%v)", remoteID, c.RemoteAddr())
+				if hello.DeviceName != "" {
+					// If the name was set in the hello return, use that to
+					// give the user more info about which device is the
+					// affected one. It probably says more than the remote
+					// IP.
+					remote = fmt.Sprintf("%q (%s %s, %v)", hello.DeviceName, hello.ClientName, hello.ClientVersion, remoteID)
+				}
+				msg := fmt.Sprintf("Connecting to %s: %s", remote, err)
 				warningFor(remoteID, msg)
 			} else {
 				// It's something else - connection reset or whatever
@@ -347,7 +359,7 @@ func (s *Service) connect() {
 				}
 
 				if connected && dialerFactory.Priority() >= ct.Priority {
-					l.Debugf("Not dialing using %s as priorty is less than current connection (%d >= %d)", dialerFactory, dialerFactory.Priority(), ct.Priority)
+					l.Debugf("Not dialing using %s as priority is less than current connection (%d >= %d)", dialerFactory, dialerFactory.Priority(), ct.Priority)
 					continue
 				}
 
@@ -607,3 +619,9 @@ func warningFor(dev protocol.DeviceID, msg string) {
 		l.Warnln(msg)
 	}
 }
+
+func tlsTimedHandshake(tc *tls.Conn) error {
+	tc.SetDeadline(time.Now().Add(tlsHandshakeTimeout))
+	defer tc.SetDeadline(time.Time{})
+	return tc.Handshake()
+}

lib/connections/structs.go

@@ -70,7 +70,7 @@ type Model interface {
 	ConnectedTo(remoteID protocol.DeviceID) bool
 	IsPaused(remoteID protocol.DeviceID) bool
 	OnHello(protocol.DeviceID, net.Addr, protocol.HelloResult)
-	GetHello(protocol.DeviceID) protocol.Version13HelloMessage
+	GetHello(protocol.DeviceID) protocol.HelloIntf
 }
 
 // serviceFunc wraps a function to create a suture.Service without stop

lib/connections/tcp_dial.go

@@ -40,7 +40,7 @@ func (d *tcpDialer) Dial(id protocol.DeviceID, uri *url.URL) (IntermediateConnec
 	}
 
 	tc := tls.Client(conn, d.tlsCfg)
-	err = tc.Handshake()
+	err = tlsTimedHandshake(tc)
 	if err != nil {
 		tc.Close()
 		return IntermediateConnection{}, err

lib/connections/tcp_listen.go

@@ -108,7 +108,7 @@ func (t *tcpListener) Serve() {
 		}
 
 		tc := tls.Server(conn, t.tlsCfg)
-		err = tc.Handshake()
+		err = tlsTimedHandshake(tc)
 		if err != nil {
 			l.Infoln("TLS handshake (BEP/tcp):", err)
 			tc.Close()

lib/db/blockmap_test.go

@@ -73,7 +73,7 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
 
 	m := NewBlockMap(db, db.folderIdx.ID([]byte("folder1")))
 
-	f3.Flags |= protocol.FlagDirectory
+	f3.Type = protocol.FileInfoTypeDirectory
 
 	err := m.Add([]protocol.FileInfo{f1, f2, f3})
 	if err != nil {
@@ -99,9 +99,11 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
 		return true
 	})
 
-	f3.Flags = f1.Flags
-	f1.Flags |= protocol.FlagDeleted
-	f2.Flags |= protocol.FlagInvalid
+	f3.Permissions = f1.Permissions
+	f3.Deleted = f1.Deleted
+	f3.Invalid = f1.Invalid
+	f1.Deleted = true
+	f2.Invalid = true
 
 	// Should remove
 	err = m.Update([]protocol.FileInfo{f1, f2, f3})
@@ -145,9 +147,15 @@ func TestBlockMapAddUpdateWipe(t *testing.T) {
 		t.Fatal("db not empty")
 	}
 
-	f1.Flags = 0
-	f2.Flags = 0
-	f3.Flags = 0
+	f1.Deleted = false
+	f1.Invalid = false
+	f1.Permissions = 0
+	f2.Deleted = false
+	f2.Invalid = false
+	f2.Permissions = 0
+	f3.Deleted = false
+	f3.Invalid = false
+	f3.Permissions = 0
 }
 
 func TestBlockFinderLookup(t *testing.T) {
@@ -187,7 +195,7 @@ func TestBlockFinderLookup(t *testing.T) {
 		t.Fatal("Incorrect count", counter)
 	}
 
-	f1.Flags |= protocol.FlagDeleted
+	f1.Deleted = true
 
 	err = m1.Update([]protocol.FileInfo{f1})
 	if err != nil {
@@ -212,7 +220,7 @@ func TestBlockFinderLookup(t *testing.T) {
 		t.Fatal("Incorrect count")
 	}
 
-	f1.Flags = 0
+	f1.Deleted = false
 }
 
 func TestBlockFinderFix(t *testing.T) {

lib/db/leveldb.go

@@ -4,9 +4,6 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at http://mozilla.org/MPL/2.0/.
 
-//go:generate -command genxdr go run ../../vendor/github.com/calmh/xdr/cmd/genxdr/main.go
-//go:generate genxdr -o leveldb_xdr.go leveldb.go
-
 package db
 
 import (
@@ -46,25 +43,16 @@ const (
 	KeyTypeDeviceIdx
 )
 
-type fileVersion struct {
-	version protocol.Vector
-	device  []byte
-}
-
-type VersionList struct {
-	versions []fileVersion
-}
-
 func (l VersionList) String() string {
 	var b bytes.Buffer
 	var id protocol.DeviceID
 	b.WriteString("{")
-	for i, v := range l.versions {
+	for i, v := range l.Versions {
 		if i > 0 {
 			b.WriteString(", ")
 		}
-		copy(id[:], v.device)
-		fmt.Fprintf(&b, "{%d, %v}", v.version, id)
+		copy(id[:], v.Device)
+		fmt.Fprintf(&b, "{%d, %v}", v.Version, id)
 	}
 	b.WriteString("}")
 	return b.String()
@@ -101,7 +89,7 @@ func getFile(db dbReader, key []byte) (protocol.FileInfo, bool) {
 	}
 
 	var f protocol.FileInfo
-	err = f.UnmarshalXDR(bs)
+	err = f.Unmarshal(bs)
 	if err != nil {
 		panic(err)
 	}

lib/db/leveldb_convert.go

@@ -1,114 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package db
-
-import (
-	"bytes"
-
-	"github.com/syndtr/goleveldb/leveldb"
-)
-
-// convertKeyFormat converts from the v0.12 to the v0.13 database format, to
-// avoid having to do rescan. The change is in the key format for folder
-// labels, so we basically just iterate over the database rewriting keys as
-// necessary.
-func convertKeyFormat(from, to *leveldb.DB) error {
-	l.Infoln("Converting database key format")
-	blocks, files, globals, unchanged := 0, 0, 0, 0
-
-	dbi := newDBInstance(to)
-	i := from.NewIterator(nil, nil)
-	for i.Next() {
-		key := i.Key()
-		switch key[0] {
-		case KeyTypeBlock:
-			folder, file := oldFromBlockKey(key)
-			folderIdx := dbi.folderIdx.ID([]byte(folder))
-			hash := key[1+64:]
-			newKey := blockKeyInto(nil, hash, folderIdx, file)
-			if err := to.Put(newKey, i.Value(), nil); err != nil {
-				return err
-			}
-			blocks++
-
-		case KeyTypeDevice:
-			newKey := dbi.deviceKey(oldDeviceKeyFolder(key), oldDeviceKeyDevice(key), oldDeviceKeyName(key))
-			if err := to.Put(newKey, i.Value(), nil); err != nil {
-				return err
-			}
-			files++
-
-		case KeyTypeGlobal:
-			newKey := dbi.globalKey(oldGlobalKeyFolder(key), oldGlobalKeyName(key))
-			if err := to.Put(newKey, i.Value(), nil); err != nil {
-				return err
-			}
-			globals++
-
-		case KeyTypeVirtualMtime:
-			// Cannot be converted, we drop it instead :(
-
-		default:
-			if err := to.Put(key, i.Value(), nil); err != nil {
-				return err
-			}
-			unchanged++
-		}
-	}
-
-	l.Infof("Converted %d blocks, %d files, %d globals (%d unchanged).", blocks, files, globals, unchanged)
-
-	return nil
-}
-
-func oldDeviceKeyFolder(key []byte) []byte {
-	folder := key[1 : 1+64]
-	izero := bytes.IndexByte(folder, 0)
-	if izero < 0 {
-		return folder
-	}
-	return folder[:izero]
-}
-
-func oldDeviceKeyDevice(key []byte) []byte {
-	return key[1+64 : 1+64+32]
-}
-
-func oldDeviceKeyName(key []byte) []byte {
-	return key[1+64+32:]
-}
-
-func oldGlobalKeyName(key []byte) []byte {
-	return key[1+64:]
-}
-
-func oldGlobalKeyFolder(key []byte) []byte {
-	folder := key[1 : 1+64]
-	izero := bytes.IndexByte(folder, 0)
-	if izero < 0 {
-		return folder
-	}
-	return folder[:izero]
-}
-
-func oldFromBlockKey(data []byte) (string, string) {
-	if len(data) < 1+64+32+1 {
-		panic("Incorrect key length")
-	}
-	if data[0] != KeyTypeBlock {
-		panic("Incorrect key type")
-	}
-
-	file := string(data[1+64+32:])
-
-	slice := data[1 : 1+64]
-	izero := bytes.IndexByte(slice, 0)
-	if izero > -1 {
-		return string(slice[:izero]), file
-	}
-	return string(slice), file
-}

lib/db/leveldb_convert_test.go

@@ -1,136 +0,0 @@
-// Copyright (C) 2015 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package db
-
-import (
-	"archive/zip"
-	"io"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/syndtr/goleveldb/leveldb"
-)
-
-func TestLabelConversion(t *testing.T) {
-	os.RemoveAll("testdata/oldformat.db")
-	defer os.RemoveAll("testdata/oldformat.db")
-	os.RemoveAll("testdata/newformat.db")
-	defer os.RemoveAll("testdata/newformat.db")
-
-	if err := unzip("testdata/oldformat.db.zip", "testdata"); err != nil {
-		t.Fatal(err)
-	}
-
-	odb, err := leveldb.OpenFile("testdata/oldformat.db", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	ldb, err := leveldb.OpenFile("testdata/newformat.db", nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err = convertKeyFormat(odb, ldb); err != nil {
-		t.Fatal(err)
-	}
-	ldb.Close()
-	odb.Close()
-
-	inst, err := Open("testdata/newformat.db")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	fs := NewFileSet("default", inst)
-	files, deleted, _ := fs.GlobalSize()
-	if files+deleted != 953 {
-		// Expected number of global entries determined by
-		// ../../bin/stindex testdata/oldformat.db/ | grep global | grep -c default
-		t.Errorf("Conversion error, global list differs (%d != 953)", files+deleted)
-	}
-
-	files, deleted, _ = fs.LocalSize()
-	if files+deleted != 953 {
-		t.Errorf("Conversion error, device list differs (%d != 953)", files+deleted)
-	}
-
-	f := NewBlockFinder(inst)
-	// [block] F:"default" H:1c25dea9003cc16216e2a22900be1ec1cc5aaf270442904e2f9812c314e929d8 N:"f/f2/f25f1b3e6e029231b933531b2138796d" I:3
-	h := []byte{0x1c, 0x25, 0xde, 0xa9, 0x00, 0x3c, 0xc1, 0x62, 0x16, 0xe2, 0xa2, 0x29, 0x00, 0xbe, 0x1e, 0xc1, 0xcc, 0x5a, 0xaf, 0x27, 0x04, 0x42, 0x90, 0x4e, 0x2f, 0x98, 0x12, 0xc3, 0x14, 0xe9, 0x29, 0xd8}
-	found := 0
-	f.Iterate([]string{"default"}, h, func(folder, file string, idx int32) bool {
-		if folder == "default" && file == filepath.FromSlash("f/f2/f25f1b3e6e029231b933531b2138796d") && idx == 3 {
-			found++
-		}
-		return true
-	})
-	if found != 1 {
-		t.Errorf("Found %d blocks instead of expected 1", found)
-	}
-
-	inst.Close()
-}
-
-func unzip(src, dest string) error {
-	r, err := zip.OpenReader(src)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err := r.Close(); err != nil {
-			panic(err)
-		}
-	}()
-
-	os.MkdirAll(dest, 0755)
-
-	// Closure to address file descriptors issue with all the deferred .Close() methods
-	extractAndWriteFile := func(f *zip.File) error {
-		rc, err := f.Open()
-		if err != nil {
-			return err
-		}
-		defer func() {
-			if err := rc.Close(); err != nil {
-				panic(err)
-			}
-		}()
-
-		path := filepath.Join(dest, f.Name)
-
-		if f.FileInfo().IsDir() {
-			os.MkdirAll(path, f.Mode())
-		} else {
-			f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
-			if err != nil {
-				return err
-			}
-			defer func() {
-				if err := f.Close(); err != nil {
-					panic(err)
-				}
-			}()
-
-			_, err = io.Copy(f, rc)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
-	for _, f := range r.File {
-		err := extractAndWriteFile(f)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

lib/db/leveldb_dbinstance.go

@@ -10,12 +10,10 @@ import (
 	"bytes"
 	"encoding/binary"
 	"os"
-	"path/filepath"
 	"sort"
 	"strings"
 	"sync/atomic"
 
-	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/sync"
 	"github.com/syndtr/goleveldb/leveldb"
@@ -29,10 +27,10 @@ import (
 type deletionHandler func(t readWriteTransaction, folder, device, name []byte, dbi iterator.Iterator) int64
 
 type Instance struct {
+	committed int64 // this must be the first attribute in the struct to ensure 64 bit alignment on 32 bit plaforms
 	*leveldb.DB
 	folderIdx *smallIndex
 	deviceIdx *smallIndex
-	committed int64
 }
 
 const (
@@ -48,16 +46,6 @@ func Open(file string) (*Instance, error) {
 		WriteBuffer:            4 << 20,
 	}
 
-	if _, err := os.Stat(file); os.IsNotExist(err) {
-		// The file we are looking to open does not exist. This may be the
-		// first launch so we should look for an old version and try to
-		// convert it.
-		if err := checkConvertDatabase(file); err != nil {
-			l.Infoln("Converting old database:", err)
-			l.Infoln("Will rescan from scratch.")
-		}
-	}
-
 	db, err := leveldb.OpenFile(file, opts)
 	if leveldbIsCorrupted(err) {
 		db, err = leveldb.RecoverFile(file, opts)
@@ -151,12 +139,12 @@ func (db *Instance) genericReplace(folder, device []byte, fs []protocol.FileInfo
 
 		case moreFs && moreDb && cmp == 0:
 			// File exists on both sides - compare versions. We might get an
-			// update with the same version and different flags if a device has
-			// marked a file as invalid, so handle that too.
+			// update with the same version if a device has marked a file as
+			// invalid, so handle that too.
 			l.Debugln("generic replace; exists - compare")
 			var ef FileInfoTruncated
-			ef.UnmarshalXDR(dbi.Value())
-			if !fs[fsi].Version.Equal(ef.Version) || fs[fsi].Flags != ef.Flags {
+			ef.Unmarshal(dbi.Value())
+			if !fs[fsi].Version.Equal(ef.Version) || fs[fsi].Invalid != ef.Invalid {
 				l.Debugln("generic replace; differs - insert")
 				if lv := t.insertFile(folder, device, fs[fsi]); lv > maxLocalVer {
 					maxLocalVer = lv
@@ -232,13 +220,12 @@ func (db *Instance) updateFiles(folder, device []byte, fs []protocol.FileInfo, l
 		}
 
 		var ef FileInfoTruncated
-		err = ef.UnmarshalXDR(bs)
+		err = ef.Unmarshal(bs)
 		if err != nil {
 			panic(err)
 		}
-		// Flags might change without the version being bumped when we set the
-		// invalid flag on an existing file.
-		if !ef.Version.Equal(f.Version) || ef.Flags != f.Flags {
+		// The Invalid flag might change without the version being bumped.
+		if !ef.Version.Equal(f.Version) || ef.Invalid != f.Invalid {
 			if isLocalDevice {
 				localSize.removeFile(ef)
 				localSize.addFile(f)
@@ -308,7 +295,7 @@ func (db *Instance) withAllFolderTruncated(folder []byte, fn func(device []byte,
 		// struct, which in turn references the buffer it was unmarshalled
 		// from. dbi.Value() just returns an internal slice that it reuses, so
 		// we need to copy it.
-		err := f.UnmarshalXDR(append([]byte{}, dbi.Value()...))
+		err := f.Unmarshal(append([]byte{}, dbi.Value()...))
 		if err != nil {
 			panic(err)
 		}
@@ -347,16 +334,16 @@ func (db *Instance) getGlobal(folder, file []byte, truncate bool) (FileIntf, boo
 	}
 
 	var vl VersionList
-	err = vl.UnmarshalXDR(bs)
+	err = vl.Unmarshal(bs)
 	if err != nil {
 		panic(err)
 	}
-	if len(vl.versions) == 0 {
+	if len(vl.Versions) == 0 {
 		l.Debugln(k)
 		panic("no versions?")
 	}
 
-	k = db.deviceKey(folder, vl.versions[0].device, file)
+	k = db.deviceKey(folder, vl.Versions[0].Device, file)
 	bs, err = t.Get(k, nil)
 	if err != nil {
 		panic(err)
@@ -384,11 +371,11 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
 	var fk []byte
 	for dbi.Next() {
 		var vl VersionList
-		err := vl.UnmarshalXDR(dbi.Value())
+		err := vl.Unmarshal(dbi.Value())
 		if err != nil {
 			panic(err)
 		}
-		if len(vl.versions) == 0 {
+		if len(vl.Versions) == 0 {
 			l.Debugln(dbi.Key())
 			panic("no versions?")
 		}
@@ -398,13 +385,13 @@ func (db *Instance) withGlobal(folder, prefix []byte, truncate bool, fn Iterator
 			return
 		}
 
-		fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.versions[0].device, name)
+		fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.Versions[0].Device, name)
 		bs, err := t.Get(fk, nil)
 		if err != nil {
 			l.Debugf("folder: %q (%x)", folder, folder)
 			l.Debugf("key: %q (%x)", dbi.Key(), dbi.Key())
 			l.Debugf("vl: %v", vl)
-			l.Debugf("vl.versions[0].device: %x", vl.versions[0].device)
+			l.Debugf("vl.Versions[0].Device: %x", vl.Versions[0].Device)
 			l.Debugf("name: %q (%x)", name, name)
 			l.Debugf("fk: %q", fk)
 			l.Debugf("fk: %x %x %x",
@@ -436,17 +423,17 @@ func (db *Instance) availability(folder, file []byte) []protocol.DeviceID {
 	}
 
 	var vl VersionList
-	err = vl.UnmarshalXDR(bs)
+	err = vl.Unmarshal(bs)
 	if err != nil {
 		panic(err)
 	}
 
 	var devices []protocol.DeviceID
-	for _, v := range vl.versions {
-		if !v.version.Equal(vl.versions[0].version) {
+	for _, v := range vl.Versions {
+		if !v.Version.Equal(vl.Versions[0].Version) {
 			break
 		}
-		n := protocol.DeviceIDFromBytes(v.device)
+		n := protocol.DeviceIDFromBytes(v.Device)
 		devices = append(devices, n)
 	}
 
@@ -464,11 +451,11 @@ func (db *Instance) withNeed(folder, device []byte, truncate bool, fn Iterator)
 nextFile:
 	for dbi.Next() {
 		var vl VersionList
-		err := vl.UnmarshalXDR(dbi.Value())
+		err := vl.Unmarshal(dbi.Value())
 		if err != nil {
 			panic(err)
 		}
-		if len(vl.versions) == 0 {
+		if len(vl.Versions) == 0 {
 			l.Debugln(dbi.Key())
 			panic("no versions?")
 		}
@@ -476,29 +463,29 @@ nextFile:
 		have := false // If we have the file, any version
 		need := false // If we have a lower version of the file
 		var haveVersion protocol.Vector
-		for _, v := range vl.versions {
-			if bytes.Equal(v.device, device) {
+		for _, v := range vl.Versions {
+			if bytes.Equal(v.Device, device) {
 				have = true
-				haveVersion = v.version
+				haveVersion = v.Version
 				// XXX: This marks Concurrent (i.e. conflicting) changes as
 				// needs. Maybe we should do that, but it needs special
 				// handling in the puller.
-				need = !v.version.GreaterEqual(vl.versions[0].version)
+				need = !v.Version.GreaterEqual(vl.Versions[0].Version)
 				break
 			}
 		}
 
 		if need || !have {
 			name := db.globalKeyName(dbi.Key())
-			needVersion := vl.versions[0].version
+			needVersion := vl.Versions[0].Version
 
 		nextVersion:
-			for i := range vl.versions {
-				if !vl.versions[i].version.Equal(needVersion) {
+			for i := range vl.Versions {
+				if !vl.Versions[i].Version.Equal(needVersion) {
 					// We haven't found a valid copy of the file with the needed version.
 					continue nextFile
 				}
-				fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.versions[i].device, name)
+				fk = db.deviceKeyInto(fk[:cap(fk)], folder, vl.Versions[i].Device, name)
 				bs, err := t.Get(fk, nil)
 				if err != nil {
 					var id protocol.DeviceID
@@ -528,7 +515,7 @@ nextFile:
 					continue nextFile
 				}
 
-				l.Debugf("need folder=%q device=%v name=%q need=%v have=%v haveV=%d globalV=%d", folder, protocol.DeviceIDFromBytes(device), name, need, have, haveVersion, vl.versions[0].version)
+				l.Debugf("need folder=%q device=%v name=%q need=%v have=%v haveV=%d globalV=%d", folder, protocol.DeviceIDFromBytes(device), name, need, have, haveVersion, vl.Versions[0].Version)
 
 				if cont := fn(gf); !cont {
 					return
@@ -601,7 +588,7 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
 	for dbi.Next() {
 		gk := dbi.Key()
 		var vl VersionList
-		err := vl.UnmarshalXDR(dbi.Value())
+		err := vl.Unmarshal(dbi.Value())
 		if err != nil {
 			panic(err)
 		}
@@ -613,8 +600,8 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
 
 		name := db.globalKeyName(gk)
 		var newVL VersionList
-		for i, version := range vl.versions {
-			fk = db.deviceKeyInto(fk[:cap(fk)], folder, version.device, name)
+		for i, version := range vl.Versions {
+			fk = db.deviceKeyInto(fk[:cap(fk)], folder, version.Device, name)
 
 			_, err := t.Get(fk, nil)
 			if err == leveldb.ErrNotFound {
@@ -623,10 +610,10 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
 			if err != nil {
 				panic(err)
 			}
-			newVL.versions = append(newVL.versions, version)
+			newVL.Versions = append(newVL.Versions, version)
 
 			if i == 0 {
-				fi, ok := t.getFile(folder, version.device, name)
+				fi, ok := t.getFile(folder, version.Device, name)
 				if !ok {
 					panic("nonexistent global master file")
 				}
@@ -634,8 +621,8 @@ func (db *Instance) checkGlobals(folder []byte, globalSize *sizeTracker) {
 			}
 		}
 
-		if len(newVL.versions) != len(vl.versions) {
-			t.Put(dbi.Key(), newVL.MustMarshalXDR())
+		if len(newVL.Versions) != len(vl.Versions) {
+			t.Put(dbi.Key(), mustMarshal(&newVL))
 			t.checkFlush()
 		}
 	}
@@ -715,12 +702,12 @@ func (db *Instance) globalKeyFolder(key []byte) []byte {
 func unmarshalTrunc(bs []byte, truncate bool) (FileIntf, error) {
 	if truncate {
 		var tf FileInfoTruncated
-		err := tf.UnmarshalXDR(bs)
+		err := tf.Unmarshal(bs)
 		return tf, err
 	}
 
 	var tf protocol.FileInfo
-	err := tf.UnmarshalXDR(bs)
+	err := tf.Unmarshal(bs)
 	return tf, err
 }
 
@@ -740,50 +727,6 @@ func leveldbIsCorrupted(err error) bool {
 	return false
 }
 
-// checkConvertDatabase tries to convert an existing old (v0.11) database to
-// new (v0.13) format.
-func checkConvertDatabase(dbFile string) error {
-	oldLoc := filepath.Join(filepath.Dir(dbFile), "index-v0.11.0.db")
-	if _, err := os.Stat(oldLoc); os.IsNotExist(err) {
-		// The old database file does not exist; that's ok, continue as if
-		// everything succeeded.
-		return nil
-	} else if err != nil {
-		// Any other error is weird.
-		return err
-	}
-
-	// There exists a database in the old format. We run a one time
-	// conversion from old to new.
-
-	fromDb, err := leveldb.OpenFile(oldLoc, nil)
-	if err != nil {
-		return err
-	}
-
-	toDb, err := leveldb.OpenFile(dbFile, nil)
-	if err != nil {
-		return err
-	}
-
-	err = convertKeyFormat(fromDb, toDb)
-	if err != nil {
-		return err
-	}
-
-	err = toDb.Close()
-	if err != nil {
-		return err
-	}
-
-	// We've done this one, we don't want to do it again (if the user runs
-	// -reset or so). We don't care too much about errors any more at this stage.
-	fromDb.Close()
-	osutil.Rename(oldLoc, oldLoc+".converted")
-
-	return nil
-}
-
 // A smallIndex is an in memory bidirectional []byte to uint32 map. It gives
 // fast lookups in both directions and persists to the database. Don't use for
 // storing more items than fit comfortably in RAM.

lib/db/leveldb_transactions.go

@@ -83,7 +83,7 @@ func (t readWriteTransaction) insertFile(folder, device []byte, file protocol.Fi
 
 	name := []byte(file.Name)
 	nk := t.db.deviceKey(folder, device, name)
-	t.Put(nk, file.MustMarshalXDR())
+	t.Put(nk, mustMarshal(&file))
 
 	return file.LocalVersion
 }
@@ -105,14 +105,14 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
 	var hasOldFile bool
 	// Remove the device from the current version list
 	if len(svl) != 0 {
-		err = fl.UnmarshalXDR(svl)
+		err = fl.Unmarshal(svl)
 		if err != nil {
 			panic(err)
 		}
 
-		for i := range fl.versions {
-			if bytes.Equal(fl.versions[i].device, device) {
-				if fl.versions[i].version.Equal(file.Version) {
+		for i := range fl.Versions {
+			if bytes.Equal(fl.Versions[i].Device, device) {
+				if fl.Versions[i].Version.Equal(file.Version) {
 					// No need to do anything
 					return false
 				}
@@ -120,29 +120,29 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
 				if i == 0 {
 					// Keep the current newest file around so we can subtract it from
 					// the globalSize if we replace it.
-					oldFile, hasOldFile = t.getFile(folder, fl.versions[0].device, name)
+					oldFile, hasOldFile = t.getFile(folder, fl.Versions[0].Device, name)
 				}
 
-				fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
+				fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...)
 				break
 			}
 		}
 	}
 
-	nv := fileVersion{
-		device:  device,
-		version: file.Version,
+	nv := FileVersion{
+		Device:  device,
+		Version: file.Version,
 	}
 
 	insertedAt := -1
 	// Find a position in the list to insert this file. The file at the front
 	// of the list is the newer, the "global".
-	for i := range fl.versions {
-		switch fl.versions[i].version.Compare(file.Version) {
+	for i := range fl.Versions {
+		switch fl.Versions[i].Version.Compare(file.Version) {
 		case protocol.Equal, protocol.Lesser:
 			// The version at this point in the list is equal to or lesser
 			// ("older") than us. We insert ourselves in front of it.
-			fl.versions = insertVersion(fl.versions, i, nv)
+			fl.Versions = insertVersion(fl.Versions, i, nv)
 			insertedAt = i
 			goto done
 
@@ -153,12 +153,12 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
 			// "Greater" in the condition above is just based on the device
 			// IDs in the version vector, which is not the only thing we use
 			// to determine the winner.)
-			of, ok := t.getFile(folder, fl.versions[i].device, name)
+			of, ok := t.getFile(folder, fl.Versions[i].Device, name)
 			if !ok {
 				panic("file referenced in version list does not exist")
 			}
 			if file.WinsConflict(of) {
-				fl.versions = insertVersion(fl.versions, i, nv)
+				fl.Versions = insertVersion(fl.Versions, i, nv)
 				insertedAt = i
 				goto done
 			}
@@ -166,8 +166,8 @@ func (t readWriteTransaction) updateGlobal(folder, device []byte, file protocol.
 	}
 
 	// We didn't find a position for an insert above, so append to the end.
-	fl.versions = append(fl.versions, nv)
-	insertedAt = len(fl.versions) - 1
+	fl.Versions = append(fl.Versions, nv)
+	insertedAt = len(fl.Versions) - 1
 
 done:
 	if insertedAt == 0 {
@@ -178,9 +178,9 @@ done:
 			if hasOldFile {
 				// We have the old file that was removed at the head of the list.
 				globalSize.removeFile(oldFile)
-			} else if len(fl.versions) > 1 {
+			} else if len(fl.Versions) > 1 {
 				// The previous newest version is now at index 1, grab it from there.
-				oldFile, ok := t.getFile(folder, fl.versions[1].device, name)
+				oldFile, ok := t.getFile(folder, fl.Versions[1].Device, name)
 				if !ok {
 					panic("file referenced in version list does not exist")
 				}
@@ -190,7 +190,7 @@ done:
 	}
 
 	l.Debugf("new global after update: %v", fl)
-	t.Put(gk, fl.MustMarshalXDR())
+	t.Put(gk, mustMarshal(&fl))
 
 	return true
 }
@@ -210,14 +210,14 @@ func (t readWriteTransaction) removeFromGlobal(folder, device, file []byte, glob
 	}
 
 	var fl VersionList
-	err = fl.UnmarshalXDR(svl)
+	err = fl.Unmarshal(svl)
 	if err != nil {
 		panic(err)
 	}
 
 	removed := false
-	for i := range fl.versions {
-		if bytes.Equal(fl.versions[i].device, device) {
+	for i := range fl.Versions {
+		if bytes.Equal(fl.Versions[i].Device, device) {
 			if i == 0 && globalSize != nil {
 				f, ok := t.getFile(folder, device, file)
 				if !ok {
@@ -226,18 +226,18 @@ func (t readWriteTransaction) removeFromGlobal(folder, device, file []byte, glob
 				globalSize.removeFile(f)
 				removed = true
 			}
-			fl.versions = append(fl.versions[:i], fl.versions[i+1:]...)
+			fl.Versions = append(fl.Versions[:i], fl.Versions[i+1:]...)
 			break
 		}
 	}
 
-	if len(fl.versions) == 0 {
+	if len(fl.Versions) == 0 {
 		t.Delete(gk)
 	} else {
 		l.Debugf("new global after remove: %v", fl)
-		t.Put(gk, fl.MustMarshalXDR())
+		t.Put(gk, mustMarshal(&fl))
 		if removed {
-			f, ok := t.getFile(folder, fl.versions[0].device, file)
+			f, ok := t.getFile(folder, fl.Versions[0].Device, file)
 			if !ok {
 				panic("new global is nonexistent file")
 			}
@@ -246,9 +246,21 @@ func (t readWriteTransaction) removeFromGlobal(folder, device, file []byte, glob
 	}
 }
 
-func insertVersion(vl []fileVersion, i int, v fileVersion) []fileVersion {
-	t := append(vl, fileVersion{})
+func insertVersion(vl []FileVersion, i int, v FileVersion) []FileVersion {
+	t := append(vl, FileVersion{})
 	copy(t[i+1:], t[i:])
 	t[i] = v
 	return t
 }
+
+type marshaller interface {
+	Marshal() ([]byte, error)
+}
+
+func mustMarshal(f marshaller) []byte {
+	bs, err := f.Marshal()
+	if err != nil {
+		panic(err)
+	}
+	return bs
+}

lib/db/leveldb_xdr.go

@@ -1,142 +0,0 @@
-// ************************************************************
-// This file is automatically generated by genxdr. Do not edit.
-// ************************************************************
-
-package db
-
-import (
-	"github.com/calmh/xdr"
-)
-
-/*
-
-fileVersion Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                       Vector Structure                        \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                 device (length + padded data)                 \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct fileVersion {
-	Vector version;
-	opaque device<>;
-}
-
-*/
-
-func (o fileVersion) XDRSize() int {
-	return o.version.XDRSize() +
-		4 + len(o.device) + xdr.Padding(len(o.device))
-}
-
-func (o fileVersion) MarshalXDR() ([]byte, error) {
-	buf := make([]byte, o.XDRSize())
-	m := &xdr.Marshaller{Data: buf}
-	return buf, o.MarshalXDRInto(m)
-}
-
-func (o fileVersion) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o fileVersion) MarshalXDRInto(m *xdr.Marshaller) error {
-	if err := o.version.MarshalXDRInto(m); err != nil {
-		return err
-	}
-	m.MarshalBytes(o.device)
-	return m.Error
-}
-
-func (o *fileVersion) UnmarshalXDR(bs []byte) error {
-	u := &xdr.Unmarshaller{Data: bs}
-	return o.UnmarshalXDRFrom(u)
-}
-func (o *fileVersion) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	(&o.version).UnmarshalXDRFrom(u)
-	o.device = u.UnmarshalBytes()
-	return u.Error
-}
-
-/*
-
-VersionList Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                      Number of versions                       |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\              Zero or more fileVersion Structures              \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct VersionList {
-	fileVersion versions<>;
-}
-
-*/
-
-func (o VersionList) XDRSize() int {
-	return 4 + xdr.SizeOfSlice(o.versions)
-}
-
-func (o VersionList) MarshalXDR() ([]byte, error) {
-	buf := make([]byte, o.XDRSize())
-	m := &xdr.Marshaller{Data: buf}
-	return buf, o.MarshalXDRInto(m)
-}
-
-func (o VersionList) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o VersionList) MarshalXDRInto(m *xdr.Marshaller) error {
-	m.MarshalUint32(uint32(len(o.versions)))
-	for i := range o.versions {
-		if err := o.versions[i].MarshalXDRInto(m); err != nil {
-			return err
-		}
-	}
-	return m.Error
-}
-
-func (o *VersionList) UnmarshalXDR(bs []byte) error {
-	u := &xdr.Unmarshaller{Data: bs}
-	return o.UnmarshalXDRFrom(u)
-}
-func (o *VersionList) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	_versionsSize := int(u.UnmarshalUint32())
-	if _versionsSize < 0 {
-		return xdr.ElementSizeExceeded("versions", _versionsSize, 0)
-	} else if _versionsSize == 0 {
-		o.versions = nil
-	} else {
-		if _versionsSize <= len(o.versions) {
-			o.versions = o.versions[:_versionsSize]
-		} else {
-			o.versions = make([]fileVersion, _versionsSize)
-		}
-		for i := range o.versions {
-			(&o.versions[i]).UnmarshalXDRFrom(u)
-		}
-	}
-	return u.Error
-}

lib/db/set.go

@@ -31,9 +31,10 @@ type FileSet struct {
 }
 
 // FileIntf is the set of methods implemented by both protocol.FileInfo and
-// protocol.FileInfoTruncated.
+// FileInfoTruncated.
 type FileIntf interface {
-	Size() int64
+	FileSize() int64
+	FileName() string
 	IsDeleted() bool
 	IsInvalid() bool
 	IsDirectory() bool
@@ -42,7 +43,7 @@ type FileIntf interface {
 }
 
 // The Iterator is called with either a protocol.FileInfo or a
-// protocol.FileInfoTruncated (depending on the method) and returns true to
+// FileInfoTruncated (depending on the method) and returns true to
 // continue iteration, false to stop.
 type Iterator func(f FileIntf) bool
 
@@ -64,7 +65,7 @@ func (s *sizeTracker) addFile(f FileIntf) {
 	} else {
 		s.files++
 	}
-	s.bytes += f.Size()
+	s.bytes += f.FileSize()
 	s.mut.Unlock()
 }
 
@@ -79,7 +80,7 @@ func (s *sizeTracker) removeFile(f FileIntf) {
 	} else {
 		s.files--
 	}
-	s.bytes -= f.Size()
+	s.bytes -= f.FileSize()
 	if s.deleted < 0 || s.files < 0 {
 		panic("bug: removed more than added")
 	}

lib/db/set_test.go

@@ -88,7 +88,7 @@ func (l fileList) String() string {
 	var b bytes.Buffer
 	b.WriteString("[]protocol.FileList{\n")
 	for _, f := range l {
-		fmt.Fprintf(&b, "  %q: #%d, %d bytes, %d blocks, flags=%o\n", f.Name, f.Version, f.Size(), len(f.Blocks), f.Flags)
+		fmt.Fprintf(&b, "  %q: #%d, %d bytes, %d blocks, perms=%o\n", f.Name, f.Version, f.Size, len(f.Blocks), f.Permissions)
 	}
 	b.WriteString("}")
 	return b.String()
@@ -100,35 +100,35 @@ func TestGlobalSet(t *testing.T) {
 	m := db.NewFileSet("test", ldb)
 
 	local0 := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(3)},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(8)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(3)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "z", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(8)},
 	}
 	local1 := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(3)},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(3)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "z", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Deleted: true},
 	}
 	localTot := fileList{
 		local0[0],
 		local0[1],
 		local0[2],
 		local0[3],
-		protocol.FileInfo{Name: "z", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagDeleted},
+		protocol.FileInfo{Name: "z", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Deleted: true},
 	}
 
 	remote0 := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(5)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(5)},
 	}
 	remote1 := fileList{
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(6)},
-		protocol.FileInfo{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(6)},
+		protocol.FileInfo{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(7)},
 	}
 	remoteTot := fileList{
 		remote0[0],
@@ -178,7 +178,7 @@ func TestGlobalSet(t *testing.T) {
 		} else {
 			globalFiles++
 		}
-		globalBytes += f.Size()
+		globalBytes += f.FileSize()
 	}
 	gsFiles, gsDeleted, gsBytes := m.GlobalSize()
 	if gsFiles != globalFiles {
@@ -208,7 +208,7 @@ func TestGlobalSet(t *testing.T) {
 		} else {
 			haveFiles++
 		}
-		haveBytes += f.Size()
+		haveBytes += f.FileSize()
 	}
 	lsFiles, lsDeleted, lsBytes := m.LocalSize()
 	if lsFiles != haveFiles {
@@ -303,23 +303,23 @@ func TestNeedWithInvalid(t *testing.T) {
 	s := db.NewFileSet("test", ldb)
 
 	localHave := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
 	}
 	remote0Have := fileList{
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), Invalid: true},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
 	}
 	remote1Have := fileList{
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(5), Invalid: true},
+		protocol.FileInfo{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), Invalid: true},
 	}
 
 	expectedNeed := fileList{
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
 	}
 
 	s.Replace(protocol.LocalDeviceID, localHave)
@@ -340,10 +340,10 @@ func TestUpdateToInvalid(t *testing.T) {
 	s := db.NewFileSet("test", ldb)
 
 	localHave := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), Invalid: true},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
 	}
 
 	s.Replace(protocol.LocalDeviceID, localHave)
@@ -355,7 +355,7 @@ func TestUpdateToInvalid(t *testing.T) {
 		t.Errorf("Have incorrect before invalidation;\n A: %v !=\n E: %v", have, localHave)
 	}
 
-	localHave[1] = protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}, Flags: protocol.FlagInvalid}
+	localHave[1] = protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Invalid: true}
 	s.Update(protocol.LocalDeviceID, localHave[1:2])
 
 	have = fileList(haveList(s, protocol.LocalDeviceID))
@@ -372,16 +372,16 @@ func TestInvalidAvailability(t *testing.T) {
 	s := db.NewFileSet("test", ldb)
 
 	remote0Have := fileList{
-		protocol.FileInfo{Name: "both", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "r1only", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "r0only", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(7)},
-		protocol.FileInfo{Name: "none", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "both", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "r1only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(5), Invalid: true},
+		protocol.FileInfo{Name: "r0only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "none", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), Invalid: true},
 	}
 	remote1Have := fileList{
-		protocol.FileInfo{Name: "both", Version: protocol.Vector{{ID: myID, Value: 1001}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "r1only", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(7)},
-		protocol.FileInfo{Name: "r0only", Version: protocol.Vector{{ID: myID, Value: 1003}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "none", Version: protocol.Vector{{ID: myID, Value: 1004}}, Blocks: genBlocks(5), Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "both", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "r1only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(7)},
+		protocol.FileInfo{Name: "r0only", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1003}}}, Blocks: genBlocks(5), Invalid: true},
+		protocol.FileInfo{Name: "none", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1004}}}, Blocks: genBlocks(5), Invalid: true},
 	}
 
 	s.Replace(remoteDevice0, remote0Have)
@@ -410,17 +410,17 @@ func TestGlobalReset(t *testing.T) {
 	m := db.NewFileSet("test", ldb)
 
 	local := []protocol.FileInfo{
-		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	remote := []protocol.FileInfo{
-		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
-		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
+		{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	m.Replace(protocol.LocalDeviceID, local)
@@ -448,23 +448,23 @@ func TestNeed(t *testing.T) {
 	m := db.NewFileSet("test", ldb)
 
 	local := []protocol.FileInfo{
-		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	remote := []protocol.FileInfo{
-		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
-		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
+		{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	shouldNeed := []protocol.FileInfo{
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1001}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}},
-		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1001}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
+		{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	m.Replace(protocol.LocalDeviceID, local)
@@ -486,18 +486,18 @@ func TestLocalVersion(t *testing.T) {
 	m := db.NewFileSet("test", ldb)
 
 	local1 := []protocol.FileInfo{
-		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	local2 := []protocol.FileInfo{
 		local1[0],
 		// [1] deleted
 		local1[2],
-		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1002}}},
-		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
+		{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	m.Replace(protocol.LocalDeviceID, local1)
@@ -515,17 +515,17 @@ func TestListDropFolder(t *testing.T) {
 
 	s0 := db.NewFileSet("test0", ldb)
 	local1 := []protocol.FileInfo{
-		{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}},
-		{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
+		{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 	s0.Replace(protocol.LocalDeviceID, local1)
 
 	s1 := db.NewFileSet("test1", ldb)
 	local2 := []protocol.FileInfo{
-		{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1002}}},
-		{Name: "e", Version: protocol.Vector{{ID: myID, Value: 1002}}},
-		{Name: "f", Version: protocol.Vector{{ID: myID, Value: 1002}}},
+		{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
+		{Name: "e", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
+		{Name: "f", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}},
 	}
 	s1.Replace(remoteDevice0, local2)
 
@@ -566,24 +566,24 @@ func TestGlobalNeedWithInvalid(t *testing.T) {
 	s := db.NewFileSet("test1", ldb)
 
 	rem0 := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Flags: protocol.FlagInvalid},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Invalid: true},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
 	}
 	s.Replace(remoteDevice0, rem0)
 
 	rem1 := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Flags: protocol.FlagInvalid},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Invalid: true},
 	}
 	s.Replace(remoteDevice1, rem1)
 
 	total := fileList{
 		// There's a valid copy of each file, so it should be merged
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1002}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1002}}}, Blocks: genBlocks(4)},
 	}
 
 	need := fileList(needList(s, protocol.LocalDeviceID))
@@ -609,7 +609,7 @@ func TestLongPath(t *testing.T) {
 	name := b.String() // 5000 characters
 
 	local := []protocol.FileInfo{
-		{Name: string(name), Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: string(name), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	s.Replace(protocol.LocalDeviceID, local)
@@ -633,7 +633,7 @@ func TestCommitted(t *testing.T) {
 	s := db.NewFileSet("test", ldb)
 
 	local := []protocol.FileInfo{
-		{Name: string("file"), Version: protocol.Vector{{ID: myID, Value: 1000}}},
+		{Name: string("file"), Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}},
 	}
 
 	// Adding a file should increase the counter
@@ -659,12 +659,12 @@ func TestCommitted(t *testing.T) {
 
 func BenchmarkUpdateOneFile(b *testing.B) {
 	local0 := fileList{
-		protocol.FileInfo{Name: "a", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(1)},
-		protocol.FileInfo{Name: "b", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(2)},
-		protocol.FileInfo{Name: "c", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(3)},
-		protocol.FileInfo{Name: "d", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(4)},
+		protocol.FileInfo{Name: "a", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(1)},
+		protocol.FileInfo{Name: "b", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(2)},
+		protocol.FileInfo{Name: "c", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(3)},
+		protocol.FileInfo{Name: "d", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(4)},
 		// A longer name is more realistic and causes more allocations
-		protocol.FileInfo{Name: "zajksdhaskjdh/askjdhaskjdashkajshd/kasjdhaskjdhaskdjhaskdjash/dkjashdaksjdhaskdjahskdjh", Version: protocol.Vector{{ID: myID, Value: 1000}}, Blocks: genBlocks(8)},
+		protocol.FileInfo{Name: "zajksdhaskjdh/askjdhaskjdashkajshd/kasjdhaskjdhaskdjhaskdjash/dkjashdaksjdhaskdjahskdjh", Version: protocol.Vector{Counters: []protocol.Counter{{ID: myID, Value: 1000}}}, Blocks: genBlocks(8)},
 	}
 
 	ldb, err := db.Open("testdata/benchmarkupdate.db")

lib/db/structs.go

@@ -0,0 +1,57 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+//go:generate go run ../../script/protofmt.go structs.proto
+//go:generate protoc --proto_path=../../../../../:../../../../gogo/protobuf/protobuf:. --gogofast_out=. structs.proto
+
+package db
+
+import (
+	"fmt"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+func (f FileInfoTruncated) String() string {
+	return fmt.Sprintf("File{Name:%q, Permissions:0%o, Modified:%d, Version:%v, Length:%d, Deleted:%v, Invalid:%v, NoPermissions:%v}",
+		f.Name, f.Permissions, f.Modified, f.Version, f.Size, f.Deleted, f.Invalid, f.NoPermissions)
+}
+
+func (f FileInfoTruncated) IsDeleted() bool {
+	return f.Deleted
+}
+
+func (f FileInfoTruncated) IsInvalid() bool {
+	return f.Invalid
+}
+
+func (f FileInfoTruncated) IsDirectory() bool {
+	return f.Type == protocol.FileInfoTypeDirectory
+}
+
+func (f FileInfoTruncated) IsSymlink() bool {
+	switch f.Type {
+	case protocol.FileInfoTypeSymlinkDirectory, protocol.FileInfoTypeSymlinkFile, protocol.FileInfoTypeSymlinkUnknown:
+		return true
+	default:
+		return false
+	}
+}
+
+func (f FileInfoTruncated) HasPermissionBits() bool {
+	return !f.NoPermissions
+}
+
+func (f FileInfoTruncated) FileSize() int64 {
+	if f.IsDirectory() || f.IsDeleted() {
+		return 128
+	}
+	return f.Size
+}
+
+func (f FileInfoTruncated) FileName() string {
+	return f.Name
+}

lib/db/structs.pb.go

@@ -0,0 +1,914 @@
+// Code generated by protoc-gen-gogo.
+// source: structs.proto
+// DO NOT EDIT!
+
+/*
+	Package db is a generated protocol buffer package.
+
+	It is generated from these files:
+		structs.proto
+
+	It has these top-level messages:
+		FileVersion
+		VersionList
+		FileInfoTruncated
+*/
+package db
+
+import proto "github.com/gogo/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import _ "github.com/gogo/protobuf/gogoproto"
+import protocol "github.com/syncthing/syncthing/lib/protocol"
+
+import io "io"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+const _ = proto.GoGoProtoPackageIsVersion1
+
+type FileVersion struct {
+	Version protocol.Vector `protobuf:"bytes,1,opt,name=version" json:"version"`
+	Device  []byte          `protobuf:"bytes,2,opt,name=device,proto3" json:"device,omitempty"`
+}
+
+func (m *FileVersion) Reset()                    { *m = FileVersion{} }
+func (m *FileVersion) String() string            { return proto.CompactTextString(m) }
+func (*FileVersion) ProtoMessage()               {}
+func (*FileVersion) Descriptor() ([]byte, []int) { return fileDescriptorStructs, []int{0} }
+
+type VersionList struct {
+	Versions []FileVersion `protobuf:"bytes,1,rep,name=versions" json:"versions"`
+}
+
+func (m *VersionList) Reset()                    { *m = VersionList{} }
+func (*VersionList) ProtoMessage()               {}
+func (*VersionList) Descriptor() ([]byte, []int) { return fileDescriptorStructs, []int{1} }
+
+// Must be the same as FileInfo but without the blocks field
+type FileInfoTruncated struct {
+	Name          string                `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	Type          protocol.FileInfoType `protobuf:"varint,2,opt,name=type,proto3,enum=protocol.FileInfoType" json:"type,omitempty"`
+	Size          int64                 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"`
+	Permissions   uint32                `protobuf:"varint,4,opt,name=permissions,proto3" json:"permissions,omitempty"`
+	Modified      int64                 `protobuf:"varint,5,opt,name=modified,proto3" json:"modified,omitempty"`
+	Deleted       bool                  `protobuf:"varint,6,opt,name=deleted,proto3" json:"deleted,omitempty"`
+	Invalid       bool                  `protobuf:"varint,7,opt,name=invalid,proto3" json:"invalid,omitempty"`
+	NoPermissions bool                  `protobuf:"varint,8,opt,name=no_permissions,json=noPermissions,proto3" json:"no_permissions,omitempty"`
+	Version       protocol.Vector       `protobuf:"bytes,9,opt,name=version" json:"version"`
+	LocalVersion  int64                 `protobuf:"varint,10,opt,name=local_version,json=localVersion,proto3" json:"local_version,omitempty"`
+}
+
+func (m *FileInfoTruncated) Reset()                    { *m = FileInfoTruncated{} }
+func (*FileInfoTruncated) ProtoMessage()               {}
+func (*FileInfoTruncated) Descriptor() ([]byte, []int) { return fileDescriptorStructs, []int{2} }
+
+func init() {
+	proto.RegisterType((*FileVersion)(nil), "db.FileVersion")
+	proto.RegisterType((*VersionList)(nil), "db.VersionList")
+	proto.RegisterType((*FileInfoTruncated)(nil), "db.FileInfoTruncated")
+}
+func (m *FileVersion) Marshal() (data []byte, err error) {
+	size := m.ProtoSize()
+	data = make([]byte, size)
+	n, err := m.MarshalTo(data)
+	if err != nil {
+		return nil, err
+	}
+	return data[:n], nil
+}
+
+func (m *FileVersion) MarshalTo(data []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	data[i] = 0xa
+	i++
+	i = encodeVarintStructs(data, i, uint64(m.Version.ProtoSize()))
+	n1, err := m.Version.MarshalTo(data[i:])
+	if err != nil {
+		return 0, err
+	}
+	i += n1
+	if len(m.Device) > 0 {
+		data[i] = 0x12
+		i++
+		i = encodeVarintStructs(data, i, uint64(len(m.Device)))
+		i += copy(data[i:], m.Device)
+	}
+	return i, nil
+}
+
+func (m *VersionList) Marshal() (data []byte, err error) {
+	size := m.ProtoSize()
+	data = make([]byte, size)
+	n, err := m.MarshalTo(data)
+	if err != nil {
+		return nil, err
+	}
+	return data[:n], nil
+}
+
+func (m *VersionList) MarshalTo(data []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.Versions) > 0 {
+		for _, msg := range m.Versions {
+			data[i] = 0xa
+			i++
+			i = encodeVarintStructs(data, i, uint64(msg.ProtoSize()))
+			n, err := msg.MarshalTo(data[i:])
+			if err != nil {
+				return 0, err
+			}
+			i += n
+		}
+	}
+	return i, nil
+}
+
+func (m *FileInfoTruncated) Marshal() (data []byte, err error) {
+	size := m.ProtoSize()
+	data = make([]byte, size)
+	n, err := m.MarshalTo(data)
+	if err != nil {
+		return nil, err
+	}
+	return data[:n], nil
+}
+
+func (m *FileInfoTruncated) MarshalTo(data []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.Name) > 0 {
+		data[i] = 0xa
+		i++
+		i = encodeVarintStructs(data, i, uint64(len(m.Name)))
+		i += copy(data[i:], m.Name)
+	}
+	if m.Type != 0 {
+		data[i] = 0x10
+		i++
+		i = encodeVarintStructs(data, i, uint64(m.Type))
+	}
+	if m.Size != 0 {
+		data[i] = 0x18
+		i++
+		i = encodeVarintStructs(data, i, uint64(m.Size))
+	}
+	if m.Permissions != 0 {
+		data[i] = 0x20
+		i++
+		i = encodeVarintStructs(data, i, uint64(m.Permissions))
+	}
+	if m.Modified != 0 {
+		data[i] = 0x28
+		i++
+		i = encodeVarintStructs(data, i, uint64(m.Modified))
+	}
+	if m.Deleted {
+		data[i] = 0x30
+		i++
+		if m.Deleted {
+			data[i] = 1
+		} else {
+			data[i] = 0
+		}
+		i++
+	}
+	if m.Invalid {
+		data[i] = 0x38
+		i++
+		if m.Invalid {
+			data[i] = 1
+		} else {
+			data[i] = 0
+		}
+		i++
+	}
+	if m.NoPermissions {
+		data[i] = 0x40
+		i++
+		if m.NoPermissions {
+			data[i] = 1
+		} else {
+			data[i] = 0
+		}
+		i++
+	}
+	data[i] = 0x4a
+	i++
+	i = encodeVarintStructs(data, i, uint64(m.Version.ProtoSize()))
+	n2, err := m.Version.MarshalTo(data[i:])
+	if err != nil {
+		return 0, err
+	}
+	i += n2
+	if m.LocalVersion != 0 {
+		data[i] = 0x50
+		i++
+		i = encodeVarintStructs(data, i, uint64(m.LocalVersion))
+	}
+	return i, nil
+}
+
+func encodeFixed64Structs(data []byte, offset int, v uint64) int {
+	data[offset] = uint8(v)
+	data[offset+1] = uint8(v >> 8)
+	data[offset+2] = uint8(v >> 16)
+	data[offset+3] = uint8(v >> 24)
+	data[offset+4] = uint8(v >> 32)
+	data[offset+5] = uint8(v >> 40)
+	data[offset+6] = uint8(v >> 48)
+	data[offset+7] = uint8(v >> 56)
+	return offset + 8
+}
+func encodeFixed32Structs(data []byte, offset int, v uint32) int {
+	data[offset] = uint8(v)
+	data[offset+1] = uint8(v >> 8)
+	data[offset+2] = uint8(v >> 16)
+	data[offset+3] = uint8(v >> 24)
+	return offset + 4
+}
+func encodeVarintStructs(data []byte, offset int, v uint64) int {
+	for v >= 1<<7 {
+		data[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	data[offset] = uint8(v)
+	return offset + 1
+}
+func (m *FileVersion) ProtoSize() (n int) {
+	var l int
+	_ = l
+	l = m.Version.ProtoSize()
+	n += 1 + l + sovStructs(uint64(l))
+	l = len(m.Device)
+	if l > 0 {
+		n += 1 + l + sovStructs(uint64(l))
+	}
+	return n
+}
+
+func (m *VersionList) ProtoSize() (n int) {
+	var l int
+	_ = l
+	if len(m.Versions) > 0 {
+		for _, e := range m.Versions {
+			l = e.ProtoSize()
+			n += 1 + l + sovStructs(uint64(l))
+		}
+	}
+	return n
+}
+
+func (m *FileInfoTruncated) ProtoSize() (n int) {
+	var l int
+	_ = l
+	l = len(m.Name)
+	if l > 0 {
+		n += 1 + l + sovStructs(uint64(l))
+	}
+	if m.Type != 0 {
+		n += 1 + sovStructs(uint64(m.Type))
+	}
+	if m.Size != 0 {
+		n += 1 + sovStructs(uint64(m.Size))
+	}
+	if m.Permissions != 0 {
+		n += 1 + sovStructs(uint64(m.Permissions))
+	}
+	if m.Modified != 0 {
+		n += 1 + sovStructs(uint64(m.Modified))
+	}
+	if m.Deleted {
+		n += 2
+	}
+	if m.Invalid {
+		n += 2
+	}
+	if m.NoPermissions {
+		n += 2
+	}
+	l = m.Version.ProtoSize()
+	n += 1 + l + sovStructs(uint64(l))
+	if m.LocalVersion != 0 {
+		n += 1 + sovStructs(uint64(m.LocalVersion))
+	}
+	return n
+}
+
+func sovStructs(x uint64) (n int) {
+	for {
+		n++
+		x >>= 7
+		if x == 0 {
+			break
+		}
+	}
+	return n
+}
+func sozStructs(x uint64) (n int) {
+	return sovStructs(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (m *FileVersion) Unmarshal(data []byte) error {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowStructs
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: FileVersion: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: FileVersion: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if err := m.Version.Unmarshal(data[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Device", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				byteLen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Device = append(m.Device[:0], data[iNdEx:postIndex]...)
+			if m.Device == nil {
+				m.Device = []byte{}
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipStructs(data[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *VersionList) Unmarshal(data []byte) error {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowStructs
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: VersionList: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: VersionList: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Versions", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Versions = append(m.Versions, FileVersion{})
+			if err := m.Versions[len(m.Versions)-1].Unmarshal(data[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		default:
+			iNdEx = preIndex
+			skippy, err := skipStructs(data[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func (m *FileInfoTruncated) Unmarshal(data []byte) error {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowStructs
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: FileInfoTruncated: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: FileInfoTruncated: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Name = string(data[iNdEx:postIndex])
+			iNdEx = postIndex
+		case 2:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType)
+			}
+			m.Type = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.Type |= (protocol.FileInfoType(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Size", wireType)
+			}
+			m.Size = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.Size |= (int64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 4:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Permissions", wireType)
+			}
+			m.Permissions = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.Permissions |= (uint32(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 5:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Modified", wireType)
+			}
+			m.Modified = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.Modified |= (int64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 6:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Deleted", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				v |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.Deleted = bool(v != 0)
+		case 7:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Invalid", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				v |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.Invalid = bool(v != 0)
+		case 8:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field NoPermissions", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				v |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.NoPermissions = bool(v != 0)
+		case 9:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType)
+			}
+			var msglen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				msglen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if msglen < 0 {
+				return ErrInvalidLengthStructs
+			}
+			postIndex := iNdEx + msglen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			if err := m.Version.Unmarshal(data[iNdEx:postIndex]); err != nil {
+				return err
+			}
+			iNdEx = postIndex
+		case 10:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field LocalVersion", wireType)
+			}
+			m.LocalVersion = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.LocalVersion |= (int64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipStructs(data[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthStructs
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func skipStructs(data []byte) (n int, err error) {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return 0, ErrIntOverflowStructs
+			}
+			if iNdEx >= l {
+				return 0, io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		wireType := int(wire & 0x7)
+		switch wireType {
+		case 0:
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				iNdEx++
+				if data[iNdEx-1] < 0x80 {
+					break
+				}
+			}
+			return iNdEx, nil
+		case 1:
+			iNdEx += 8
+			return iNdEx, nil
+		case 2:
+			var length int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowStructs
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				length |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			iNdEx += length
+			if length < 0 {
+				return 0, ErrInvalidLengthStructs
+			}
+			return iNdEx, nil
+		case 3:
+			for {
+				var innerWire uint64
+				var start int = iNdEx
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return 0, ErrIntOverflowStructs
+					}
+					if iNdEx >= l {
+						return 0, io.ErrUnexpectedEOF
+					}
+					b := data[iNdEx]
+					iNdEx++
+					innerWire |= (uint64(b) & 0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				innerWireType := int(innerWire & 0x7)
+				if innerWireType == 4 {
+					break
+				}
+				next, err := skipStructs(data[start:])
+				if err != nil {
+					return 0, err
+				}
+				iNdEx = start + next
+			}
+			return iNdEx, nil
+		case 4:
+			return iNdEx, nil
+		case 5:
+			iNdEx += 4
+			return iNdEx, nil
+		default:
+			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+		}
+	}
+	panic("unreachable")
+}
+
+var (
+	ErrInvalidLengthStructs = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowStructs   = fmt.Errorf("proto: integer overflow")
+)
+
+var fileDescriptorStructs = []byte{
+	// 401 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x51, 0x4f, 0xcb, 0xd3, 0x30,
+	0x1c, 0x6e, 0xb7, 0xba, 0xf5, 0x4d, 0xdf, 0x4e, 0x0d, 0x32, 0xca, 0x0e, 0xdd, 0x98, 0x08, 0x22,
+	0xd8, 0xe9, 0xc4, 0x8b, 0xc7, 0x1d, 0x06, 0x82, 0x07, 0x29, 0x32, 0x8f, 0xa3, 0x4d, 0xb2, 0x2e,
+	0xd0, 0x26, 0xa5, 0x49, 0x07, 0xf3, 0x93, 0x78, 0xdc, 0xc7, 0xd9, 0xd1, 0x2f, 0xa0, 0xe8, 0xfc,
+	0x22, 0x66, 0x49, 0x3b, 0x7b, 0x7c, 0x0f, 0x81, 0xdf, 0x93, 0xe7, 0xcf, 0xef, 0x21, 0x01, 0xbe,
+	0x90, 0x55, 0x8d, 0xa4, 0x88, 0xca, 0x8a, 0x4b, 0x0e, 0x7b, 0x38, 0x9d, 0xbc, 0xce, 0xa8, 0xdc,
+	0xd7, 0x69, 0x84, 0x78, 0xb1, 0xc8, 0x78, 0xc6, 0x17, 0x9a, 0x4a, 0xeb, 0x9d, 0x46, 0x1a, 0xe8,
+	0xc9, 0x58, 0x26, 0xef, 0x3b, 0x72, 0x71, 0x64, 0x48, 0xee, 0x29, 0xcb, 0x3a, 0x53, 0x4e, 0x53,
+	0x93, 0x80, 0x78, 0xbe, 0x48, 0x49, 0x69, 0x6c, 0xf3, 0xaf, 0xc0, 0x5b, 0xd3, 0x9c, 0x6c, 0x48,
+	0x25, 0x28, 0x67, 0xf0, 0x0d, 0x18, 0x1e, 0xcc, 0x18, 0xd8, 0x33, 0xfb, 0xa5, 0xb7, 0x7c, 0x12,
+	0xb5, 0xa6, 0x68, 0x43, 0x90, 0xe4, 0xd5, 0xca, 0x39, 0xff, 0x9a, 0x5a, 0x71, 0x2b, 0x83, 0x63,
+	0x30, 0xc0, 0xe4, 0x40, 0x11, 0x09, 0x7a, 0xca, 0x70, 0x1f, 0x37, 0x68, 0xbe, 0x06, 0x5e, 0x13,
+	0xfa, 0x89, 0x0a, 0x09, 0xdf, 0x02, 0xb7, 0x71, 0x08, 0x95, 0xdc, 0x57, 0xc9, 0x8f, 0x23, 0x9c,
+	0x46, 0x9d, 0xdd, 0x4d, 0xf0, 0x4d, 0xf6, 0xc1, 0xf9, 0x7e, 0x9a, 0x5a, 0xf3, 0x9f, 0x3d, 0xf0,
+	0xf4, 0xaa, 0xfa, 0xc8, 0x76, 0xfc, 0x4b, 0x55, 0x33, 0x94, 0x48, 0x82, 0x21, 0x04, 0x0e, 0x4b,
+	0x0a, 0xa2, 0x4b, 0xde, 0xc5, 0x7a, 0x86, 0xaf, 0x80, 0x23, 0x8f, 0xa5, 0xe9, 0x31, 0x5a, 0x8e,
+	0xff, 0x17, 0xbf, 0xd9, 0x15, 0x1b, 0x6b, 0xcd, 0xd5, 0x2f, 0xe8, 0x37, 0x12, 0xf4, 0x95, 0xb6,
+	0x1f, 0xeb, 0x19, 0xce, 0x80, 0x57, 0x92, 0xaa, 0xa0, 0xc2, 0xb4, 0x74, 0x14, 0xe5, 0xc7, 0xdd,
+	0x2b, 0x38, 0x01, 0x6e, 0xc1, 0x31, 0xdd, 0x51, 0x82, 0x83, 0x47, 0xda, 0x79, 0xc3, 0x30, 0x00,
+	0x43, 0x4c, 0x72, 0xa2, 0xca, 0x05, 0x03, 0x45, 0xb9, 0x71, 0x0b, 0xaf, 0x0c, 0x65, 0x87, 0x24,
+	0xa7, 0x38, 0x18, 0x1a, 0xa6, 0x81, 0xf0, 0x05, 0x18, 0x31, 0xbe, 0xed, 0x2e, 0x75, 0xb5, 0xc0,
+	0x67, 0xfc, 0x73, 0x67, 0x6d, 0xe7, 0x53, 0xee, 0x1e, 0xf6, 0x29, 0xcf, 0x81, 0x9f, 0x73, 0x94,
+	0xe4, 0xdb, 0xd6, 0x07, 0x74, 0xdb, 0x7b, 0x7d, 0xd9, 0xbc, 0xb7, 0x79, 0xdf, 0xd5, 0xb3, 0xf3,
+	0x9f, 0xd0, 0x3a, 0x5f, 0x42, 0xfb, 0x87, 0x3a, 0xbf, 0x2f, 0xa1, 0x75, 0xfa, 0x1b, 0xda, 0xe9,
+	0x40, 0x2f, 0x78, 0xf7, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xe4, 0xb1, 0x7f, 0x07, 0x98, 0x02, 0x00,
+	0x00,
+}

lib/db/structs.proto

@@ -0,0 +1,35 @@
+syntax = "proto3";
+
+package db;
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+import "github.com/syncthing/syncthing/lib/protocol/bep.proto";
+
+option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.sizer_all) = false;
+option (gogoproto.protosizer_all) = true;
+
+message FileVersion {
+    protocol.Vector version = 1 [(gogoproto.nullable) = false];
+    bytes           device  = 2;
+}
+
+message VersionList {
+    option (gogoproto.goproto_stringer) = false;
+    repeated FileVersion versions = 1 [(gogoproto.nullable) = false];
+}
+
+// Must be the same as FileInfo but without the blocks field
+message FileInfoTruncated {
+    option (gogoproto.goproto_stringer) = false;
+    string                name           = 1;
+    protocol.FileInfoType type           = 2;
+    int64                 size           = 3;
+    uint32                permissions    = 4;
+    int64                 modified       = 5;
+    bool                  deleted        = 6;
+    bool                  invalid        = 7;
+    bool                  no_permissions = 8;
+    protocol.Vector       version        = 9 [(gogoproto.nullable) = false];
+    int64                 local_version  = 10;
+}

lib/db/truncated.go

@@ -1,52 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package db
-
-import (
-	"github.com/calmh/xdr"
-	"github.com/syncthing/syncthing/lib/protocol"
-)
-
-type FileInfoTruncated struct {
-	protocol.FileInfo
-}
-
-func (o *FileInfoTruncated) UnmarshalXDR(bs []byte) error {
-	return o.UnmarshalXDRFrom(&xdr.Unmarshaller{Data: bs})
-}
-
-func (o *FileInfoTruncated) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	o.Name = u.UnmarshalStringMax(8192)
-	o.Flags = u.UnmarshalUint32()
-	o.Modified = int64(u.UnmarshalUint64())
-	(&o.Version).UnmarshalXDRFrom(u)
-	o.LocalVersion = int64(u.UnmarshalUint64())
-	_BlocksSize := int(u.UnmarshalUint32())
-	if _BlocksSize < 0 {
-		return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 10000000)
-	} else if _BlocksSize == 0 {
-		o.Blocks = nil
-	} else {
-		if _BlocksSize > 10000000 {
-			return xdr.ElementSizeExceeded("Blocks", _BlocksSize, 10000000)
-		}
-		for i := 0; i < _BlocksSize; i++ {
-			size := int64(u.UnmarshalUint32())
-			o.CachedSize += size
-			u.UnmarshalBytes()
-		}
-	}
-
-	return u.Error
-}
-
-func BlocksToSize(num int) int64 {
-	if num < 2 {
-		return protocol.BlockSize / 2
-	}
-	return int64(num-1)*protocol.BlockSize + protocol.BlockSize/2
-}

lib/discover/discover.go

@@ -26,6 +26,7 @@ type CacheEntry struct {
 	when       time.Time // When did we get the result
 	found      bool      // Is it a success (cacheTime applies) or a failure (negCacheTime applies)?
 	validUntil time.Time // Validity time, overrides normal calculation
+	instanceID int64     // for local discovery, the instance ID (random on each restart)
 }
 
 // A FinderService is a Finder that has background activity and must be run as

lib/discover/local.go

@@ -4,10 +4,14 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this file,
 // You can obtain one at http://mozilla.org/MPL/2.0/.
 
+//go:generate go run ../../script/protofmt.go local.proto
+//go:generate protoc --proto_path=../../../../../:../../../../gogo/protobuf/protobuf:. --gogofast_out=. local.proto
+
 package discover
 
 import (
 	"bytes"
+	"encoding/binary"
 	"encoding/hex"
 	"io"
 	"net"
@@ -18,6 +22,7 @@ import (
 	"github.com/syncthing/syncthing/lib/beacon"
 	"github.com/syncthing/syncthing/lib/events"
 	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/rand"
 	"github.com/thejerf/suture"
 )
 
@@ -38,6 +43,8 @@ type localClient struct {
 const (
 	BroadcastInterval = 30 * time.Second
 	CacheLifeTime     = 3 * BroadcastInterval
+	Magic             = uint32(0x2EA7D90B) // same as in BEP
+	v13Magic          = uint32(0x7D79BC40) // previous version
 )
 
 func NewLocal(id protocol.DeviceID, addr string, addrList AddressLister) (FinderService, error) {
@@ -107,25 +114,20 @@ func (c *localClient) Error() error {
 }
 
 func (c *localClient) announcementPkt() Announce {
-	var addrs []Address
-	for _, addr := range c.addrList.AllAddresses() {
-		addrs = append(addrs, Address{
-			URL: addr,
-		})
-	}
-
 	return Announce{
-		Magic: AnnouncementMagic,
-		This: Device{
-			ID:        c.myID[:],
-			Addresses: addrs,
-		},
+		ID:         c.myID[:],
+		Addresses:  c.addrList.AllAddresses(),
+		InstanceID: rand.Int63(),
 	}
 }
 
 func (c *localClient) sendLocalAnnouncements() {
+	msg := make([]byte, 4)
+	binary.BigEndian.PutUint32(msg, Magic)
+
 	var pkt = c.announcementPkt()
-	msg := pkt.MustMarshalXDR()
+	bs, _ := pkt.Marshal()
+	msg = append(msg, bs...)
 
 	for {
 		c.beacon.Send(msg)
@@ -138,26 +140,44 @@ func (c *localClient) sendLocalAnnouncements() {
 }
 
 func (c *localClient) recvAnnouncements(b beacon.Interface) {
+	warnedAbout := make(map[string]bool)
 	for {
 		buf, addr := b.Recv()
+		if len(buf) < 4 {
+			l.Debugf("discover: short packet from %s")
+			continue
+		}
+
+		magic := binary.BigEndian.Uint32(buf)
+		switch magic {
+		case Magic:
+			// All good
+
+		case v13Magic:
+			// Old version
+			if !warnedAbout[addr.String()] {
+				l.Warnf("Incompatible (v0.13) local discovery packet from %v - upgrade that device to connect", addr)
+				warnedAbout[addr.String()] = true
+			}
+			continue
+
+		default:
+			l.Debugf("discover: Incorrect magic %x from %s", magic, addr)
+			continue
+		}
 
 		var pkt Announce
-		err := pkt.UnmarshalXDR(buf)
+		err := pkt.Unmarshal(buf[4:])
 		if err != nil && err != io.EOF {
 			l.Debugf("discover: Failed to unmarshal local announcement from %s:\n%s", addr, hex.Dump(buf))
 			continue
 		}
 
-		if pkt.Magic != AnnouncementMagic {
-			l.Debugf("discover: Incorrect magic from %s: %s != %s", addr, pkt.Magic, AnnouncementMagic)
-			continue
-		}
-
-		l.Debugf("discover: Received local announcement from %s for %s", addr, protocol.DeviceIDFromBytes(pkt.This.ID))
+		l.Debugf("discover: Received local announcement from %s for %s", addr, protocol.DeviceIDFromBytes(pkt.ID))
 
 		var newDevice bool
-		if !bytes.Equal(pkt.This.ID, c.myID[:]) {
-			newDevice = c.registerDevice(addr, pkt.This)
+		if !bytes.Equal(pkt.ID, c.myID[:]) {
+			newDevice = c.registerDevice(addr, pkt)
 		}
 
 		if newDevice {
@@ -171,14 +191,16 @@ func (c *localClient) recvAnnouncements(b beacon.Interface) {
 	}
 }
 
-func (c *localClient) registerDevice(src net.Addr, device Device) bool {
+func (c *localClient) registerDevice(src net.Addr, device Announce) bool {
 	var id protocol.DeviceID
 	copy(id[:], device.ID)
 
 	// Remember whether we already had a valid cache entry for this device.
+	// If the instance ID has changed the remote device has restarted since
+	// we last heard from it, so we should treat it as a new device.
 
 	ce, existsAlready := c.Get(id)
-	isNewDevice := !existsAlready || time.Since(ce.when) > CacheLifeTime
+	isNewDevice := !existsAlready || time.Since(ce.when) > CacheLifeTime || ce.instanceID != device.InstanceID
 
 	// Any empty or unspecified addresses should be set to the source address
 	// of the announcement. We also skip any addresses we can't parse.
@@ -186,7 +208,7 @@ func (c *localClient) registerDevice(src net.Addr, device Device) bool {
 	l.Debugln("discover: Registering addresses for", id)
 	var validAddresses []string
 	for _, addr := range device.Addresses {
-		u, err := url.Parse(addr.URL)
+		u, err := url.Parse(addr)
 		if err != nil {
 			continue
 		}
@@ -197,6 +219,21 @@ func (c *localClient) registerDevice(src net.Addr, device Device) bool {
 		}
 
 		if len(tcpAddr.IP) == 0 || tcpAddr.IP.IsUnspecified() {
+			srcAddr, err := net.ResolveTCPAddr("tcp", src.String())
+			if err != nil {
+				continue
+			}
+
+			// Do not use IPv6 source address if requested scheme is tcp4
+			if u.Scheme == "tcp4" && srcAddr.IP.To4() == nil {
+				continue
+			}
+
+			// Do not use IPv4 source address if requested scheme is tcp6
+			if u.Scheme == "tcp6" && srcAddr.IP.To4() != nil {
+				continue
+			}
+
 			host, _, err := net.SplitHostPort(src.String())
 			if err != nil {
 				continue
@@ -204,17 +241,18 @@ func (c *localClient) registerDevice(src net.Addr, device Device) bool {
 			u.Host = net.JoinHostPort(host, strconv.Itoa(tcpAddr.Port))
 			l.Debugf("discover: Reconstructed URL is %#v", u)
 			validAddresses = append(validAddresses, u.String())
-			l.Debugf("discover: Replaced address %v in %s to get %s", tcpAddr.IP, addr.URL, u.String())
+			l.Debugf("discover: Replaced address %v in %s to get %s", tcpAddr.IP, addr, u.String())
 		} else {
-			validAddresses = append(validAddresses, addr.URL)
-			l.Debugf("discover: Accepted address %s verbatim", addr.URL)
+			validAddresses = append(validAddresses, addr)
+			l.Debugf("discover: Accepted address %s verbatim", addr)
 		}
 	}
 
 	c.Set(id, CacheEntry{
-		Addresses: validAddresses,
-		when:      time.Now(),
-		found:     true,
+		Addresses:  validAddresses,
+		when:       time.Now(),
+		found:      true,
+		instanceID: device.InstanceID,
 	})
 
 	if isNewDevice {

lib/discover/local.pb.go

@@ -0,0 +1,398 @@
+// Code generated by protoc-gen-gogo.
+// source: local.proto
+// DO NOT EDIT!
+
+/*
+	Package discover is a generated protocol buffer package.
+
+	It is generated from these files:
+		local.proto
+
+	It has these top-level messages:
+		Announce
+*/
+package discover
+
+import proto "github.com/gogo/protobuf/proto"
+import fmt "fmt"
+import math "math"
+import _ "github.com/gogo/protobuf/gogoproto"
+
+import io "io"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+const _ = proto.GoGoProtoPackageIsVersion1
+
+type Announce struct {
+	ID         []byte   `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
+	Addresses  []string `protobuf:"bytes,2,rep,name=addresses" json:"addresses,omitempty"`
+	InstanceID int64    `protobuf:"varint,3,opt,name=instance_id,json=instanceId,proto3" json:"instance_id,omitempty"`
+}
+
+func (m *Announce) Reset()                    { *m = Announce{} }
+func (m *Announce) String() string            { return proto.CompactTextString(m) }
+func (*Announce) ProtoMessage()               {}
+func (*Announce) Descriptor() ([]byte, []int) { return fileDescriptorLocal, []int{0} }
+
+func init() {
+	proto.RegisterType((*Announce)(nil), "discover.Announce")
+}
+func (m *Announce) Marshal() (data []byte, err error) {
+	size := m.ProtoSize()
+	data = make([]byte, size)
+	n, err := m.MarshalTo(data)
+	if err != nil {
+		return nil, err
+	}
+	return data[:n], nil
+}
+
+func (m *Announce) MarshalTo(data []byte) (int, error) {
+	var i int
+	_ = i
+	var l int
+	_ = l
+	if len(m.ID) > 0 {
+		data[i] = 0xa
+		i++
+		i = encodeVarintLocal(data, i, uint64(len(m.ID)))
+		i += copy(data[i:], m.ID)
+	}
+	if len(m.Addresses) > 0 {
+		for _, s := range m.Addresses {
+			data[i] = 0x12
+			i++
+			l = len(s)
+			for l >= 1<<7 {
+				data[i] = uint8(uint64(l)&0x7f | 0x80)
+				l >>= 7
+				i++
+			}
+			data[i] = uint8(l)
+			i++
+			i += copy(data[i:], s)
+		}
+	}
+	if m.InstanceID != 0 {
+		data[i] = 0x18
+		i++
+		i = encodeVarintLocal(data, i, uint64(m.InstanceID))
+	}
+	return i, nil
+}
+
+func encodeFixed64Local(data []byte, offset int, v uint64) int {
+	data[offset] = uint8(v)
+	data[offset+1] = uint8(v >> 8)
+	data[offset+2] = uint8(v >> 16)
+	data[offset+3] = uint8(v >> 24)
+	data[offset+4] = uint8(v >> 32)
+	data[offset+5] = uint8(v >> 40)
+	data[offset+6] = uint8(v >> 48)
+	data[offset+7] = uint8(v >> 56)
+	return offset + 8
+}
+func encodeFixed32Local(data []byte, offset int, v uint32) int {
+	data[offset] = uint8(v)
+	data[offset+1] = uint8(v >> 8)
+	data[offset+2] = uint8(v >> 16)
+	data[offset+3] = uint8(v >> 24)
+	return offset + 4
+}
+func encodeVarintLocal(data []byte, offset int, v uint64) int {
+	for v >= 1<<7 {
+		data[offset] = uint8(v&0x7f | 0x80)
+		v >>= 7
+		offset++
+	}
+	data[offset] = uint8(v)
+	return offset + 1
+}
+func (m *Announce) ProtoSize() (n int) {
+	var l int
+	_ = l
+	l = len(m.ID)
+	if l > 0 {
+		n += 1 + l + sovLocal(uint64(l))
+	}
+	if len(m.Addresses) > 0 {
+		for _, s := range m.Addresses {
+			l = len(s)
+			n += 1 + l + sovLocal(uint64(l))
+		}
+	}
+	if m.InstanceID != 0 {
+		n += 1 + sovLocal(uint64(m.InstanceID))
+	}
+	return n
+}
+
+func sovLocal(x uint64) (n int) {
+	for {
+		n++
+		x >>= 7
+		if x == 0 {
+			break
+		}
+	}
+	return n
+}
+func sozLocal(x uint64) (n int) {
+	return sovLocal(uint64((x << 1) ^ uint64((int64(x) >> 63))))
+}
+func (m *Announce) Unmarshal(data []byte) error {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		preIndex := iNdEx
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return ErrIntOverflowLocal
+			}
+			if iNdEx >= l {
+				return io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		fieldNum := int32(wire >> 3)
+		wireType := int(wire & 0x7)
+		if wireType == 4 {
+			return fmt.Errorf("proto: Announce: wiretype end group for non-group")
+		}
+		if fieldNum <= 0 {
+			return fmt.Errorf("proto: Announce: illegal tag %d (wire type %d)", fieldNum, wire)
+		}
+		switch fieldNum {
+		case 1:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType)
+			}
+			var byteLen int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowLocal
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				byteLen |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			if byteLen < 0 {
+				return ErrInvalidLengthLocal
+			}
+			postIndex := iNdEx + byteLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.ID = append(m.ID[:0], data[iNdEx:postIndex]...)
+			if m.ID == nil {
+				m.ID = []byte{}
+			}
+			iNdEx = postIndex
+		case 2:
+			if wireType != 2 {
+				return fmt.Errorf("proto: wrong wireType = %d for field Addresses", wireType)
+			}
+			var stringLen uint64
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowLocal
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				stringLen |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			intStringLen := int(stringLen)
+			if intStringLen < 0 {
+				return ErrInvalidLengthLocal
+			}
+			postIndex := iNdEx + intStringLen
+			if postIndex > l {
+				return io.ErrUnexpectedEOF
+			}
+			m.Addresses = append(m.Addresses, string(data[iNdEx:postIndex]))
+			iNdEx = postIndex
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field InstanceID", wireType)
+			}
+			m.InstanceID = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowLocal
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.InstanceID |= (int64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		default:
+			iNdEx = preIndex
+			skippy, err := skipLocal(data[iNdEx:])
+			if err != nil {
+				return err
+			}
+			if skippy < 0 {
+				return ErrInvalidLengthLocal
+			}
+			if (iNdEx + skippy) > l {
+				return io.ErrUnexpectedEOF
+			}
+			iNdEx += skippy
+		}
+	}
+
+	if iNdEx > l {
+		return io.ErrUnexpectedEOF
+	}
+	return nil
+}
+func skipLocal(data []byte) (n int, err error) {
+	l := len(data)
+	iNdEx := 0
+	for iNdEx < l {
+		var wire uint64
+		for shift := uint(0); ; shift += 7 {
+			if shift >= 64 {
+				return 0, ErrIntOverflowLocal
+			}
+			if iNdEx >= l {
+				return 0, io.ErrUnexpectedEOF
+			}
+			b := data[iNdEx]
+			iNdEx++
+			wire |= (uint64(b) & 0x7F) << shift
+			if b < 0x80 {
+				break
+			}
+		}
+		wireType := int(wire & 0x7)
+		switch wireType {
+		case 0:
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowLocal
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				iNdEx++
+				if data[iNdEx-1] < 0x80 {
+					break
+				}
+			}
+			return iNdEx, nil
+		case 1:
+			iNdEx += 8
+			return iNdEx, nil
+		case 2:
+			var length int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return 0, ErrIntOverflowLocal
+				}
+				if iNdEx >= l {
+					return 0, io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				length |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			iNdEx += length
+			if length < 0 {
+				return 0, ErrInvalidLengthLocal
+			}
+			return iNdEx, nil
+		case 3:
+			for {
+				var innerWire uint64
+				var start int = iNdEx
+				for shift := uint(0); ; shift += 7 {
+					if shift >= 64 {
+						return 0, ErrIntOverflowLocal
+					}
+					if iNdEx >= l {
+						return 0, io.ErrUnexpectedEOF
+					}
+					b := data[iNdEx]
+					iNdEx++
+					innerWire |= (uint64(b) & 0x7F) << shift
+					if b < 0x80 {
+						break
+					}
+				}
+				innerWireType := int(innerWire & 0x7)
+				if innerWireType == 4 {
+					break
+				}
+				next, err := skipLocal(data[start:])
+				if err != nil {
+					return 0, err
+				}
+				iNdEx = start + next
+			}
+			return iNdEx, nil
+		case 4:
+			return iNdEx, nil
+		case 5:
+			iNdEx += 4
+			return iNdEx, nil
+		default:
+			return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
+		}
+	}
+	panic("unreachable")
+}
+
+var (
+	ErrInvalidLengthLocal = fmt.Errorf("proto: negative length found during unmarshaling")
+	ErrIntOverflowLocal   = fmt.Errorf("proto: integer overflow")
+)
+
+var fileDescriptorLocal = []byte{
+	// 194 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xce, 0xc9, 0x4f, 0x4e,
+	0xcc, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x48, 0xc9, 0x2c, 0x4e, 0xce, 0x2f, 0x4b,
+	0x2d, 0x92, 0xd2, 0x4d, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf,
+	0x4f, 0xcf, 0xd7, 0x07, 0x2b, 0x48, 0x2a, 0x4d, 0x03, 0xf3, 0xc0, 0x1c, 0x30, 0x0b, 0xa2, 0x51,
+	0xa9, 0x90, 0x8b, 0xc3, 0x31, 0x2f, 0x2f, 0xbf, 0x34, 0x2f, 0x39, 0x55, 0x48, 0x8c, 0x8b, 0x29,
+	0x33, 0x45, 0x82, 0x51, 0x81, 0x51, 0x83, 0xc7, 0x89, 0xed, 0xd1, 0x3d, 0x79, 0x26, 0x4f, 0x97,
+	0x20, 0xa0, 0x88, 0x90, 0x0c, 0x17, 0x67, 0x62, 0x4a, 0x4a, 0x51, 0x6a, 0x71, 0x71, 0x6a, 0xb1,
+	0x04, 0x93, 0x02, 0xb3, 0x06, 0x67, 0x10, 0x42, 0x40, 0x48, 0x9f, 0x8b, 0x3b, 0x33, 0xaf, 0xb8,
+	0x24, 0x11, 0x68, 0x42, 0x3c, 0x50, 0x3b, 0x33, 0x50, 0x3b, 0xb3, 0x13, 0x1f, 0x50, 0x3b, 0x97,
+	0x27, 0x54, 0x18, 0x68, 0x0c, 0x17, 0x4c, 0x89, 0x67, 0x8a, 0x93, 0xc8, 0x89, 0x87, 0x72, 0x0c,
+	0x27, 0x1e, 0xc9, 0x31, 0x5e, 0x00, 0xe2, 0x07, 0x8f, 0xe4, 0x18, 0x16, 0x3c, 0x96, 0x63, 0x4c,
+	0x62, 0x03, 0xbb, 0xc7, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff, 0x91, 0x3f, 0x96, 0x25, 0xd7, 0x00,
+	0x00, 0x00,
+}

lib/discover/local.proto

@@ -0,0 +1,15 @@
+syntax = "proto3";
+
+package discover;
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.sizer_all) = false;
+option (gogoproto.protosizer_all) = true;
+
+message Announce {
+    bytes           id          = 1 [(gogoproto.customname) = "ID"];
+    repeated string addresses   = 2;
+    int64           instance_id = 3 [(gogoproto.customname) = "InstanceID"];
+}

lib/discover/local_test.go

@@ -0,0 +1,81 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package discover
+
+import (
+	"net"
+	"testing"
+
+	"github.com/syncthing/syncthing/lib/protocol"
+)
+
+func TestRandomLocalInstanceID(t *testing.T) {
+	c, err := NewLocal(protocol.LocalDeviceID, ":0", &fakeAddressLister{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	go c.Serve()
+	defer c.Stop()
+
+	lc := c.(*localClient)
+
+	p0 := lc.announcementPkt()
+	p1 := lc.announcementPkt()
+	if p0.InstanceID == p1.InstanceID {
+		t.Error("each generated packet should have a new instance id")
+	}
+}
+
+func TestLocalInstanceIDShouldTriggerNew(t *testing.T) {
+	c, err := NewLocal(protocol.LocalDeviceID, ":0", &fakeAddressLister{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	lc := c.(*localClient)
+	src := &net.UDPAddr{IP: []byte{10, 20, 30, 40}, Port: 50}
+
+	new := lc.registerDevice(src, Announce{
+		ID:         []byte{10, 20, 30, 40, 50, 60, 70, 80, 90},
+		Addresses:  []string{"tcp://0.0.0.0:22000"},
+		InstanceID: 1234567890,
+	})
+
+	if !new {
+		t.Fatal("first register should be new")
+	}
+
+	new = lc.registerDevice(src, Announce{
+		ID:         []byte{10, 20, 30, 40, 50, 60, 70, 80, 90},
+		Addresses:  []string{"tcp://0.0.0.0:22000"},
+		InstanceID: 1234567890,
+	})
+
+	if new {
+		t.Fatal("second register should not be new")
+	}
+
+	new = lc.registerDevice(src, Announce{
+		ID:         []byte{42, 10, 20, 30, 40, 50, 60, 70, 80, 90},
+		Addresses:  []string{"tcp://0.0.0.0:22000"},
+		InstanceID: 1234567890,
+	})
+
+	if !new {
+		t.Fatal("new device ID should be new")
+	}
+
+	new = lc.registerDevice(src, Announce{
+		ID:         []byte{10, 20, 30, 40, 50, 60, 70, 80, 90},
+		Addresses:  []string{"tcp://0.0.0.0:22000"},
+		InstanceID: 91234567890,
+	})
+
+	if !new {
+		t.Fatal("new instance ID should be new")
+	}
+}

lib/discover/localpackets.go

@@ -1,29 +0,0 @@
-// Copyright (C) 2014 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-//go:generate -command genxdr go run ../../vendor/github.com/calmh/xdr/cmd/genxdr/main.go
-//go:generate genxdr -o localpackets_xdr.go localpackets.go
-
-package discover
-
-const (
-	AnnouncementMagic = 0x7D79BC40
-)
-
-type Announce struct {
-	Magic uint32
-	This  Device
-	Extra []Device // max:16
-}
-
-type Device struct {
-	ID        []byte    // max:32
-	Addresses []Address // max:16
-}
-
-type Address struct {
-	URL string // max:2083
-}

lib/discover/localpackets_xdr.go

@@ -1,246 +0,0 @@
-// ************************************************************
-// This file is automatically generated by genxdr. Do not edit.
-// ************************************************************
-
-package discover
-
-import (
-	"github.com/calmh/xdr"
-)
-
-/*
-
-Announce Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                             Magic                             |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                       Device Structure                        \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                        Number of Extra                        |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                Zero or more Device Structures                 \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct Announce {
-	unsigned int Magic;
-	Device This;
-	Device Extra<16>;
-}
-
-*/
-
-func (o Announce) XDRSize() int {
-	return 4 +
-		o.This.XDRSize() +
-		4 + xdr.SizeOfSlice(o.Extra)
-}
-
-func (o Announce) MarshalXDR() ([]byte, error) {
-	buf := make([]byte, o.XDRSize())
-	m := &xdr.Marshaller{Data: buf}
-	return buf, o.MarshalXDRInto(m)
-}
-
-func (o Announce) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o Announce) MarshalXDRInto(m *xdr.Marshaller) error {
-	m.MarshalUint32(o.Magic)
-	if err := o.This.MarshalXDRInto(m); err != nil {
-		return err
-	}
-	if l := len(o.Extra); l > 16 {
-		return xdr.ElementSizeExceeded("Extra", l, 16)
-	}
-	m.MarshalUint32(uint32(len(o.Extra)))
-	for i := range o.Extra {
-		if err := o.Extra[i].MarshalXDRInto(m); err != nil {
-			return err
-		}
-	}
-	return m.Error
-}
-
-func (o *Announce) UnmarshalXDR(bs []byte) error {
-	u := &xdr.Unmarshaller{Data: bs}
-	return o.UnmarshalXDRFrom(u)
-}
-func (o *Announce) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	o.Magic = u.UnmarshalUint32()
-	(&o.This).UnmarshalXDRFrom(u)
-	_ExtraSize := int(u.UnmarshalUint32())
-	if _ExtraSize < 0 {
-		return xdr.ElementSizeExceeded("Extra", _ExtraSize, 16)
-	} else if _ExtraSize == 0 {
-		o.Extra = nil
-	} else {
-		if _ExtraSize > 16 {
-			return xdr.ElementSizeExceeded("Extra", _ExtraSize, 16)
-		}
-		if _ExtraSize <= len(o.Extra) {
-			o.Extra = o.Extra[:_ExtraSize]
-		} else {
-			o.Extra = make([]Device, _ExtraSize)
-		}
-		for i := range o.Extra {
-			(&o.Extra[i]).UnmarshalXDRFrom(u)
-		}
-	}
-	return u.Error
-}
-
-/*
-
-Device Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                   ID (length + padded data)                   \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-|                      Number of Addresses                      |
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                Zero or more Address Structures                \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct Device {
-	opaque ID<32>;
-	Address Addresses<16>;
-}
-
-*/
-
-func (o Device) XDRSize() int {
-	return 4 + len(o.ID) + xdr.Padding(len(o.ID)) +
-		4 + xdr.SizeOfSlice(o.Addresses)
-}
-
-func (o Device) MarshalXDR() ([]byte, error) {
-	buf := make([]byte, o.XDRSize())
-	m := &xdr.Marshaller{Data: buf}
-	return buf, o.MarshalXDRInto(m)
-}
-
-func (o Device) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o Device) MarshalXDRInto(m *xdr.Marshaller) error {
-	if l := len(o.ID); l > 32 {
-		return xdr.ElementSizeExceeded("ID", l, 32)
-	}
-	m.MarshalBytes(o.ID)
-	if l := len(o.Addresses); l > 16 {
-		return xdr.ElementSizeExceeded("Addresses", l, 16)
-	}
-	m.MarshalUint32(uint32(len(o.Addresses)))
-	for i := range o.Addresses {
-		if err := o.Addresses[i].MarshalXDRInto(m); err != nil {
-			return err
-		}
-	}
-	return m.Error
-}
-
-func (o *Device) UnmarshalXDR(bs []byte) error {
-	u := &xdr.Unmarshaller{Data: bs}
-	return o.UnmarshalXDRFrom(u)
-}
-func (o *Device) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	o.ID = u.UnmarshalBytesMax(32)
-	_AddressesSize := int(u.UnmarshalUint32())
-	if _AddressesSize < 0 {
-		return xdr.ElementSizeExceeded("Addresses", _AddressesSize, 16)
-	} else if _AddressesSize == 0 {
-		o.Addresses = nil
-	} else {
-		if _AddressesSize > 16 {
-			return xdr.ElementSizeExceeded("Addresses", _AddressesSize, 16)
-		}
-		if _AddressesSize <= len(o.Addresses) {
-			o.Addresses = o.Addresses[:_AddressesSize]
-		} else {
-			o.Addresses = make([]Address, _AddressesSize)
-		}
-		for i := range o.Addresses {
-			(&o.Addresses[i]).UnmarshalXDRFrom(u)
-		}
-	}
-	return u.Error
-}
-
-/*
-
-Address Structure:
-
- 0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-/                                                               /
-\                  URL (length + padded data)                   \
-/                                                               /
-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-
-struct Address {
-	string URL<2083>;
-}
-
-*/
-
-func (o Address) XDRSize() int {
-	return 4 + len(o.URL) + xdr.Padding(len(o.URL))
-}
-
-func (o Address) MarshalXDR() ([]byte, error) {
-	buf := make([]byte, o.XDRSize())
-	m := &xdr.Marshaller{Data: buf}
-	return buf, o.MarshalXDRInto(m)
-}
-
-func (o Address) MustMarshalXDR() []byte {
-	bs, err := o.MarshalXDR()
-	if err != nil {
-		panic(err)
-	}
-	return bs
-}
-
-func (o Address) MarshalXDRInto(m *xdr.Marshaller) error {
-	if l := len(o.URL); l > 2083 {
-		return xdr.ElementSizeExceeded("URL", l, 2083)
-	}
-	m.MarshalString(o.URL)
-	return m.Error
-}
-
-func (o *Address) UnmarshalXDR(bs []byte) error {
-	u := &xdr.Unmarshaller{Data: bs}
-	return o.UnmarshalXDRFrom(u)
-}
-func (o *Address) UnmarshalXDRFrom(u *xdr.Unmarshaller) error {
-	o.URL = u.UnmarshalStringMax(2083)
-	return u.Error
-}

lib/events/events.go

@@ -111,16 +111,20 @@ func (t EventType) MarshalText() ([]byte, error) {
 const BufferSize = 64
 
 type Logger struct {
-	subs   []*Subscription
-	nextID int
-	mutex  sync.Mutex
+	subs                []*Subscription
+	nextSubscriptionIDs []int
+	nextGlobalID        int
+	mutex               sync.Mutex
 }
 
 type Event struct {
-	ID   int         `json:"id"`
-	Time time.Time   `json:"time"`
-	Type EventType   `json:"type"`
-	Data interface{} `json:"data"`
+	// Per-subscription sequential event ID. Named "id" for backwards compatibility with the REST API
+	SubscriptionID int `json:"id"`
+	// Global ID of the event across all subscriptions
+	GlobalID int         `json:"globalID"`
+	Time     time.Time   `json:"time"`
+	Type     EventType   `json:"type"`
+	Data     interface{} `json:"data"`
 }
 
 type Subscription struct {
@@ -144,16 +148,21 @@ func NewLogger() *Logger {
 
 func (l *Logger) Log(t EventType, data interface{}) {
 	l.mutex.Lock()
-	dl.Debugln("log", l.nextID, t, data)
-	l.nextID++
+	dl.Debugln("log", l.nextGlobalID, t, data)
+	l.nextGlobalID++
+
 	e := Event{
-		ID:   l.nextID,
-		Time: time.Now(),
-		Type: t,
-		Data: data,
+		GlobalID: l.nextGlobalID,
+		Time:     time.Now(),
+		Type:     t,
+		Data:     data,
 	}
-	for _, s := range l.subs {
+
+	for i, s := range l.subs {
 		if s.mask&t != 0 {
+			e.SubscriptionID = l.nextSubscriptionIDs[i]
+			l.nextSubscriptionIDs[i]++
+
 			select {
 			case s.events <- e:
 			default:
@@ -182,6 +191,7 @@ func (l *Logger) Subscribe(mask EventType) *Subscription {
 	}
 
 	l.subs = append(l.subs, s)
+	l.nextSubscriptionIDs = append(l.nextSubscriptionIDs, 1)
 	l.mutex.Unlock()
 	return s
 }
@@ -192,9 +202,15 @@ func (l *Logger) Unsubscribe(s *Subscription) {
 	for i, ss := range l.subs {
 		if s == ss {
 			last := len(l.subs) - 1
+
 			l.subs[i] = l.subs[last]
 			l.subs[last] = nil
 			l.subs = l.subs[:last]
+
+			l.nextSubscriptionIDs[i] = l.nextSubscriptionIDs[last]
+			l.nextSubscriptionIDs[last] = 0
+			l.nextSubscriptionIDs = l.nextSubscriptionIDs[:last]
+
 			break
 		}
 	}
@@ -234,7 +250,7 @@ type bufferedSubscription struct {
 	sub  *Subscription
 	buf  []Event
 	next int
-	cur  int
+	cur  int // Current SubscriptionID
 	mut  sync.Mutex
 	cond *stdsync.Cond
 }
@@ -270,7 +286,7 @@ func (s *bufferedSubscription) pollingLoop() {
 		s.mut.Lock()
 		s.buf[s.next] = ev
 		s.next = (s.next + 1) % len(s.buf)
-		s.cur = ev.ID
+		s.cur = ev.SubscriptionID
 		s.cond.Broadcast()
 		s.mut.Unlock()
 	}
@@ -285,12 +301,12 @@ func (s *bufferedSubscription) Since(id int, into []Event) []Event {
 	}
 
 	for i := s.next; i < len(s.buf); i++ {
-		if s.buf[i].ID > id {
+		if s.buf[i].SubscriptionID > id {
 			into = append(into, s.buf[i])
 		}
 	}
 	for i := 0; i < s.next; i++ {
-		if s.buf[i].ID > id {
+		if s.buf[i].SubscriptionID > id {
 			into = append(into, s.buf[i])
 		}
 	}

lib/events/events_test.go

@@ -128,7 +128,7 @@ func TestUnsubscribe(t *testing.T) {
 	}
 }
 
-func TestIDs(t *testing.T) {
+func TestGlobalIDs(t *testing.T) {
 	l := events.NewLogger()
 
 	s := l.Subscribe(events.AllEvents)
@@ -144,7 +144,7 @@ func TestIDs(t *testing.T) {
 	if ev.Data.(string) != "foo" {
 		t.Fatal("Incorrect event:", ev)
 	}
-	id := ev.ID
+	id := ev.GlobalID
 
 	ev, err = s.Poll(timeout)
 	if err != nil {
@@ -153,8 +153,48 @@ func TestIDs(t *testing.T) {
 	if ev.Data.(string) != "bar" {
 		t.Fatal("Incorrect event:", ev)
 	}
-	if ev.ID != id+1 {
-		t.Fatalf("ID not incremented (%d != %d)", ev.ID, id+1)
+	if ev.GlobalID != id+1 {
+		t.Fatalf("ID not incremented (%d != %d)", ev.GlobalID, id+1)
+	}
+}
+
+func TestSubscriptionIDs(t *testing.T) {
+	l := events.NewLogger()
+
+	s := l.Subscribe(events.DeviceConnected)
+	defer l.Unsubscribe(s)
+
+	l.Log(events.DeviceDisconnected, "a")
+	l.Log(events.DeviceConnected, "b")
+	l.Log(events.DeviceConnected, "c")
+	l.Log(events.DeviceDisconnected, "d")
+
+	ev, err := s.Poll(timeout)
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+
+	if ev.GlobalID != 2 {
+		t.Fatal("Incorrect GlobalID:", ev.GlobalID)
+	}
+	if ev.SubscriptionID != 1 {
+		t.Fatal("Incorrect SubscriptionID:", ev.SubscriptionID)
+	}
+
+	ev, err = s.Poll(timeout)
+	if err != nil {
+		t.Fatal("Unexpected error:", err)
+	}
+	if ev.GlobalID != 3 {
+		t.Fatal("Incorrect GlobalID:", ev.GlobalID)
+	}
+	if ev.SubscriptionID != 2 {
+		t.Fatal("Incorrect SubscriptionID:", ev.SubscriptionID)
+	}
+
+	ev, err = s.Poll(timeout)
+	if err != events.ErrTimeout {
+		t.Fatal("Unexpected error:", err)
 	}
 }
 
@@ -179,11 +219,92 @@ func TestBufferedSub(t *testing.T) {
 	for recv < 10*events.BufferSize {
 		evs := bs.Since(recv, nil)
 		for _, ev := range evs {
-			if ev.ID != recv+1 {
-				t.Fatalf("Incorrect ID; %d != %d", ev.ID, recv+1)
+			if ev.GlobalID != recv+1 {
+				t.Fatalf("Incorrect ID; %d != %d", ev.GlobalID, recv+1)
 			}
-			recv = ev.ID
+			recv = ev.GlobalID
+		}
+	}
+}
+
+func BenchmarkBufferedSub(b *testing.B) {
+	l := events.NewLogger()
+
+	s := l.Subscribe(events.AllEvents)
+	defer l.Unsubscribe(s)
+	bufferSize := events.BufferSize
+	bs := events.NewBufferedSubscription(s, bufferSize)
+
+	// The coord channel paces the sender according to the receiver,
+	// ensuring that no events are dropped. The benchmark measures sending +
+	// receiving + synchronization overhead.
+
+	coord := make(chan struct{}, bufferSize)
+	for i := 0; i < bufferSize-1; i++ {
+		coord <- struct{}{}
+	}
+
+	// Receive the events
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		recv := 0
+		var evs []events.Event
+		for i := 0; i < b.N; {
+			evs = bs.Since(recv, evs[:0])
+			for _, ev := range evs {
+				if ev.GlobalID != recv+1 {
+					b.Fatal("skipped event", ev.GlobalID, recv)
+				}
+				recv = ev.GlobalID
+				coord <- struct{}{}
+			}
+			i += len(evs)
+		}
+	}()
+
+	// Send the events
+	eventData := map[string]string{
+		"foo":   "bar",
+		"other": "data",
+		"and":   "something else",
+	}
+	for i := 0; i < b.N; i++ {
+		l.Log(events.DeviceConnected, eventData)
+		<-coord
+	}
+
+	<-done
+	b.ReportAllocs()
+}
+
+func TestSinceUsesSubscriptionId(t *testing.T) {
+	l := events.NewLogger()
+
+	s := l.Subscribe(events.DeviceConnected)
+	defer l.Unsubscribe(s)
+	bs := events.NewBufferedSubscription(s, 10*events.BufferSize)
+
+	l.Log(events.DeviceConnected, "a") // SubscriptionID = 1
+	l.Log(events.DeviceDisconnected, "b")
+	l.Log(events.DeviceDisconnected, "c")
+	l.Log(events.DeviceConnected, "d") // SubscriptionID = 2
+
+	// We need to loop for the events, as they may not all have been
+	// delivered to the buffered subscription when we get here.
+	t0 := time.Now()
+	for time.Since(t0) < time.Second {
+		events := bs.Since(0, nil)
+		if len(events) == 2 {
+			break
+		}
+		if len(events) > 2 {
+			t.Fatal("Incorrect number of events:", len(events))
 		}
 	}
 
+	events := bs.Since(1, nil)
+	if len(events) != 1 {
+		t.Fatal("Incorrect number of events:", len(events))
+	}
 }

lib/model/folder.go

@@ -10,7 +10,7 @@ import "time"
 
 type folder struct {
 	stateTracker
-	scan  folderscan
+	scan  folderScanner
 	model *Model
 	stop  chan struct{}
 }

lib/model/folderscan.go

@@ -1,49 +0,0 @@
-// Copyright (C) 2016 The Syncthing Authors.
-//
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this file,
-// You can obtain one at http://mozilla.org/MPL/2.0/.
-
-package model
-
-import (
-	"math/rand"
-	"time"
-)
-
-type rescanRequest struct {
-	subdirs []string
-	err     chan error
-}
-
-// bundle all folder scan activity
-type folderscan struct {
-	interval time.Duration
-	timer    *time.Timer
-	now      chan rescanRequest
-	delay    chan time.Duration
-}
-
-func (s *folderscan) reschedule() {
-	if s.interval == 0 {
-		return
-	}
-	// Sleep a random time between 3/4 and 5/4 of the configured interval.
-	sleepNanos := (s.interval.Nanoseconds()*3 + rand.Int63n(2*s.interval.Nanoseconds())) / 4
-	interval := time.Duration(sleepNanos) * time.Nanosecond
-	l.Debugln(s, "next rescan in", interval)
-	s.timer.Reset(interval)
-}
-
-func (s *folderscan) Scan(subdirs []string) error {
-	req := rescanRequest{
-		subdirs: subdirs,
-		err:     make(chan error),
-	}
-	s.now <- req
-	return <-req.err
-}
-
-func (s *folderscan) Delay(next time.Duration) {
-	s.delay <- next
-}

lib/model/folderscanner.go

@@ -0,0 +1,63 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package model
+
+import (
+	"github.com/syncthing/syncthing/lib/config"
+	"math/rand"
+	"time"
+)
+
+type rescanRequest struct {
+	subdirs []string
+	err     chan error
+}
+
+// bundle all folder scan activity
+type folderScanner struct {
+	interval time.Duration
+	timer    *time.Timer
+	now      chan rescanRequest
+	delay    chan time.Duration
+}
+
+func newFolderScanner(config config.FolderConfiguration) folderScanner {
+	return folderScanner{
+		interval: time.Duration(config.RescanIntervalS) * time.Second,
+		timer:    time.NewTimer(time.Millisecond), // The first scan should be done immediately.
+		now:      make(chan rescanRequest),
+		delay:    make(chan time.Duration),
+	}
+}
+
+func (f *folderScanner) Reschedule() {
+	if f.interval == 0 {
+		return
+	}
+	// Sleep a random time between 3/4 and 5/4 of the configured interval.
+	sleepNanos := (f.interval.Nanoseconds()*3 + rand.Int63n(2*f.interval.Nanoseconds())) / 4
+	interval := time.Duration(sleepNanos) * time.Nanosecond
+	l.Debugln(f, "next rescan in", interval)
+	f.timer.Reset(interval)
+}
+
+func (f *folderScanner) Scan(subdirs []string) error {
+	req := rescanRequest{
+		subdirs: subdirs,
+		err:     make(chan error),
+	}
+	f.now <- req
+	return <-req.err
+}
+
+func (f *folderScanner) Delay(next time.Duration) {
+	f.delay <- next
+}
+
+func (f *folderScanner) HasNoInterval() bool {
+	return f.interval == 0
+}

lib/model/folderstate.go

@@ -46,6 +46,13 @@ type stateTracker struct {
 	changed time.Time
 }
 
+func newStateTracker(id string) stateTracker {
+	return stateTracker{
+		folderID: id,
+		mut:      sync.NewMutex(),
+	}
+}
+
 // setState sets the new folder state, for states other than FolderError.
 func (s *stateTracker) setState(newState folderState) {
 	if newState == FolderError {

lib/model/model.go

@@ -40,25 +40,23 @@ import (
 
 // How many files to send in each Index/IndexUpdate message.
 const (
-	indexTargetSize   = 250 * 1024 // Aim for making index messages no larger than 250 KiB (uncompressed)
-	indexPerFileSize  = 250        // Each FileInfo is approximately this big, in bytes, excluding BlockInfos
-	indexPerBlockSize = 40         // Each BlockInfo is approximately this big
-	indexBatchSize    = 1000       // Either way, don't include more files than this
+	indexTargetSize = 250 * 1024 // Aim for making index messages no larger than 250 KiB (uncompressed)
+	indexBatchSize  = 1000       // Either way, don't include more files than this
 )
 
 type service interface {
-	Serve()
-	Stop()
-	Jobs() ([]string, []string) // In progress, Queued
 	BringToFront(string)
 	DelayScan(d time.Duration)
-	IndexUpdated() // Remote index was updated notification
+	IndexUpdated()              // Remote index was updated notification
+	Jobs() ([]string, []string) // In progress, Queued
 	Scan(subs []string) error
+	Serve()
+	Stop()
 
-	setState(state folderState)
-	setError(err error)
-	clearError()
 	getState() (folderState, time.Time, error)
+	setState(state folderState)
+	clearError()
+	setError(err error)
 }
 
 type Availability struct {
@@ -95,7 +93,7 @@ type Model struct {
 
 	conn              map[protocol.DeviceID]connections.Connection
 	helloMessages     map[protocol.DeviceID]protocol.HelloResult
-	deviceClusterConf map[protocol.DeviceID]protocol.ClusterConfigMessage
+	deviceClusterConf map[protocol.DeviceID]protocol.ClusterConfig
 	devicePaused      map[protocol.DeviceID]bool
 	deviceDownloads   map[protocol.DeviceID]*deviceDownloadState
 	pmut              sync.RWMutex // protects the above
@@ -110,6 +108,7 @@ var (
 
 // errors returned by the CheckFolderHealth method
 var (
+	errFolderPathEmpty     = errors.New("folder path empty")
 	errFolderPathMissing   = errors.New("folder path missing")
 	errFolderMarkerMissing = errors.New("folder marker missing")
 	errHomeDiskNoSpace     = errors.New("home disk has insufficient free space")
@@ -148,7 +147,7 @@ func NewModel(cfg *config.Wrapper, id protocol.DeviceID, deviceName, clientName,
 		folderStatRefs:     make(map[string]*stats.FolderStatisticsReference),
 		conn:               make(map[protocol.DeviceID]connections.Connection),
 		helloMessages:      make(map[protocol.DeviceID]protocol.HelloResult),
-		deviceClusterConf:  make(map[protocol.DeviceID]protocol.ClusterConfigMessage),
+		deviceClusterConf:  make(map[protocol.DeviceID]protocol.ClusterConfig),
 		devicePaused:       make(map[protocol.DeviceID]bool),
 		deviceDownloads:    make(map[protocol.DeviceID]*deviceDownloadState),
 		fmut:               sync.NewRWMutex(),
@@ -413,7 +412,7 @@ func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
 		// This might might be more than it really is, because some blocks can be of a smaller size.
 		downloaded = int64(counts[ft.Name] * protocol.BlockSize)
 
-		fileNeed = ft.Size() - downloaded
+		fileNeed = ft.Size - downloaded
 		if fileNeed < 0 {
 			fileNeed = 0
 		}
@@ -435,7 +434,7 @@ func sizeOfFile(f db.FileIntf) (files, deleted int, bytes int64) {
 	} else {
 		deleted++
 	}
-	bytes += f.Size()
+	bytes += f.FileSize()
 	return
 }
 
@@ -547,12 +546,7 @@ func (m *Model) NeedFolderFiles(folder string, page, perpage int) ([]db.FileInfo
 
 // Index is called when a new device is connected and we receive their full index.
 // Implements the protocol.Model interface.
-func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo, flags uint32, options []protocol.Option) {
-	if flags != 0 {
-		l.Warnln("protocol error: unknown flags 0x%x in Index message", flags)
-		return
-	}
-
+func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) {
 	l.Debugf("IDX(in): %s %q: %d files", deviceID, folder, len(fs))
 
 	if !m.folderSharedWith(folder, deviceID) {
@@ -574,7 +568,7 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F
 	}
 
 	if !ok {
-		l.Fatalf("Index for nonexistant folder %q", folder)
+		l.Fatalf("Index for nonexistent folder %q", folder)
 	}
 
 	m.pmut.RLock()
@@ -594,12 +588,7 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F
 
 // IndexUpdate is called for incremental updates to connected devices' indexes.
 // Implements the protocol.Model interface.
-func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo, flags uint32, options []protocol.Option) {
-	if flags != 0 {
-		l.Warnln("protocol error: unknown flags 0x%x in IndexUpdate message", flags)
-		return
-	}
-
+func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []protocol.FileInfo) {
 	l.Debugf("%v IDXUP(in): %s / %q: %d files", m, deviceID, folder, len(fs))
 
 	if !m.folderSharedWith(folder, deviceID) {
@@ -615,7 +604,7 @@ func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []prot
 	m.fmut.RUnlock()
 
 	if !ok {
-		l.Fatalf("IndexUpdate for nonexistant folder %q", folder)
+		l.Fatalf("IndexUpdate for nonexistent folder %q", folder)
 	}
 
 	m.pmut.RLock()
@@ -650,7 +639,7 @@ func (m *Model) folderSharedWithUnlocked(folder string, deviceID protocol.Device
 	return false
 }
 
-func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterConfigMessage) {
+func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterConfig) {
 	// Check the peer device's announced folders against our own. Emits events
 	// for folders that we don't expect (unknown or not shared).
 	// Also, collect a list of folders we do share, and if he's interested in
@@ -659,21 +648,7 @@ func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterCon
 	tempIndexFolders := make([]string, 0, len(cm.Folders))
 
 	m.fmut.Lock()
-nextFolder:
 	for _, folder := range cm.Folders {
-		cfg := m.folderCfgs[folder.ID]
-
-		if folder.Flags&^protocol.FlagFolderAll != 0 {
-			// There are flags set that we don't know what they mean. Scary!
-			l.Warnf("Device %v: unknown flags for folder %s", deviceID, folder.ID)
-			cfg.Invalid = fmt.Sprintf("Unknown flags from device %v", deviceID)
-			m.cfg.SetFolder(cfg)
-			if srv := m.folderRunners[folder.ID]; srv != nil {
-				srv.setError(fmt.Errorf(cfg.Invalid))
-			}
-			continue nextFolder
-		}
-
 		if !m.folderSharedWithUnlocked(folder.ID, deviceID) {
 			events.Default.Log(events.FolderRejected, map[string]string{
 				"folder":      folder.ID,
@@ -683,7 +658,7 @@ nextFolder:
 			l.Infof("Unexpected folder ID %q sent from device %q; ensure that the folder exists and that this device is selected under \"Share With\" in the folder configuration.", folder.ID, deviceID)
 			continue
 		}
-		if folder.Flags&protocol.FlagFolderDisabledTempIndexes == 0 {
+		if !folder.DisableTempIndexes {
 			tempIndexFolders = append(tempIndexFolders, folder.ID)
 		}
 	}
@@ -737,7 +712,7 @@ nextFolder:
 					}
 
 					// The introducers' introducers are also our introducers.
-					if device.Flags&protocol.FlagIntroducer != 0 {
+					if device.Introducer {
 						l.Infof("Device %v is now also an introducer", id)
 						newDeviceCfg.Introducer = true
 					}
@@ -808,7 +783,7 @@ func (m *Model) Close(device protocol.DeviceID, err error) {
 
 // Request returns the specified data segment by reading it from local disk.
 // Implements the protocol.Model interface.
-func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, flags uint32, options []protocol.Option, buf []byte) error {
+func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset int64, hash []byte, fromTemporary bool, buf []byte) error {
 	if offset < 0 {
 		return protocol.ErrInvalid
 	}
@@ -817,14 +792,8 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 		l.Warnf("Request from %s for file %s in unshared folder %q", deviceID, name, folder)
 		return protocol.ErrNoSuchFile
 	}
-
-	if flags != 0 && flags != protocol.FlagFromTemporary {
-		// We currently support only no flags, or FromTemporary flag.
-		return fmt.Errorf("protocol error: unknown flags 0x%x in Request message", flags)
-	}
-
 	if deviceID != protocol.LocalDeviceID {
-		l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d f=%d", m, deviceID, folder, name, offset, len(buf), flags)
+		l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d t=%v", m, deviceID, folder, name, offset, len(buf), fromTemporary)
 	}
 	m.fmut.RLock()
 	folderCfg := m.folderCfgs[folder]
@@ -884,7 +853,7 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 
 	// Only check temp files if the flag is set, and if we are set to advertise
 	// the temp indexes.
-	if flags&protocol.FlagFromTemporary != 0 && !folderCfg.DisableTempIndexes {
+	if fromTemporary && !folderCfg.DisableTempIndexes {
 		tempFn := filepath.Join(folderPath, defTempNamer.TempName(name))
 		if err := readOffsetIntoBuf(tempFn, offset, buf); err == nil {
 			return nil
@@ -1031,8 +1000,8 @@ func (m *Model) OnHello(remoteID protocol.DeviceID, addr net.Addr, hello protoco
 }
 
 // GetHello is called when we are about to connect to some remote device.
-func (m *Model) GetHello(protocol.DeviceID) protocol.Version13HelloMessage {
-	return protocol.Version13HelloMessage{
+func (m *Model) GetHello(protocol.DeviceID) protocol.HelloIntf {
+	return &protocol.Hello{
 		DeviceName:    m.deviceName,
 		ClientName:    m.clientName,
 		ClientVersion: m.clientVersion,
@@ -1105,7 +1074,7 @@ func (m *Model) PauseDevice(device protocol.DeviceID) {
 	events.Default.Log(events.DevicePaused, map[string]string{"device": device.String()})
 }
 
-func (m *Model) DownloadProgress(device protocol.DeviceID, folder string, updates []protocol.FileDownloadProgressUpdate, flags uint32, options []protocol.Option) {
+func (m *Model) DownloadProgress(device protocol.DeviceID, folder string, updates []protocol.FileDownloadProgressUpdate) {
 	if !m.folderSharedWith(folder, device) {
 		return
 	}
@@ -1242,13 +1211,13 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold
 
 		if len(batch) == indexBatchSize || currentBatchSize > indexTargetSize {
 			if initial {
-				if err = conn.Index(folder, batch, 0, nil); err != nil {
+				if err = conn.Index(folder, batch); err != nil {
 					return false
 				}
 				l.Debugf("sendIndexes for %s-%s/%q: %d files (<%d bytes) (initial index)", deviceID, name, folder, len(batch), currentBatchSize)
 				initial = false
 			} else {
-				if err = conn.IndexUpdate(folder, batch, 0, nil); err != nil {
+				if err = conn.IndexUpdate(folder, batch); err != nil {
 					return false
 				}
 				l.Debugf("sendIndexes for %s-%s/%q: %d files (<%d bytes) (batched update)", deviceID, name, folder, len(batch), currentBatchSize)
@@ -1259,17 +1228,17 @@ func sendIndexTo(initial bool, minLocalVer int64, conn protocol.Connection, fold
 		}
 
 		batch = append(batch, f)
-		currentBatchSize += indexPerFileSize + len(f.Blocks)*indexPerBlockSize
+		currentBatchSize += f.ProtoSize()
 		return true
 	})
 
 	if initial && err == nil {
-		err = conn.Index(folder, batch, 0, nil)
+		err = conn.Index(folder, batch)
 		if err == nil {
 			l.Debugf("sendIndexes for %s-%s/%q: %d files (small initial index)", deviceID, name, folder, len(batch))
 		}
 	} else if len(batch) > 0 && err == nil {
-		err = conn.IndexUpdate(folder, batch, 0, nil)
+		err = conn.IndexUpdate(folder, batch)
 		if err == nil {
 			l.Debugf("sendIndexes for %s-%s/%q: %d files (last batch)", deviceID, name, folder, len(batch))
 		}
@@ -1324,11 +1293,14 @@ func (m *Model) localChangeDetected(folder, path string, files []protocol.FileIn
 		objType := "file"
 		action := "modified"
 
-		// If our local vector is verison 1 AND it is the only version vector so far seen for this file then
-		// it is a new file.  Else if it is > 1 it's not new, and if it is 1 but another shortId version vector
-		// exists then it is new for us but created elsewhere so the file is still not new but modified by us.
-		// Only if it is truly new do we change this to 'added', else we leave it as 'modified'.
-		if len(file.Version) == 1 && file.Version[0].Value == 1 {
+		// If our local vector is version 1 AND it is the only version
+		// vector so far seen for this file then it is a new file.  Else if
+		// it is > 1 it's not new, and if it is 1 but another shortId
+		// version vector exists then it is new for us but created elsewhere
+		// so the file is still not new but modified by us. Only if it is
+		// truly new do we change this to 'added', else we leave it as
+		// 'modified'.
+		if len(file.Version.Counters) == 1 && file.Version.Counters[0].Value == 1 {
 			action = "added"
 		}
 
@@ -1339,11 +1311,8 @@ func (m *Model) localChangeDetected(folder, path string, files []protocol.FileIn
 			action = "deleted"
 		}
 
-		// If the file is a level or more deep then the forward slash seperator is embedded
-		// in the filename and makes the path look wierd on windows, so lets fix it
-		filename := filepath.FromSlash(file.Name)
-		// And append it to the filepath
-		path := filepath.Join(path, filename)
+		// The full file path, adjusted to the local path separator character.
+		path := filepath.Join(path, filepath.FromSlash(file.Name))
 
 		events.Default.Log(events.LocalChangeDetected, map[string]string{
 			"folder": folder,
@@ -1431,10 +1400,10 @@ func (m *Model) ScanFolders() map[string]error {
 }
 
 func (m *Model) ScanFolder(folder string) error {
-	return m.ScanFolderSubs(folder, nil)
+	return m.ScanFolderSubdirs(folder, nil)
 }
 
-func (m *Model) ScanFolderSubs(folder string, subs []string) error {
+func (m *Model) ScanFolderSubdirs(folder string, subs []string) error {
 	m.fmut.Lock()
 	runner, ok := m.folderRunners[folder]
 	m.fmut.Unlock()
@@ -1449,13 +1418,13 @@ func (m *Model) ScanFolderSubs(folder string, subs []string) error {
 	return runner.Scan(subs)
 }
 
-func (m *Model) internalScanFolderSubdirs(folder string, subs []string) error {
-	for i, sub := range subs {
+func (m *Model) internalScanFolderSubdirs(folder string, subDirs []string) error {
+	for i, sub := range subDirs {
 		sub = osutil.NativeFilename(sub)
 		if p := filepath.Clean(filepath.Join(folder, sub)); !strings.HasPrefix(p, folder) {
 			return errors.New("invalid subpath")
 		}
-		subs[i] = sub
+		subDirs[i] = sub
 	}
 
 	m.fmut.Lock()
@@ -1488,7 +1457,7 @@ func (m *Model) internalScanFolderSubdirs(folder string, subs []string) error {
 	// Clean the list of subitems to ensure that we start at a known
 	// directory, and don't scan subdirectories of things we've already
 	// scanned.
-	subs = unifySubs(subs, func(f string) bool {
+	subDirs = unifySubs(subDirs, func(f string) bool {
 		_, ok := fs.Get(protocol.LocalDeviceID, f)
 		return ok
 	})
@@ -1503,7 +1472,7 @@ func (m *Model) internalScanFolderSubdirs(folder string, subs []string) error {
 	fchan, err := scanner.Walk(scanner.Config{
 		Folder:                folderCfg.ID,
 		Dir:                   folderCfg.Path(),
-		Subs:                  subs,
+		Subs:                  subDirs,
 		Matcher:               ignores,
 		BlockSize:             protocol.BlockSize,
 		TempNamer:             defTempNamer,
@@ -1556,15 +1525,15 @@ func (m *Model) internalScanFolderSubdirs(folder string, subs []string) error {
 		m.updateLocalsFromScanning(folder, batch)
 	}
 
-	if len(subs) == 0 {
+	if len(subDirs) == 0 {
 		// If we have no specific subdirectories to traverse, set it to one
 		// empty prefix so we traverse the entire folder contents once.
-		subs = []string{""}
+		subDirs = []string{""}
 	}
 
 	// Do a scan of the database for each prefix, to check for deleted files.
 	batch = batch[:0]
-	for _, sub := range subs {
+	for _, sub := range subDirs {
 		var iterError error
 
 		fs.WithPrefixedHaveTruncated(protocol.LocalDeviceID, sub, func(fi db.FileIntf) bool {
@@ -1583,10 +1552,14 @@ func (m *Model) internalScanFolderSubdirs(folder string, subs []string) error {
 					// File has been ignored or an unsupported symlink. Set invalid bit.
 					l.Debugln("setting invalid bit on ignored", f)
 					nf := protocol.FileInfo{
-						Name:     f.Name,
-						Flags:    f.Flags | protocol.FlagInvalid,
-						Modified: f.Modified,
-						Version:  f.Version, // The file is still the same, so don't bump version
+						Name:          f.Name,
+						Type:          f.Type,
+						Size:          f.Size,
+						Modified:      f.Modified,
+						Permissions:   f.Permissions,
+						NoPermissions: f.NoPermissions,
+						Invalid:       true,
+						Version:       f.Version, // The file is still the same, so don't bump version
 					}
 					batch = append(batch, nf)
 				} else if _, err := osutil.Lstat(filepath.Join(folderCfg.Path(), f.Name)); err != nil {
@@ -1601,16 +1574,13 @@ func (m *Model) internalScanFolderSubdirs(folder string, subs []string) error {
 
 					nf := protocol.FileInfo{
 						Name:     f.Name,
-						Flags:    f.Flags | protocol.FlagDeleted,
+						Type:     f.Type,
+						Size:     f.Size,
 						Modified: f.Modified,
+						Deleted:  true,
 						Version:  f.Version.Update(m.shortID),
 					}
 
-					// The deleted file might have been ignored at some
-					// point, but it currently isn't so we make sure to
-					// clear the invalid bit.
-					nf.Flags &^= protocol.FlagInvalid
-
 					batch = append(batch, nf)
 				}
 			}
@@ -1676,30 +1646,21 @@ func (m *Model) numHashers(folder string) int {
 
 // generateClusterConfig returns a ClusterConfigMessage that is correct for
 // the given peer device
-func (m *Model) generateClusterConfig(device protocol.DeviceID) protocol.ClusterConfigMessage {
-	var message protocol.ClusterConfigMessage
+func (m *Model) generateClusterConfig(device protocol.DeviceID) protocol.ClusterConfig {
+	var message protocol.ClusterConfig
 
 	m.fmut.RLock()
 	for _, folder := range m.deviceFolders[device] {
 		folderCfg := m.cfg.Folders()[folder]
 		protocolFolder := protocol.Folder{
-			ID:    folder,
-			Label: folderCfg.Label,
+			ID:                 folder,
+			Label:              folderCfg.Label,
+			ReadOnly:           folderCfg.Type == config.FolderTypeReadOnly,
+			IgnorePermissions:  folderCfg.IgnorePerms,
+			IgnoreDelete:       folderCfg.IgnoreDelete,
+			DisableTempIndexes: folderCfg.DisableTempIndexes,
 		}
-		var flags uint32
-		if folderCfg.Type == config.FolderTypeReadOnly {
-			flags |= protocol.FlagFolderReadOnly
-		}
-		if folderCfg.IgnorePerms {
-			flags |= protocol.FlagFolderIgnorePerms
-		}
-		if folderCfg.IgnoreDelete {
-			flags |= protocol.FlagFolderIgnoreDelete
-		}
-		if folderCfg.DisableTempIndexes {
-			flags |= protocol.FlagFolderDisabledTempIndexes
-		}
-		protocolFolder.Flags = flags
+
 		for _, device := range m.folderDevices[folder] {
 			// DeviceID is a value type, but with an underlying array. Copy it
 			// so we don't grab aliases to the same array later on in device[:]
@@ -1711,14 +1672,11 @@ func (m *Model) generateClusterConfig(device protocol.DeviceID) protocol.Cluster
 				ID:          device[:],
 				Name:        deviceCfg.Name,
 				Addresses:   deviceCfg.Addresses,
-				Compression: uint32(deviceCfg.Compression),
+				Compression: deviceCfg.Compression,
 				CertName:    deviceCfg.CertName,
-				Flags:       protocol.FlagShareTrusted,
+				Introducer:  deviceCfg.Introducer,
 			}
 
-			if deviceCfg.Introducer {
-				protocolDevice.Flags |= protocol.FlagIntroducer
-			}
 			protocolFolder.Devices = append(protocolFolder.Devices, protocolDevice)
 		}
 		message.Folders = append(message.Folders, protocolFolder)
@@ -1763,7 +1721,7 @@ func (m *Model) Override(folder string) {
 		have, ok := fs.Get(protocol.LocalDeviceID, need.Name)
 		if !ok || have.Name != need.Name {
 			// We are missing the file
-			need.Flags |= protocol.FlagDeleted
+			need.Deleted = true
 			need.Blocks = nil
 			need.Version = need.Version.Update(m.shortID)
 		} else {
@@ -1872,7 +1830,7 @@ func (m *Model) GlobalDirectoryTree(folder, prefix string, levels int, dirsonly
 
 		if !dirsonly && base != "" {
 			last[base] = []interface{}{
-				time.Unix(f.Modified, 0), f.Size(),
+				time.Unix(f.Modified, 0), f.FileSize(),
 			}
 		}
 
@@ -1953,6 +1911,10 @@ func (m *Model) CheckFolderHealth(id string) error {
 
 // checkFolderPath returns nil if the folder path exists and has the marker file.
 func (m *Model) checkFolderPath(folder config.FolderConfiguration) error {
+	if folder.Path() == "" {
+		return errFolderPathEmpty
+	}
+
 	if fi, err := os.Stat(folder.Path()); err != nil || !fi.IsDir() {
 		return errFolderPathMissing
 	}
@@ -2181,11 +2143,7 @@ func mapDeviceCfgs(devices []config.DeviceConfiguration) map[protocol.DeviceID]s
 
 func filterIndex(folder string, fs []protocol.FileInfo, dropDeletes bool, ignores *ignore.Matcher) []protocol.FileInfo {
 	for i := 0; i < len(fs); {
-		if fs[i].Flags&^protocol.FlagsAll != 0 {
-			l.Debugln("dropping update for file with unknown bits set", fs[i])
-			fs[i] = fs[len(fs)-1]
-			fs = fs[:len(fs)-1]
-		} else if fs[i].IsDeleted() && dropDeletes {
+		if fs[i].IsDeleted() && dropDeletes {
 			l.Debugln("dropping update for undesired delete", fs[i])
 			fs[i] = fs[len(fs)-1]
 			fs = fs[:len(fs)-1]

lib/model/model_test.go

@@ -55,19 +55,19 @@ func init() {
 var testDataExpected = map[string]protocol.FileInfo{
 	"foo": {
 		Name:     "foo",
-		Flags:    0,
+		Type:     protocol.FileInfoTypeFile,
 		Modified: 0,
 		Blocks:   []protocol.BlockInfo{{Offset: 0x0, Size: 0x7, Hash: []uint8{0xae, 0xc0, 0x70, 0x64, 0x5f, 0xe5, 0x3e, 0xe3, 0xb3, 0x76, 0x30, 0x59, 0x37, 0x61, 0x34, 0xf0, 0x58, 0xcc, 0x33, 0x72, 0x47, 0xc9, 0x78, 0xad, 0xd1, 0x78, 0xb6, 0xcc, 0xdf, 0xb0, 0x1, 0x9f}}},
 	},
 	"empty": {
 		Name:     "empty",
-		Flags:    0,
+		Type:     protocol.FileInfoTypeFile,
 		Modified: 0,
 		Blocks:   []protocol.BlockInfo{{Offset: 0x0, Size: 0x0, Hash: []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}}},
 	},
 	"bar": {
 		Name:     "bar",
-		Flags:    0,
+		Type:     protocol.FileInfoTypeFile,
 		Modified: 0,
 		Blocks:   []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}},
 	},
@@ -77,8 +77,9 @@ func init() {
 	// Fix expected test data to match reality
 	for n, f := range testDataExpected {
 		fi, _ := os.Stat("testdata/" + n)
-		f.Flags = uint32(fi.Mode())
+		f.Permissions = uint32(fi.Mode())
 		f.Modified = fi.ModTime().Unix()
+		f.Size = fi.Size()
 		testDataExpected[n] = f
 	}
 }
@@ -98,7 +99,7 @@ func TestRequest(t *testing.T) {
 
 	// Existing, shared file
 	bs = bs[:6]
-	err := m.Request(device1, "default", "foo", 0, nil, 0, nil, bs)
+	err := m.Request(device1, "default", "foo", 0, nil, false, bs)
 	if err != nil {
 		t.Error(err)
 	}
@@ -107,32 +108,32 @@ func TestRequest(t *testing.T) {
 	}
 
 	// Existing, nonshared file
-	err = m.Request(device2, "default", "foo", 0, nil, 0, nil, bs)
+	err = m.Request(device2, "default", "foo", 0, nil, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Nonexistent file
-	err = m.Request(device1, "default", "nonexistent", 0, nil, 0, nil, bs)
+	err = m.Request(device1, "default", "nonexistent", 0, nil, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Shared folder, but disallowed file name
-	err = m.Request(device1, "default", "../walk.go", 0, nil, 0, nil, bs)
+	err = m.Request(device1, "default", "../walk.go", 0, nil, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Negative offset
-	err = m.Request(device1, "default", "foo", -4, nil, 0, nil, bs[:0])
+	err = m.Request(device1, "default", "foo", -4, nil, false, bs[:0])
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
 
 	// Larger block than available
 	bs = bs[:42]
-	err = m.Request(device1, "default", "foo", 0, nil, 0, nil, bs)
+	err = m.Request(device1, "default", "foo", 0, nil, false, bs)
 	if err == nil {
 		t.Error("Unexpected nil error on insecure file read")
 	}
@@ -168,11 +169,11 @@ func benchmarkIndex(b *testing.B, nfiles int) {
 	m.ServeBackground()
 
 	files := genFiles(nfiles)
-	m.Index(device1, "default", files, 0, nil)
+	m.Index(device1, "default", files)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		m.Index(device1, "default", files, 0, nil)
+		m.Index(device1, "default", files)
 	}
 	b.ReportAllocs()
 }
@@ -199,11 +200,11 @@ func benchmarkIndexUpdate(b *testing.B, nfiles, nufiles int) {
 	files := genFiles(nfiles)
 	ufiles := genFiles(nufiles)
 
-	m.Index(device1, "default", files, 0, nil)
+	m.Index(device1, "default", files)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		m.IndexUpdate(device1, "default", ufiles, 0, nil)
+		m.IndexUpdate(device1, "default", ufiles)
 	}
 	b.ReportAllocs()
 }
@@ -211,8 +212,6 @@ func benchmarkIndexUpdate(b *testing.B, nfiles, nufiles int) {
 type downloadProgressMessage struct {
 	folder  string
 	updates []protocol.FileDownloadProgressUpdate
-	flags   uint32
-	options []protocol.Option
 }
 
 type FakeConnection struct {
@@ -240,11 +239,11 @@ func (f FakeConnection) Option(string) string {
 	return ""
 }
 
-func (FakeConnection) Index(string, []protocol.FileInfo, uint32, []protocol.Option) error {
+func (FakeConnection) Index(string, []protocol.FileInfo) error {
 	return nil
 }
 
-func (FakeConnection) IndexUpdate(string, []protocol.FileInfo, uint32, []protocol.Option) error {
+func (FakeConnection) IndexUpdate(string, []protocol.FileInfo) error {
 	return nil
 }
 
@@ -252,7 +251,7 @@ func (f FakeConnection) Request(folder, name string, offset int64, size int, has
 	return f.requestData, nil
 }
 
-func (FakeConnection) ClusterConfig(protocol.ClusterConfigMessage) {}
+func (FakeConnection) ClusterConfig(protocol.ClusterConfig) {}
 
 func (FakeConnection) Ping() bool {
 	return true
@@ -266,12 +265,10 @@ func (FakeConnection) Statistics() protocol.Statistics {
 	return protocol.Statistics{}
 }
 
-func (f *FakeConnection) DownloadProgress(folder string, updates []protocol.FileDownloadProgressUpdate, flags uint32, options []protocol.Option) {
+func (f *FakeConnection) DownloadProgress(folder string, updates []protocol.FileDownloadProgressUpdate) {
 	f.downloadProgressMessages = append(f.downloadProgressMessages, downloadProgressMessage{
 		folder:  folder,
 		updates: updates,
-		flags:   flags,
-		options: options,
 	})
 }
 
@@ -305,7 +302,7 @@ func BenchmarkRequest(b *testing.B) {
 		},
 		Connection: fc,
 	}, protocol.HelloResult{})
-	m.Index(device1, "default", files, 0, nil)
+	m.Index(device1, "default", files)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -451,13 +448,13 @@ func TestClusterConfig(t *testing.T) {
 	if id := r.Devices[0].ID; !bytes.Equal(id, device1[:]) {
 		t.Errorf("Incorrect device ID %x != %x", id, device1)
 	}
-	if r.Devices[0].Flags&protocol.FlagIntroducer == 0 {
+	if !r.Devices[0].Introducer {
 		t.Error("Device1 should be flagged as Introducer")
 	}
 	if id := r.Devices[1].ID; !bytes.Equal(id, device2[:]) {
 		t.Errorf("Incorrect device ID %x != %x", id, device2)
 	}
-	if r.Devices[1].Flags&protocol.FlagIntroducer != 0 {
+	if r.Devices[1].Introducer {
 		t.Error("Device2 should not be flagged as Introducer")
 	}
 
@@ -471,13 +468,13 @@ func TestClusterConfig(t *testing.T) {
 	if id := r.Devices[0].ID; !bytes.Equal(id, device1[:]) {
 		t.Errorf("Incorrect device ID %x != %x", id, device1)
 	}
-	if r.Devices[0].Flags&protocol.FlagIntroducer == 0 {
+	if !r.Devices[0].Introducer {
 		t.Error("Device1 should be flagged as Introducer")
 	}
 	if id := r.Devices[1].ID; !bytes.Equal(id, device2[:]) {
 		t.Errorf("Incorrect device ID %x != %x", id, device2)
 	}
-	if r.Devices[1].Flags&protocol.FlagIntroducer != 0 {
+	if r.Devices[1].Introducer {
 		t.Error("Device2 should not be flagged as Introducer")
 	}
 }
@@ -566,50 +563,12 @@ func TestIgnores(t *testing.T) {
 
 	// Invalid path, marker should be missing, hence returns an error.
 	m.AddFolder(config.FolderConfiguration{ID: "fresh", RawPath: "XXX"})
-	ignores, _, err = m.GetIgnores("fresh")
+	_, _, err = m.GetIgnores("fresh")
 	if err == nil {
 		t.Error("No error")
 	}
 }
 
-func TestRefuseUnknownBits(t *testing.T) {
-	db := db.OpenMemory()
-	m := NewModel(defaultConfig, protocol.LocalDeviceID, "device", "syncthing", "dev", db, nil)
-	m.AddFolder(defaultFolderConfig)
-	m.ServeBackground()
-
-	m.ScanFolder("default")
-	m.Index(device1, "default", []protocol.FileInfo{
-		{
-			Name:  "invalid1",
-			Flags: (protocol.FlagsAll + 1) &^ protocol.FlagInvalid,
-		},
-		{
-			Name:  "invalid2",
-			Flags: (protocol.FlagsAll + 2) &^ protocol.FlagInvalid,
-		},
-		{
-			Name:  "invalid3",
-			Flags: (1 << 31) &^ protocol.FlagInvalid,
-		},
-		{
-			Name:  "valid",
-			Flags: protocol.FlagsAll &^ (protocol.FlagInvalid | protocol.FlagSymlink),
-		},
-	}, 0, nil)
-
-	for _, name := range []string{"invalid1", "invalid2", "invalid3"} {
-		f, ok := m.CurrentGlobalFile("default", name)
-		if ok || f.Name == name {
-			t.Error("Invalid file found or name match")
-		}
-	}
-	f, ok := m.CurrentGlobalFile("default", "valid")
-	if !ok || f.Name != "valid" {
-		t.Error("Valid file not found or name mismatch", ok, f)
-	}
-}
-
 func TestROScanRecovery(t *testing.T) {
 	ldb := db.OpenMemory()
 	set := db.NewFileSet("default", ldb)
@@ -642,9 +601,6 @@ func TestROScanRecovery(t *testing.T) {
 	waitFor := func(status string) error {
 		timeout := time.Now().Add(2 * time.Second)
 		for {
-			if time.Now().After(timeout) {
-				return fmt.Errorf("Timed out waiting for status: %s, current status: %s", status, m.cfg.Folders()["default"].Invalid)
-			}
 			_, _, err := m.State("default")
 			if err == nil && status == "" {
 				return nil
@@ -652,6 +608,10 @@ func TestROScanRecovery(t *testing.T) {
 			if err != nil && err.Error() == status {
 				return nil
 			}
+
+			if time.Now().After(timeout) {
+				return fmt.Errorf("Timed out waiting for status: %s, current status: %v", status, err)
+			}
 			time.Sleep(10 * time.Millisecond)
 		}
 	}
@@ -727,9 +687,6 @@ func TestRWScanRecovery(t *testing.T) {
 	waitFor := func(status string) error {
 		timeout := time.Now().Add(2 * time.Second)
 		for {
-			if time.Now().After(timeout) {
-				return fmt.Errorf("Timed out waiting for status: %s, current status: %s", status, m.cfg.Folders()["default"].Invalid)
-			}
 			_, _, err := m.State("default")
 			if err == nil && status == "" {
 				return nil
@@ -737,6 +694,10 @@ func TestRWScanRecovery(t *testing.T) {
 			if err != nil && err.Error() == status {
 				return nil
 			}
+
+			if time.Now().After(timeout) {
+				return fmt.Errorf("Timed out waiting for status: %s, current status: %v", status, err)
+			}
 			time.Sleep(10 * time.Millisecond)
 		}
 	}
@@ -787,17 +748,18 @@ func TestGlobalDirectoryTree(t *testing.T) {
 	m.ServeBackground()
 
 	b := func(isfile bool, path ...string) protocol.FileInfo {
-		flags := uint32(protocol.FlagDirectory)
+		typ := protocol.FileInfoTypeDirectory
 		blocks := []protocol.BlockInfo{}
 		if isfile {
-			flags = 0
+			typ = protocol.FileInfoTypeFile
 			blocks = []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}}
 		}
 		return protocol.FileInfo{
 			Name:     filepath.Join(path...),
-			Flags:    flags,
+			Type:     typ,
 			Modified: 0x666,
 			Blocks:   blocks,
+			Size:     0xa,
 		}
 	}
 
@@ -869,7 +831,7 @@ func TestGlobalDirectoryTree(t *testing.T) {
 		return string(bytes)
 	}
 
-	m.Index(device1, "default", testdata, 0, nil)
+	m.Index(device1, "default", testdata)
 
 	result := m.GlobalDirectoryTree("default", "", -1, false)
 
@@ -1037,17 +999,18 @@ func TestGlobalDirectorySelfFixing(t *testing.T) {
 	m.ServeBackground()
 
 	b := func(isfile bool, path ...string) protocol.FileInfo {
-		flags := uint32(protocol.FlagDirectory)
+		typ := protocol.FileInfoTypeDirectory
 		blocks := []protocol.BlockInfo{}
 		if isfile {
-			flags = 0
+			typ = protocol.FileInfoTypeFile
 			blocks = []protocol.BlockInfo{{Offset: 0x0, Size: 0xa, Hash: []uint8{0x2f, 0x72, 0xcc, 0x11, 0xa6, 0xfc, 0xd0, 0x27, 0x1e, 0xce, 0xf8, 0xc6, 0x10, 0x56, 0xee, 0x1e, 0xb1, 0x24, 0x3b, 0xe3, 0x80, 0x5b, 0xf9, 0xa9, 0xdf, 0x98, 0xf9, 0x2f, 0x76, 0x36, 0xb0, 0x5c}}}
 		}
 		return protocol.FileInfo{
 			Name:     filepath.Join(path...),
-			Flags:    flags,
+			Type:     typ,
 			Modified: 0x666,
 			Blocks:   blocks,
+			Size:     0xa,
 		}
 	}
 
@@ -1128,7 +1091,7 @@ func TestGlobalDirectorySelfFixing(t *testing.T) {
 		return string(bytes)
 	}
 
-	m.Index(device1, "default", testdata, 0, nil)
+	m.Index(device1, "default", testdata)
 
 	result := m.GlobalDirectoryTree("default", "", -1, false)
 
@@ -1213,7 +1176,7 @@ func benchmarkTree(b *testing.B, n1, n2 int) {
 	m.ScanFolder("default")
 	files := genDeepFiles(n1, n2)
 
-	m.Index(device1, "default", files, 0, nil)
+	m.Index(device1, "default", files)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
@@ -1242,12 +1205,12 @@ func TestIgnoreDelete(t *testing.T) {
 	}
 
 	// Mark it for deletion
-	f.Flags = protocol.FlagDeleted
+	f.Deleted = true
 	f.Version = f.Version.Update(142) // arbitrary short remote ID
 	f.Blocks = nil
 
 	// Send the index
-	m.Index(device1, "default", []protocol.FileInfo{f}, 0, nil)
+	m.Index(device1, "default", []protocol.FileInfo{f})
 
 	// Make sure we ignored it
 	f, ok = m.CurrentGlobalFile("default", "foo")
@@ -1402,7 +1365,7 @@ func TestIssue3028(t *testing.T) {
 
 	os.Remove("testdata/testrm")
 	os.Remove("testdata/testrm2")
-	m.ScanFolderSubs("default", []string{"testrm", "testrm2"})
+	m.ScanFolderSubdirs("default", []string{"testrm", "testrm2"})
 
 	// Verify that the number of files decreased by two and the number of
 	// deleted files increases by two

lib/model/progressemitter.go

@@ -137,7 +137,7 @@ func (t *ProgressEmitter) sendDownloadProgressMessages() {
 			updates := state.update(folder, activePullers)
 
 			if len(updates) > 0 {
-				conn.DownloadProgress(folder, updates, 0, nil)
+				conn.DownloadProgress(folder, updates)
 			}
 		}
 	}

lib/model/progressemitter_test.go

@@ -100,7 +100,6 @@ func TestProgressEmitter(t *testing.T) {
 }
 
 func TestSendDownloadProgressMessages(t *testing.T) {
-
 	c := config.Wrap("/tmp/test", config.Configuration{})
 	c.SetOptions(config.OptionsConfiguration{
 		ProgressUpdateIntervalS: 0,
@@ -112,7 +111,7 @@ func TestSendDownloadProgressMessages(t *testing.T) {
 	p := NewProgressEmitter(c)
 	p.temporaryIndexSubscribe(fc, []string{"folder", "folder2"})
 
-	expect := func(updateIdx int, state *sharedPullerState, updateType uint32, version protocol.Vector, blocks []int32, remove bool) {
+	expect := func(updateIdx int, state *sharedPullerState, updateType protocol.FileDownloadProgressUpdateType, version protocol.Vector, blocks []int32, remove bool) {
 		messageIdx := -1
 		for i, msg := range fc.downloadProgressMessages {
 			if msg.folder == state.folder {
@@ -346,7 +345,7 @@ func TestSendDownloadProgressMessages(t *testing.T) {
 		file: protocol.FileInfo{
 			Name:    "state5",
 			Version: v1,
-			Flags:   protocol.FlagDirectory,
+			Type:    protocol.FileInfoTypeDirectory,
 			Blocks:  blocks,
 		},
 		mut:              sync.NewRWMutex(),
@@ -359,7 +358,7 @@ func TestSendDownloadProgressMessages(t *testing.T) {
 		file: protocol.FileInfo{
 			Name:    "state6",
 			Version: v1,
-			Flags:   protocol.FlagSymlink,
+			Type:    protocol.FileInfoTypeSymlinkUnknown,
 		},
 		mut:              sync.NewRWMutex(),
 		available:        []int32{1, 2, 3},