Fix handling of default values in config (fixes #83)

Fix isTempName to work on Windows (fixes #80)

.gitignore

@@ -1,3 +1,4 @@
 syncthing
+syncthing.exe
 *.tar.gz
-dist
+*.zip

assets.sh

@@ -18,7 +18,7 @@ cd gui
 for f in $(find . -type f) ; do 
 	f="${f#./}"
 	echo "gr, _ = gzip.NewReader(bytes.NewBuffer([]byte{"
-	gzip -c $f | od -vt x1 | sed 's/^[0-9a-f]*//' | sed 's/\([0-9a-f][0-9a-f]\)/0x\1,/g'
+	gzip -n -c $f | od -vt x1 | sed 's/^[0-9a-f]*//' | sed 's/\([0-9a-f][0-9a-f]\)/0x\1,/g'
 	echo "}))"
 	echo "data, _ = ioutil.ReadAll(gr)"
 	echo "Assets[\"$f\"] = data"

build.sh

@@ -2,48 +2,90 @@
 
 export COPYFILE_DISABLE=true
 
+distFiles=(README.md LICENSE) # apart from the binary itself
 version=$(git describe --always)
-buildDir=dist
 
-if [[ $fast != yes ]] ; then
+build() {
+	go build -ldflags "-w -X main.Version $version" ./cmd/syncthing	
+}
+
+prepare() {
 	./assets.sh | gofmt > auto/gui.files.go
 	go get -d
+}
+
+test() {
 	go test ./...
-fi
+}	
 
-if [[ -z $1 ]] ; then
-	go build -ldflags "-X main.Version $version"
-elif [[ $1 == "tar" ]] ; then
-	go build -ldflags "-X main.Version $version" \
-	&& mkdir syncthing-dist \
-	&& cp syncthing README.md LICENSE syncthing-dist \
-	&& tar zcvf syncthing-dist.tar.gz syncthing-dist \
-	&& rm -rf syncthing-dist
-elif [[ $1 == "all" ]] ; then
-	rm -rf "$buildDir"
-	mkdir -p "$buildDir" || exit 1
+tarDist() {
+	name="$1"
+	mkdir -p "$name"
+	cp syncthing "${distFiles[@]}" "$name"
+	tar zcvf "$name.tar.gz" "$name"
+	rm -rf "$name"
+}
 
-	export GOARM=7
-	for os in darwin-amd64 linux-amd64 linux-arm freebsd-amd64 windows-amd64 ; do
-		echo "$os"
-		export name="syncthing-$os"
-		export GOOS=${os%-*}
-		export GOARCH=${os#*-}
-		go build -ldflags "-X main.Version $version"
-		mkdir -p "$name"
-		cp README.md LICENSE "$name"
-		case $GOOS in
-			windows)
-				cp syncthing.exe "$buildDir/$name.exe"
-				mv syncthing.exe "$name"
-				zip -qr "$buildDir/$name.zip" "$name"
-				;;
-			*)
-				cp syncthing "$buildDir/$name"
-				mv syncthing "$name"
-				tar zcf "$buildDir/$name.tar.gz" "$name"
-				;;
-		esac
-		rm -r  "$name"
-	done
-fi
+zipDist() {
+	name="$1"
+	mkdir -p "$name"
+	cp syncthing.exe "${distFiles[@]}" "$name"
+	zip -r "$name.zip" "$name"
+	rm -rf "$name"
+}
+
+case "$1" in
+	"")
+		build
+		;;
+
+	tar)
+		rm -f *.tar.gz *.zip
+		prepare
+		test || exit 1
+		build
+
+		eval $(go env)
+		name="syncthing-$GOOS-$GOARCH-$version"
+
+		tarDist "$name"
+		;;
+
+	all)
+		rm -f *.tar.gz *.zip
+		prepare
+		test || exit 1
+
+		export GOARM=7
+		for os in darwin-amd64 linux-amd64 linux-arm freebsd-amd64 windows-amd64 ; do
+			export GOOS=${os%-*}
+			export GOARCH=${os#*-}
+
+			build
+			
+			name="syncthing-$os-$version"
+			case $GOOS in
+				windows)
+					zipDist "$name"
+					rm -f syncthing.exe
+					;;
+				*)
+					tarDist "$name"
+					rm -f syncthing
+					;;
+			esac
+		done
+		;;
+
+	upload)
+		tag=$(git describe)
+		shopt -s nullglob
+		for f in *gz *zip ; do
+			relup calmh/syncthing "$tag" "$f"
+		done
+		;;
+
+	*)
+		echo "Unknown build parameter $1"
+		;;
+esac

cid/cid.go

@@ -0,0 +1,42 @@
+package cid
+
+type Map struct {
+	toCid  map[string]int
+	toName []string
+}
+
+func NewMap() *Map {
+	return &Map{
+		toCid: make(map[string]int),
+	}
+}
+
+func (m *Map) Get(name string) int {
+	cid, ok := m.toCid[name]
+	if ok {
+		return cid
+	}
+
+	// Find a free slot to get a new ID
+	for i, n := range m.toName {
+		if n == "" {
+			m.toName[i] = name
+			m.toCid[name] = i
+			return i
+		}
+	}
+
+	// Add it to the end since we didn't find a free slot
+	m.toName = append(m.toName, name)
+	cid = len(m.toName) - 1
+	m.toCid[name] = cid
+	return cid
+}
+
+func (m *Map) Clear(name string) {
+	cid, ok := m.toCid[name]
+	if ok {
+		m.toName[cid] = ""
+		delete(m.toCid, name)
+	}
+}

cmd/.gitignore

@@ -0,0 +1 @@
+!syncthing

cmd/syncthing/config.go

@@ -36,7 +36,7 @@ type OptionsConfiguration struct {
 	FollowSymlinks     bool     `xml:"followSymlinks" default:"true" ini:"follow-symlinks"`
 	GUIEnabled         bool     `xml:"guiEnabled" default:"true" ini:"gui-enabled"`
 	GUIAddress         string   `xml:"guiAddress" default:"127.0.0.1:8080" ini:"gui-address"`
-	GlobalAnnServer    string   `xml:"globalAnnounceServer" default:"syncthing.nym.se:22025" ini:"global-announce-server"`
+	GlobalAnnServer    string   `xml:"globalAnnounceServer" default:"announce.syncthing.net:22025" ini:"global-announce-server"`
 	GlobalAnnEnabled   bool     `xml:"globalAnnounceEnabled" default:"true" ini:"global-announce-enabled"`
 	LocalAnnEnabled    bool     `xml:"localAnnounceEnabled" default:"true" ini:"local-announce-enabled"`
 	ParallelRequests   int      `xml:"parallelRequests" default:"16" ini:"parallel-requests"`
@@ -60,11 +60,6 @@ func setDefaults(data interface{}) error {
 			case string:
 				f.SetString(v)
 
-			case []string:
-				rv := reflect.MakeSlice(reflect.TypeOf([]string{}), 1, 1)
-				rv.Index(0).SetString(v)
-				f.Set(rv)
-
 			case int:
 				i, err := strconv.ParseInt(v, 10, 64)
 				if err != nil {
@@ -75,6 +70,11 @@ func setDefaults(data interface{}) error {
 			case bool:
 				f.SetBool(v == "true")
 
+			case []string:
+				// We don't do anything with string slices here. Any default
+				// we set will be appended to by the XML decoder, so we fill
+				// those after decoding.
+
 			default:
 				panic(f.Type())
 			}
@@ -83,6 +83,30 @@ func setDefaults(data interface{}) error {
 	return nil
 }
 
+// fillNilSlices sets default value on slices that are still nil.
+func fillNilSlices(data interface{}) error {
+	s := reflect.ValueOf(data).Elem()
+	t := s.Type()
+
+	for i := 0; i < s.NumField(); i++ {
+		f := s.Field(i)
+		tag := t.Field(i).Tag
+
+		v := tag.Get("default")
+		if len(v) > 0 {
+			switch f.Interface().(type) {
+			case []string:
+				if f.IsNil() {
+					rv := reflect.MakeSlice(reflect.TypeOf([]string{}), 1, 1)
+					rv.Index(0).SetString(v)
+					f.Set(rv)
+				}
+			}
+		}
+	}
+	return nil
+}
+
 func readConfigINI(m map[string]string, data interface{}) error {
 	s := reflect.ValueOf(data).Elem()
 	t := s.Type()
@@ -154,6 +178,8 @@ func readConfigXML(rd io.Reader) (Configuration, error) {
 		err = xml.NewDecoder(rd).Decode(&cfg)
 	}
 
+	fillNilSlices(&cfg.Options)
+
 	cfg.Options.ListenAddress = uniqueStrings(cfg.Options.ListenAddress)
 	return cfg, err
 }

cmd/syncthing/config_test.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"bytes"
+	"io"
+	"reflect"
+	"testing"
+)
+
+func TestDefaultValues(t *testing.T) {
+	expected := OptionsConfiguration{
+		ListenAddress:      []string{":22000"},
+		ReadOnly:           false,
+		AllowDelete:        true,
+		FollowSymlinks:     true,
+		GUIEnabled:         true,
+		GUIAddress:         "127.0.0.1:8080",
+		GlobalAnnServer:    "announce.syncthing.net:22025",
+		GlobalAnnEnabled:   true,
+		LocalAnnEnabled:    true,
+		ParallelRequests:   16,
+		MaxSendKbps:        0,
+		RescanIntervalS:    60,
+		ReconnectIntervalS: 60,
+		MaxChangeKbps:      1000,
+	}
+
+	cfg, err := readConfigXML(bytes.NewReader(nil))
+	if err != io.EOF {
+		t.Error(err)
+	}
+
+	if !reflect.DeepEqual(cfg.Options, expected) {
+		t.Errorf("Default config differs;\n  E: %#v\n  A: %#v", expected, cfg.Options)
+	}
+}
+
+func TestNoListenAddress(t *testing.T) {
+	data := []byte(`<configuration version="1">
+    <repository directory="~/Sync">
+        <node id="..." name="...">
+            <address>dynamic</address>
+        </node>
+    </repository>
+    <options>
+        <listenAddress></listenAddress>
+    </options>
+</configuration>
+`)
+
+	cfg, err := readConfigXML(bytes.NewReader(data))
+	if err != nil {
+		t.Error(err)
+	}
+
+	expected := []string{""}
+	if !reflect.DeepEqual(cfg.Options.ListenAddress, expected) {
+		t.Errorf("Unexpected ListenAddress %#v", cfg.Options.ListenAddress)
+	}
+}
+
+func TestOverriddenValues(t *testing.T) {
+	data := []byte(`<configuration version="1">
+    <repository directory="~/Sync">
+        <node id="..." name="...">
+            <address>dynamic</address>
+        </node>
+    </repository>
+    <options>
+       <listenAddress>:23000</listenAddress>
+        <readOnly>true</readOnly>
+        <allowDelete>false</allowDelete>
+        <followSymlinks>false</followSymlinks>
+        <guiEnabled>false</guiEnabled>
+        <guiAddress>125.2.2.2:8080</guiAddress>
+        <globalAnnounceServer>syncthing.nym.se:22025</globalAnnounceServer>
+        <globalAnnounceEnabled>false</globalAnnounceEnabled>
+        <localAnnounceEnabled>false</localAnnounceEnabled>
+        <parallelRequests>32</parallelRequests>
+        <maxSendKbps>1234</maxSendKbps>
+        <rescanIntervalS>600</rescanIntervalS>
+        <reconnectionIntervalS>6000</reconnectionIntervalS>
+        <maxChangeKbps>2345</maxChangeKbps>
+    </options>
+</configuration>
+`)
+
+	expected := OptionsConfiguration{
+		ListenAddress:      []string{":23000"},
+		ReadOnly:           true,
+		AllowDelete:        false,
+		FollowSymlinks:     false,
+		GUIEnabled:         false,
+		GUIAddress:         "125.2.2.2:8080",
+		GlobalAnnServer:    "syncthing.nym.se:22025",
+		GlobalAnnEnabled:   false,
+		LocalAnnEnabled:    false,
+		ParallelRequests:   32,
+		MaxSendKbps:        1234,
+		RescanIntervalS:    600,
+		ReconnectIntervalS: 6000,
+		MaxChangeKbps:      2345,
+	}
+
+	cfg, err := readConfigXML(bytes.NewReader(data))
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !reflect.DeepEqual(cfg.Options, expected) {
+		t.Errorf("Overridden config differs;\n  E: %#v\n  A: %#v", expected, cfg.Options)
+	}
+}

cmd/syncthing/main.go

@@ -40,7 +40,7 @@ var (
 )
 
 func main() {
-	flag.StringVar(&confDir, "home", "~/.syncthing", "Set configuration directory")
+	flag.StringVar(&confDir, "home", getDefaultConfDir(), "Set configuration directory")
 	flag.StringVar(&trace, "debug.trace", "", "(connect,net,idx,file,pull)")
 	flag.StringVar(&profiler, "debug.profiler", "", "(addr)")
 	flag.BoolVar(&showVersion, "version", false, "Show version")
@@ -138,7 +138,7 @@ func main() {
 		cfg, err = readConfigXML(nil)
 		cfg.Repositories = []RepositoryConfiguration{
 			{
-				Directory: "~/Sync",
+				Directory: path.Join(getHomeDir(), "Sync"),
 				Nodes: []NodeConfiguration{
 					{NodeID: myID, Addresses: []string{"dynamic"}},
 				},
@@ -328,6 +328,13 @@ func saveConfigLoop(cfgFile string) {
 			continue
 		}
 
+		if runtime.GOOS == "windows" {
+			err := os.Remove(cfgFile)
+			if err != nil && !os.IsNotExist(err) {
+				warnln(err)
+			}
+		}
+
 		err = os.Rename(cfgFile+".tmp", cfgFile)
 		if err != nil {
 			warnln(err)
@@ -554,16 +561,38 @@ func ensureDir(dir string, mode int) {
 }
 
 func expandTilde(p string) string {
+	if runtime.GOOS == "windows" {
+		return p
+	}
+
 	if strings.HasPrefix(p, "~/") {
-		return strings.Replace(p, "~", getHomeDir(), 1)
+		return strings.Replace(p, "~", getUnixHomeDir(), 1)
 	}
 	return p
 }
 
-func getHomeDir() string {
+func getUnixHomeDir() string {
 	home := os.Getenv("HOME")
 	if home == "" {
 		fatalln("No home directory?")
 	}
 	return home
 }
+
+func getHomeDir() string {
+	if runtime.GOOS == "windows" {
+		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
+		if home == "" {
+			home = os.Getenv("USERPROFILE")
+		}
+		return home
+	}
+	return getUnixHomeDir()
+}
+
+func getDefaultConfDir() string {
+	if runtime.GOOS == "windows" {
+		return path.Join(os.Getenv("AppData"), "syncthing")
+	}
+	return expandTilde("~/.syncthing")
+}

cmd/syncthing/model.go

@@ -191,7 +191,10 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
 			}
 		}
 
-		ci.Completion = int(100 * have / tot)
+		ci.Completion = 100
+		if tot != 0 {
+			ci.Completion = int(100 * have / tot)
+		}
 
 		res[node] = ci
 	}

cmd/syncthing/walk.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"time"
 
@@ -39,6 +40,9 @@ func (f File) NewerThan(o File) bool {
 }
 
 func isTempName(name string) bool {
+	if runtime.GOOS == "windows" {
+		name = filepath.ToSlash(name)
+	}
 	return strings.HasPrefix(path.Base(name), ".syncthing.")
 }
 

discover/discover.go

@@ -80,24 +80,75 @@ func (d *Discoverer) sendAnnouncements() {
 	var errCounter = 0
 	var err error
 
+	remote := &net.UDPAddr{
+		IP:   net.IP{255, 255, 255, 255},
+		Port: AnnouncementPort,
+	}
+
 	for errCounter < maxErrors {
-		for _, ipStr := range allBroadcasts() {
-			var addrStr = ipStr + ":21025"
+		intfs, err := net.Interfaces()
+		if err != nil {
+			log.Printf("discover/listInterfaces: %v; no local announcements", err)
+			return
+		}
 
-			remote, err := net.ResolveUDPAddr("udp4", addrStr)
-			if err != nil {
-				log.Printf("discover/external: %v; no external announcements", err)
-				return
-			}
+		for _, intf := range intfs {
+			if intf.Flags&(net.FlagBroadcast|net.FlagLoopback) == net.FlagBroadcast {
+				addrs, err := intf.Addrs()
+				if err != nil {
+					log.Println("discover/listAddrs: warning:", err)
+					errCounter++
+					continue
+				}
 
-			if Debug {
-				log.Println("send announcement -> ", remote)
-			}
-			_, _, err = d.conn.WriteMsgUDP(buf, nil, remote)
-			if err != nil {
-				log.Println("discover/write: warning:", err)
-				errCounter++
-			} else {
+				var srcAddr string
+				for _, addr := range addrs {
+					if strings.Contains(addr.String(), ".") {
+						// Found an IPv4 adress
+						parts := strings.Split(addr.String(), "/")
+						srcAddr = parts[0]
+						break
+					}
+				}
+				if len(srcAddr) == 0 {
+					if Debug {
+						log.Println("discover: debug: no source address found on interface", intf.Name)
+					}
+					continue
+				}
+
+				iaddr, err := net.ResolveUDPAddr("udp4", srcAddr+":0")
+				if err != nil {
+					log.Println("discover/resolve: warning:", err)
+					errCounter++
+					continue
+				}
+
+				conn, err := net.ListenUDP("udp4", iaddr)
+				if err != nil {
+					log.Println("discover/listen: warning:", err)
+					errCounter++
+					continue
+				}
+
+				if Debug {
+					log.Println("discover: debug: send announcement from", conn.LocalAddr(), "to", remote, "on", intf.Name)
+				}
+
+				_, err = conn.WriteTo(buf, remote)
+				if err != nil {
+					// Some interfaces don't seem to support broadcast even though the flags claims they do, i.e. vmnet
+					conn.Close()
+
+					if Debug {
+						log.Println("discover/write: debug:", err)
+					}
+
+					errCounter++
+					continue
+				}
+
+				conn.Close()
 				errCounter = 0
 			}
 		}
@@ -125,7 +176,7 @@ func (d *Discoverer) sendExtAnnouncements() {
 		if Debug {
 			log.Println("send announcement -> ", remote)
 		}
-		_, _, err = d.conn.WriteMsgUDP(buf, nil, remote)
+		_, err = d.conn.WriteTo(buf, remote)
 		if err != nil {
 			log.Println("discover/write: warning:", err)
 			errCounter++
@@ -289,38 +340,3 @@ func ipStr(ip []byte) string {
 	}
 	return strings.Join(ss, s)
 }
-
-func allBroadcasts() []string {
-	var bcasts = make(map[string]bool)
-	addrs, err := net.InterfaceAddrs()
-	if err != nil {
-		return nil
-	}
-
-	for _, addr := range addrs {
-		switch {
-		case strings.HasPrefix(addr.String(), "127."):
-			// Ignore v4 localhost
-
-		case strings.Contains(addr.String(), ":"):
-			// Ignore all v6, because we need link local multicast there which I haven't implemented
-
-		default:
-			if in, ok := addr.(*net.IPNet); ok {
-				il := len(in.IP) - 1
-				ml := len(in.Mask) - 1
-				for i := range in.Mask {
-					in.IP[il-i] = in.IP[il-i] | ^in.Mask[ml-i]
-				}
-				parts := strings.Split(in.String(), "/")
-				bcasts[parts[0]] = true
-			}
-		}
-	}
-
-	var l []string
-	for ip := range bcasts {
-		l = append(l, ip)
-	}
-	return l
-}

files/set.go

@@ -0,0 +1,173 @@
+package fileset
+
+import "sync"
+
+type File struct {
+	Key      Key
+	Modified int64
+	Flags    uint32
+	Data     interface{}
+}
+
+type Key struct {
+	Name    string
+	Version uint32
+}
+
+type fileRecord struct {
+	Usage int
+	File  File
+}
+
+type bitset uint64
+
+func (a Key) newerThan(b Key) bool {
+	return a.Version > b.Version
+}
+
+type Set struct {
+	mutex              sync.RWMutex
+	files              map[Key]fileRecord
+	remoteKey          [64]map[string]Key
+	globalAvailability map[string]bitset
+	globalKey          map[string]Key
+}
+
+func NewSet() *Set {
+	var m = Set{
+		files:              make(map[Key]fileRecord),
+		globalAvailability: make(map[string]bitset),
+		globalKey:          make(map[string]Key),
+	}
+	return &m
+}
+
+func (m *Set) AddLocal(fs []File) {
+	m.mutex.Lock()
+	m.unlockedAddRemote(0, fs)
+	m.mutex.Unlock()
+}
+
+func (m *Set) SetLocal(fs []File) {
+	m.mutex.Lock()
+	m.unlockedSetRemote(0, fs)
+	m.mutex.Unlock()
+}
+
+func (m *Set) AddRemote(cid uint, fs []File) {
+	if cid < 1 || cid > 63 {
+		panic("Connection ID must be in the range 1 - 63 inclusive")
+	}
+	m.mutex.Lock()
+	m.unlockedAddRemote(cid, fs)
+	m.mutex.Unlock()
+}
+
+func (m *Set) SetRemote(cid uint, fs []File) {
+	if cid < 1 || cid > 63 {
+		panic("Connection ID must be in the range 1 - 63 inclusive")
+	}
+	m.mutex.Lock()
+	m.unlockedSetRemote(cid, fs)
+	m.mutex.Unlock()
+}
+
+func (m *Set) unlockedAddRemote(cid uint, fs []File) {
+	remFiles := m.remoteKey[cid]
+	for _, f := range fs {
+		n := f.Key.Name
+
+		if ck, ok := remFiles[n]; ok && ck == f.Key {
+			// The remote already has exactly this file, skip it
+			continue
+		}
+
+		remFiles[n] = f.Key
+
+		// Keep the block list or increment the usage
+		if br, ok := m.files[f.Key]; !ok {
+			m.files[f.Key] = fileRecord{
+				Usage: 1,
+				File:  f,
+			}
+		} else {
+			br.Usage++
+			m.files[f.Key] = br
+		}
+
+		// Update global view
+		gk, ok := m.globalKey[n]
+		switch {
+		case ok && f.Key == gk:
+			av := m.globalAvailability[n]
+			av |= 1 << cid
+			m.globalAvailability[n] = av
+		case f.Key.newerThan(gk):
+			m.globalKey[n] = f.Key
+			m.globalAvailability[n] = 1 << cid
+		}
+	}
+}
+
+func (m *Set) unlockedSetRemote(cid uint, fs []File) {
+	// Decrement usage for all files belonging to this remote, and remove
+	// those that are no longer needed.
+	for _, fk := range m.remoteKey[cid] {
+		br, ok := m.files[fk]
+		switch {
+		case ok && br.Usage == 1:
+			delete(m.files, fk)
+		case ok && br.Usage > 1:
+			br.Usage--
+			m.files[fk] = br
+		}
+	}
+
+	// Clear existing remote remoteKey
+	m.remoteKey[cid] = make(map[string]Key)
+
+	// Recalculate global based on all remaining remoteKey
+	for n := range m.globalKey {
+		var nk Key    // newest key
+		var na bitset // newest availability
+
+		for i, rem := range m.remoteKey {
+			if rk, ok := rem[n]; ok {
+				switch {
+				case rk == nk:
+					na |= 1 << uint(i)
+				case rk.newerThan(nk):
+					nk = rk
+					na = 1 << uint(i)
+				}
+			}
+		}
+
+		if na != 0 {
+			// Someone had the file
+			m.globalKey[n] = nk
+			m.globalAvailability[n] = na
+		} else {
+			// Noone had the file
+			delete(m.globalKey, n)
+			delete(m.globalAvailability, n)
+		}
+	}
+
+	// Add new remote remoteKey to the mix
+	m.unlockedAddRemote(cid, fs)
+}
+
+func (m *Set) Need(cid uint) []File {
+	var fs []File
+	m.mutex.Lock()
+
+	for name, gk := range m.globalKey {
+		if gk.newerThan(m.remoteKey[cid][name]) {
+			fs = append(fs, m.files[gk].File)
+		}
+	}
+
+	m.mutex.Unlock()
+	return fs
+}

files/set_test.go

@@ -0,0 +1,207 @@
+package fileset
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+func TestGlobalSet(t *testing.T) {
+	m := NewSet()
+
+	local := []File{
+		File{Key{"a", 1000}, 0, 0, nil},
+		File{Key{"b", 1000}, 0, 0, nil},
+		File{Key{"c", 1000}, 0, 0, nil},
+		File{Key{"d", 1000}, 0, 0, nil},
+	}
+
+	remote := []File{
+		File{Key{"a", 1000}, 0, 0, nil},
+		File{Key{"b", 1001}, 0, 0, nil},
+		File{Key{"c", 1002}, 0, 0, nil},
+		File{Key{"e", 1000}, 0, 0, nil},
+	}
+
+	expectedGlobal := map[string]Key{
+		"a": local[0].Key,
+		"b": remote[1].Key,
+		"c": remote[2].Key,
+		"d": local[3].Key,
+		"e": remote[3].Key,
+	}
+
+	m.SetLocal(local)
+	m.SetRemote(1, remote)
+
+	if !reflect.DeepEqual(m.globalKey, expectedGlobal) {
+		t.Errorf("Global incorrect;\n%v !=\n%v", m.globalKey, expectedGlobal)
+	}
+
+	if lb := len(m.files); lb != 7 {
+		t.Errorf("Num files incorrect %d != 7\n%v", lb, m.files)
+	}
+}
+
+func BenchmarkSetLocal10k(b *testing.B) {
+	m := NewSet()
+
+	var local []File
+	for i := 0; i < 10000; i++ {
+		local = append(local, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
+	}
+
+	var remote []File
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
+	}
+
+	m.SetRemote(1, remote)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.SetLocal(local)
+	}
+}
+
+func BenchmarkSetLocal10(b *testing.B) {
+	m := NewSet()
+
+	var local []File
+	for i := 0; i < 10; i++ {
+		local = append(local, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
+	}
+
+	var remote []File
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
+	}
+
+	m.SetRemote(1, remote)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		m.SetLocal(local)
+	}
+}
+
+func BenchmarkAddLocal10k(b *testing.B) {
+	m := NewSet()
+
+	var local []File
+	for i := 0; i < 10000; i++ {
+		local = append(local, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
+	}
+
+	var remote []File
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
+	}
+
+	m.SetRemote(1, remote)
+	m.SetLocal(local)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		for j := range local {
+			local[j].Key.Version++
+		}
+		b.StartTimer()
+		m.AddLocal(local)
+	}
+}
+
+func BenchmarkAddLocal10(b *testing.B) {
+	m := NewSet()
+
+	var local []File
+	for i := 0; i < 10; i++ {
+		local = append(local, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
+	}
+
+	var remote []File
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, File{Key{fmt.Sprintf("file%d"), 1000}, 0, 0, nil})
+	}
+
+	m.SetRemote(1, remote)
+	m.SetLocal(local)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		for j := range local {
+			local[j].Key.Version++
+		}
+		m.AddLocal(local)
+	}
+}
+
+func TestGlobalReset(t *testing.T) {
+	m := NewSet()
+
+	local := []File{
+		File{Key{"a", 1000}, 0, 0, nil},
+		File{Key{"b", 1000}, 0, 0, nil},
+		File{Key{"c", 1000}, 0, 0, nil},
+		File{Key{"d", 1000}, 0, 0, nil},
+	}
+
+	remote := []File{
+		File{Key{"a", 1000}, 0, 0, nil},
+		File{Key{"b", 1001}, 0, 0, nil},
+		File{Key{"c", 1002}, 0, 0, nil},
+		File{Key{"e", 1000}, 0, 0, nil},
+	}
+
+	expectedGlobalKey := map[string]Key{
+		"a": local[0].Key,
+		"b": local[1].Key,
+		"c": local[2].Key,
+		"d": local[3].Key,
+	}
+
+	m.SetLocal(local)
+	m.SetRemote(1, remote)
+	m.SetRemote(1, nil)
+
+	if !reflect.DeepEqual(m.globalKey, expectedGlobalKey) {
+		t.Errorf("Global incorrect;\n%v !=\n%v", m.globalKey, expectedGlobalKey)
+	}
+
+	if lb := len(m.files); lb != 4 {
+		t.Errorf("Num files incorrect %d != 4\n%v", lb, m.files)
+	}
+}
+
+func TestNeed(t *testing.T) {
+	m := NewSet()
+
+	local := []File{
+		File{Key{"a", 1000}, 0, 0, nil},
+		File{Key{"b", 1000}, 0, 0, nil},
+		File{Key{"c", 1000}, 0, 0, nil},
+		File{Key{"d", 1000}, 0, 0, nil},
+	}
+
+	remote := []File{
+		File{Key{"a", 1000}, 0, 0, nil},
+		File{Key{"b", 1001}, 0, 0, nil},
+		File{Key{"c", 1002}, 0, 0, nil},
+		File{Key{"e", 1000}, 0, 0, nil},
+	}
+
+	shouldNeed := []File{
+		File{Key{"b", 1001}, 0, 0, nil},
+		File{Key{"c", 1002}, 0, 0, nil},
+		File{Key{"e", 1000}, 0, 0, nil},
+	}
+
+	m.SetLocal(local)
+	m.SetRemote(1, remote)
+
+	need := m.Need(0)
+	if !reflect.DeepEqual(need, shouldNeed) {
+		t.Errorf("Need incorrect;\n%v !=\n%v", need, shouldNeed)
+	}
+}