Fixup integration test script

Godeps/Godeps.json

@@ -2,7 +2,10 @@
 	"ImportPath": "github.com/calmh/syncthing",
 	"GoVersion": "go1.2.1",
 	"Packages": [
-		"./cmd/syncthing"
+		"./cmd/syncthing",
+		"./cmd/assets",
+		"./cmd/stcli",
+		"./discover/cmd/discosrv"
 	],
 	"Deps": [
 		{
@@ -15,10 +18,6 @@
 			"Comment": "null-81",
 			"Rev": "9cbe983aed9b0dfc73954433fead5e00866342ac"
 		},
-		{
-			"ImportPath": "github.com/calmh/ini",
-			"Rev": "386c4240a9684d91d9ec4d93651909b49c7269e1"
-		},
 		{
 			"ImportPath": "github.com/codegangsta/inject",
 			"Rev": "9aea7a2fa5b79ef7fc00f63a575e72df33b4e886"
@@ -28,6 +27,15 @@
 			"Comment": "v0.1-142-g8659df7",
 			"Rev": "8659df7a51aebe6c6120268cd5a8b4c34fa8441a"
 		},
+		{
+			"ImportPath": "github.com/codegangsta/martini-contrib/auth",
+			"Comment": "v0.1-159-g8ce6181",
+			"Rev": "8ce6181c2609699e4c7cd30994b76a850a9cdadc"
+		},
+		{
+			"ImportPath": "github.com/golang/groupcache/lru",
+			"Rev": "d781998583680cda80cf61e0b37dd0cd8da2eb52"
+		},
 		{
 			"ImportPath": "github.com/juju/ratelimit",
 			"Rev": "cbaa435c80a9716e086f25d409344b26c4039358"

Godeps/_workspace/src/github.com/calmh/ini/LICENSE

@@ -1,19 +0,0 @@
-Copyright (C) 2013 Jakob Borg
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-- The above copyright notice and this permission notice shall be included in
-  all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

Godeps/_workspace/src/github.com/calmh/ini/README.md

@@ -1,39 +0,0 @@
-ini [![Build Status](https://drone.io/github.com/calmh/ini/status.png)](https://drone.io/github.com/calmh/ini/latest)
-===
-
-Yet another .INI file parser / writer. Created because the existing ones
-were either not general enough (allowing easy access to all parts of the
-original file) or made annoying assumptions about the format. And
-probably equal parts NIH. You might want to just write your own instead
-of using this one, you know that's where you'll end up in the end
-anyhow.
-
-Documentation
--------------
-
-http://godoc.org/github.com/calmh/ini
-
-Example
--------
-
-```go
-fd, _ := os.Open("foo.ini")
-cfg := ini.Parse(fd)
-fd.Close()
-
-val := cfg.Get("general", "foo")
-cfg.Set("general", "bar", "baz")
-
-fd, _ = os.Create("bar.ini")
-err := cfg.Write(fd)
-if err != nil {
-	// ...
-}
-err = fd.Close()
-
-```
-
-License
--------
-
-MIT

Godeps/_workspace/src/github.com/calmh/ini/ini.go

@@ -1,235 +0,0 @@
-// Package ini provides trivial parsing of .INI format files.
-package ini
-
-import (
-	"bufio"
-	"fmt"
-	"io"
-	"regexp"
-	"strconv"
-	"strings"
-)
-
-// Config is a parsed INI format file.
-type Config struct {
-	sections []section
-	comments []string
-}
-
-type section struct {
-	name     string
-	comments []string
-	options  []option
-}
-
-type option struct {
-	name, value string
-}
-
-var (
-	iniSectionRe = regexp.MustCompile(`^\[(.+)\]$`)
-	iniOptionRe  = regexp.MustCompile(`^([^\s=]+)\s*=\s*(.+?)$`)
-)
-
-// Sections returns the list of sections in the file.
-func (c *Config) Sections() []string {
-	var sections []string
-	for _, sect := range c.sections {
-		sections = append(sections, sect.name)
-	}
-	return sections
-}
-
-// Options returns the list of options in a given section.
-func (c *Config) Options(section string) []string {
-	var options []string
-	for _, sect := range c.sections {
-		if sect.name == section {
-			for _, opt := range sect.options {
-				options = append(options, opt.name)
-			}
-			break
-		}
-	}
-	return options
-}
-
-// OptionMap returns the map option => value for a given section.
-func (c *Config) OptionMap(section string) map[string]string {
-	options := make(map[string]string)
-	for _, sect := range c.sections {
-		if sect.name == section {
-			for _, opt := range sect.options {
-				options[opt.name] = opt.value
-			}
-			break
-		}
-	}
-	return options
-}
-
-// Comments returns the list of comments in a given section.
-// For the empty string, returns the file comments.
-func (c *Config) Comments(section string) []string {
-	if section == "" {
-		return c.comments
-	}
-	for _, sect := range c.sections {
-		if sect.name == section {
-			return sect.comments
-		}
-	}
-	return nil
-}
-
-// AddComments appends the comment to the list of comments for the section.
-func (c *Config) AddComment(sect, comment string) {
-	if sect == "" {
-		c.comments = append(c.comments, comment)
-		return
-	}
-
-	for i, s := range c.sections {
-		if s.name == sect {
-			c.sections[i].comments = append(s.comments, comment)
-			return
-		}
-	}
-
-	c.sections = append(c.sections, section{
-		name:     sect,
-		comments: []string{comment},
-	})
-}
-
-// Parse reads the given io.Reader and returns a parsed Config object.
-func Parse(stream io.Reader) Config {
-	var cfg Config
-	var curSection string
-
-	scanner := bufio.NewScanner(bufio.NewReader(stream))
-	for scanner.Scan() {
-		line := strings.TrimSpace(scanner.Text())
-		if strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") {
-			comment := strings.TrimLeft(line, ";# ")
-			cfg.AddComment(curSection, comment)
-		} else if len(line) > 0 {
-			if m := iniSectionRe.FindStringSubmatch(line); len(m) > 0 {
-				curSection = m[1]
-			} else if m := iniOptionRe.FindStringSubmatch(line); len(m) > 0 {
-				key := m[1]
-				val := m[2]
-				if !strings.Contains(val, "\"") {
-					// If val does not contain any quote characers, we can make it
-					// a quoted string and safely let strconv.Unquote sort out any
-					// escapes
-					val = "\"" + val + "\""
-				}
-				if val[0] == '"' {
-					val, _ = strconv.Unquote(val)
-				}
-
-				cfg.Set(curSection, key, val)
-			}
-		}
-	}
-	return cfg
-}
-
-// Write writes the sections and options to the io.Writer in INI format.
-func (c *Config) Write(out io.Writer) error {
-	for _, cmt := range c.comments {
-		fmt.Fprintln(out, "; "+cmt)
-	}
-	if len(c.comments) > 0 {
-		fmt.Fprintln(out)
-	}
-
-	for _, sect := range c.sections {
-		fmt.Fprintf(out, "[%s]\n", sect.name)
-		for _, cmt := range sect.comments {
-			fmt.Fprintln(out, "; "+cmt)
-		}
-		for _, opt := range sect.options {
-			val := opt.value
-			if len(val) == 0 {
-				continue
-			}
-
-			// Quote the string if it begins or ends with space
-			needsQuoting := val[0] == ' ' || val[len(val)-1] == ' '
-
-			if !needsQuoting {
-				// Quote the string if it contains any unprintable characters
-				for _, r := range val {
-					if !strconv.IsPrint(r) {
-						needsQuoting = true
-						break
-					}
-				}
-			}
-
-			if needsQuoting {
-				val = strconv.Quote(val)
-			}
-
-			fmt.Fprintf(out, "%s=%s\n", opt.name, val)
-		}
-		fmt.Fprintln(out)
-	}
-	return nil
-}
-
-// Get gets the value from the specified section and key name, or the empty
-// string if either the section or the key is missing.
-func (c *Config) Get(section, key string) string {
-	for _, sect := range c.sections {
-		if sect.name == section {
-			for _, opt := range sect.options {
-				if opt.name == key {
-					return opt.value
-				}
-			}
-			return ""
-		}
-	}
-	return ""
-}
-
-// Set sets a value for an option in a section. If the option exists, it's
-// value will be overwritten. If the option does not exist, it will be added.
-// If the section does not exist, it will be added and the option added to it.
-func (c *Config) Set(sectionName, key, value string) {
-	for i, sect := range c.sections {
-		if sect.name == sectionName {
-			for j, opt := range sect.options {
-				if opt.name == key {
-					c.sections[i].options[j].value = value
-					return
-				}
-			}
-			c.sections[i].options = append(sect.options, option{key, value})
-			return
-		}
-	}
-
-	c.sections = append(c.sections, section{
-		name:    sectionName,
-		options: []option{{key, value}},
-	})
-}
-
-// Delete removes the option from the specified section.
-func (c *Config) Delete(section, key string) {
-	for sn, sect := range c.sections {
-		if sect.name == section {
-			for i, opt := range sect.options {
-				if opt.name == key {
-					c.sections[sn].options = append(sect.options[:i], sect.options[i+1:]...)
-					return
-				}
-			}
-			return
-		}
-	}
-}

Godeps/_workspace/src/github.com/calmh/ini/ini_test.go

@@ -1,214 +0,0 @@
-package ini_test
-
-import (
-	"bytes"
-	"strings"
-	"testing"
-
-	"github.com/calmh/ini"
-)
-
-func TestParseValues(t *testing.T) {
-	strs := []string{
-		`[general]`,
-		`k1=v1`,
-		`k2 = v2`,
-		` k3 = v3 `,
-		`k4=" quoted spaces "`,
-		`k5 = " quoted spaces " `,
-		`k6 = with\nnewline`,
-		`k7 = "with\nnewline"`,
-		`k8 = a "quoted" word`,
-		`k9 = "a \"quoted\" word"`,
-	}
-	buf := bytes.NewBufferString(strings.Join(strs, "\n"))
-	cfg := ini.Parse(buf)
-
-	correct := map[string]string{
-		"k1": "v1",
-		"k2": "v2",
-		"k3": "v3",
-		"k4": " quoted spaces ",
-		"k5": " quoted spaces ",
-		"k6": "with\nnewline",
-		"k7": "with\nnewline",
-		"k8": "a \"quoted\" word",
-		"k9": "a \"quoted\" word",
-	}
-
-	for k, v := range correct {
-		if v2 := cfg.Get("general", k); v2 != v {
-			t.Errorf("Incorrect general.%s, %q != %q", k, v2, v)
-		}
-	}
-
-	if v := cfg.Get("general", "nonexistant"); v != "" {
-		t.Errorf("Unexpected non-empty value %q", v)
-	}
-}
-
-func TestParseComments(t *testing.T) {
-	strs := []string{
-		";file comment 1",   // No leading space
-		"; file comment 2 ", // Trailing space
-		";  file comment 3", // Multiple leading spaces
-		"[general]",
-		"; b general comment 1", // Comments in unsorted order
-		"somekey = somevalue",
-		"; a general comment 2",
-		"[other]",
-		"; other comment 1", // Comments in section with no values
-		"; other comment 2",
-		"[other2]",
-		"; other2 comment 1",
-		"; other2 comment 2", // Comments on last section
-		"somekey = somevalue",
-	}
-	buf := bytes.NewBufferString(strings.Join(strs, "\n"))
-
-	correct := map[string][]string{
-		"":        []string{"file comment 1", "file comment 2", "file comment 3"},
-		"general": []string{"b general comment 1", "a general comment 2"},
-		"other":   []string{"other comment 1", "other comment 2"},
-		"other2":  []string{"other2 comment 1", "other2 comment 2"},
-	}
-
-	cfg := ini.Parse(buf)
-
-	for section, comments := range correct {
-		cmts := cfg.Comments(section)
-		if len(cmts) != len(comments) {
-			t.Errorf("Incorrect number of comments for section %q: %d != %d", section, len(cmts), len(comments))
-		} else {
-			for i := range comments {
-				if cmts[i] != comments[i] {
-					t.Errorf("Incorrect comment: %q != %q", cmts[i], comments[i])
-				}
-			}
-		}
-	}
-}
-
-func TestWrite(t *testing.T) {
-	cfg := ini.Config{}
-	cfg.Set("general", "k1", "v1")
-	cfg.Set("general", "k2", "foo bar")
-	cfg.Set("general", "k3", " foo bar ")
-	cfg.Set("general", "k4", "foo\nbar")
-
-	var out bytes.Buffer
-	cfg.Write(&out)
-
-	correct := `[general]
-k1=v1
-k2=foo bar
-k3=" foo bar "
-k4="foo\nbar"
-
-`
-	if s := out.String(); s != correct {
-		t.Errorf("Incorrect written .INI:\n%s\ncorrect:\n%s", s, correct)
-	}
-}
-
-func TestSet(t *testing.T) {
-	buf := bytes.NewBufferString("[general]\nfoo=bar\nfoo2=bar2\n")
-	cfg := ini.Parse(buf)
-
-	cfg.Set("general", "foo", "baz")  // Overwrite existing
-	cfg.Set("general", "baz", "quux") // Create new value
-	cfg.Set("other", "baz2", "quux2") // Create new section + value
-
-	var out bytes.Buffer
-	cfg.Write(&out)
-
-	correct := `[general]
-foo=baz
-foo2=bar2
-baz=quux
-
-[other]
-baz2=quux2
-
-`
-
-	if s := out.String(); s != correct {
-		t.Errorf("Incorrect INI after set:\n%s", s)
-	}
-}
-
-func TestDelete(t *testing.T) {
-	buf := bytes.NewBufferString("[general]\nfoo=bar\nfoo2=bar2\nfoo3=baz\n")
-	cfg := ini.Parse(buf)
-	cfg.Delete("general", "foo")
-	out := new(bytes.Buffer)
-	cfg.Write(out)
-	correct := "[general]\nfoo2=bar2\nfoo3=baz\n\n"
-
-	if s := out.String(); s != correct {
-		t.Errorf("Incorrect INI after delete:\n%s", s)
-	}
-
-	buf = bytes.NewBufferString("[general]\nfoo=bar\nfoo2=bar2\nfoo3=baz\n")
-	cfg = ini.Parse(buf)
-	cfg.Delete("general", "foo2")
-	out = new(bytes.Buffer)
-	cfg.Write(out)
-	correct = "[general]\nfoo=bar\nfoo3=baz\n\n"
-
-	if s := out.String(); s != correct {
-		t.Errorf("Incorrect INI after delete:\n%s", s)
-	}
-
-	buf = bytes.NewBufferString("[general]\nfoo=bar\nfoo2=bar2\nfoo3=baz\n")
-	cfg = ini.Parse(buf)
-	cfg.Delete("general", "foo3")
-	out = new(bytes.Buffer)
-	cfg.Write(out)
-	correct = "[general]\nfoo=bar\nfoo2=bar2\n\n"
-
-	if s := out.String(); s != correct {
-		t.Errorf("Incorrect INI after delete:\n%s", s)
-	}
-}
-
-func TestSetManyEquals(t *testing.T) {
-	buf := bytes.NewBufferString("[general]\nfoo=bar==\nfoo2=bar2==\n")
-	cfg := ini.Parse(buf)
-
-	cfg.Set("general", "foo", "baz==")
-
-	var out bytes.Buffer
-	cfg.Write(&out)
-
-	correct := `[general]
-foo=baz==
-foo2=bar2==
-
-`
-
-	if s := out.String(); s != correct {
-		t.Errorf("Incorrect INI after set:\n%s", s)
-	}
-}
-
-func TestRewriteDuplicate(t *testing.T) {
-	buf := bytes.NewBufferString("[general]\nfoo=bar==\nfoo=bar2==\n")
-	cfg := ini.Parse(buf)
-
-	if v := cfg.Get("general", "foo"); v != "bar2==" {
-		t.Errorf("incorrect get %q", v)
-	}
-
-	var out bytes.Buffer
-	cfg.Write(&out)
-
-	correct := `[general]
-foo=bar2==
-
-`
-
-	if s := out.String(); s != correct {
-		t.Errorf("Incorrect INI after set:\n%s", s)
-	}
-}

Godeps/_workspace/src/github.com/codegangsta/martini-contrib/auth/README.md

@@ -0,0 +1,25 @@
+# auth
+Martini middleware/handler for http basic authentication.
+
+[API Reference](http://godoc.org/github.com/codegangsta/martini-contrib/auth)
+
+## Usage
+
+~~~ go
+import (
+  "github.com/codegangsta/martini"
+  "github.com/codegangsta/martini-contrib/auth"
+)
+
+func main() {
+  m := martini.Classic()
+  // authenticate every request
+  m.Use(auth.Basic("username", "secretpassword"))
+  m.Run()
+}
+
+~~~
+
+## Authors
+* [Jeremy Saenz](http://github.com/codegangsta)
+* [Brendon Murphy](http://github.com/bemurphy)

Godeps/_workspace/src/github.com/codegangsta/martini-contrib/auth/basic.go

@@ -0,0 +1,19 @@
+package auth
+
+import (
+	"encoding/base64"
+	"net/http"
+)
+
+// Basic returns a Handler that authenticates via Basic Auth. Writes a http.StatusUnauthorized
+// if authentication fails
+func Basic(username string, password string) http.HandlerFunc {
+	var siteAuth = base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
+	return func(res http.ResponseWriter, req *http.Request) {
+		auth := req.Header.Get("Authorization")
+		if !SecureCompare(auth, "Basic "+siteAuth) {
+			res.Header().Set("WWW-Authenticate", "Basic realm=\"Authorization Required\"")
+			http.Error(res, "Not Authorized", http.StatusUnauthorized)
+		}
+	}
+}

Godeps/_workspace/src/github.com/codegangsta/martini-contrib/auth/basic_test.go

@@ -0,0 +1,45 @@
+package auth
+
+import (
+	"encoding/base64"
+	"github.com/codegangsta/martini"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func Test_BasicAuth(t *testing.T) {
+	recorder := httptest.NewRecorder()
+
+	auth := "Basic " + base64.StdEncoding.EncodeToString([]byte("foo:bar"))
+
+	m := martini.New()
+	m.Use(Basic("foo", "bar"))
+	m.Use(func(res http.ResponseWriter, req *http.Request) {
+		res.Write([]byte("hello"))
+	})
+
+	r, _ := http.NewRequest("GET", "foo", nil)
+
+	m.ServeHTTP(recorder, r)
+
+	if recorder.Code != 401 {
+		t.Error("Response not 401")
+	}
+
+	if recorder.Body.String() == "hello" {
+		t.Error("Auth block failed")
+	}
+
+	recorder = httptest.NewRecorder()
+	r.Header.Set("Authorization", auth)
+	m.ServeHTTP(recorder, r)
+
+	if recorder.Code == 401 {
+		t.Error("Response is 401")
+	}
+
+	if recorder.Body.String() != "hello" {
+		t.Error("Auth failed, got: ", recorder.Body.String())
+	}
+}

Godeps/_workspace/src/github.com/codegangsta/martini-contrib/auth/util.go

@@ -0,0 +1,15 @@
+package auth
+
+import (
+	"crypto/subtle"
+)
+
+// SecureCompare performs a constant time compare of two strings to limit timing attacks.
+func SecureCompare(given string, actual string) bool {
+	if subtle.ConstantTimeEq(int32(len(given)), int32(len(actual))) == 1 {
+		return subtle.ConstantTimeCompare([]byte(given), []byte(actual)) == 1
+	} else {
+		/* Securely compare actual to itself to keep constant time, but always return false */
+		return subtle.ConstantTimeCompare([]byte(actual), []byte(actual)) == 1 && false
+	}
+}

Godeps/_workspace/src/github.com/codegangsta/martini-contrib/auth/util_test.go

@@ -0,0 +1,26 @@
+package auth
+
+import (
+	"testing"
+)
+
+var comparetests = []struct {
+	a   string
+	b   string
+	val bool
+}{
+	{"foo", "foo", true},
+	{"bar", "bar", true},
+	{"password", "password", true},
+	{"Foo", "foo", false},
+	{"foo", "foobar", false},
+	{"password", "pass", false},
+}
+
+func Test_SecureCompare(t *testing.T) {
+	for _, tt := range comparetests {
+		if SecureCompare(tt.a, tt.b) != tt.val {
+			t.Errorf("Expected SecureCompare(%v, %v) to return %v but did not", tt.a, tt.b, tt.val)
+		}
+	}
+}

Godeps/_workspace/src/github.com/golang/groupcache/lru/lru.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2013 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package lru implements an LRU cache.
+package lru
+
+import "container/list"
+
+// Cache is an LRU cache. It is not safe for concurrent access.
+type Cache struct {
+	// MaxEntries is the maximum number of cache entries before
+	// an item is evicted. Zero means no limit.
+	MaxEntries int
+
+	// OnEvicted optionally specificies a callback function to be
+	// executed when an entry is purged from the cache.
+	OnEvicted func(key Key, value interface{})
+
+	ll    *list.List
+	cache map[interface{}]*list.Element
+}
+
+// A Key may be any value that is comparable. See http://golang.org/ref/spec#Comparison_operators
+type Key interface{}
+
+type entry struct {
+	key   Key
+	value interface{}
+}
+
+// New creates a new Cache.
+// If maxEntries is zero, the cache has no limit and it's assumed
+// that eviction is done by the caller.
+func New(maxEntries int) *Cache {
+	return &Cache{
+		MaxEntries: maxEntries,
+		ll:         list.New(),
+		cache:      make(map[interface{}]*list.Element),
+	}
+}
+
+// Add adds a value to the cache.
+func (c *Cache) Add(key Key, value interface{}) {
+	if c.cache == nil {
+		c.cache = make(map[interface{}]*list.Element)
+		c.ll = list.New()
+	}
+	if ee, ok := c.cache[key]; ok {
+		c.ll.MoveToFront(ee)
+		ee.Value.(*entry).value = value
+		return
+	}
+	ele := c.ll.PushFront(&entry{key, value})
+	c.cache[key] = ele
+	if c.MaxEntries != 0 && c.ll.Len() > c.MaxEntries {
+		c.RemoveOldest()
+	}
+}
+
+// Get looks up a key's value from the cache.
+func (c *Cache) Get(key Key) (value interface{}, ok bool) {
+	if c.cache == nil {
+		return
+	}
+	if ele, hit := c.cache[key]; hit {
+		c.ll.MoveToFront(ele)
+		return ele.Value.(*entry).value, true
+	}
+	return
+}
+
+// Remove removes the provided key from the cache.
+func (c *Cache) Remove(key Key) {
+	if c.cache == nil {
+		return
+	}
+	if ele, hit := c.cache[key]; hit {
+		c.removeElement(ele)
+	}
+}
+
+// RemoveOldest removes the oldest item from the cache.
+func (c *Cache) RemoveOldest() {
+	if c.cache == nil {
+		return
+	}
+	ele := c.ll.Back()
+	if ele != nil {
+		c.removeElement(ele)
+	}
+}
+
+func (c *Cache) removeElement(e *list.Element) {
+	c.ll.Remove(e)
+	kv := e.Value.(*entry)
+	delete(c.cache, kv.key)
+	if c.OnEvicted != nil {
+		c.OnEvicted(kv.key, kv.value)
+	}
+}
+
+// Len returns the number of items in the cache.
+func (c *Cache) Len() int {
+	if c.cache == nil {
+		return 0
+	}
+	return c.ll.Len()
+}

Godeps/_workspace/src/github.com/golang/groupcache/lru/lru_test.go

@@ -0,0 +1,73 @@
+/*
+Copyright 2013 Google Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package lru
+
+import (
+	"testing"
+)
+
+type simpleStruct struct {
+	int
+	string
+}
+
+type complexStruct struct {
+	int
+	simpleStruct
+}
+
+var getTests = []struct {
+	name       string
+	keyToAdd   interface{}
+	keyToGet   interface{}
+	expectedOk bool
+}{
+	{"string_hit", "myKey", "myKey", true},
+	{"string_miss", "myKey", "nonsense", false},
+	{"simple_struct_hit", simpleStruct{1, "two"}, simpleStruct{1, "two"}, true},
+	{"simeple_struct_miss", simpleStruct{1, "two"}, simpleStruct{0, "noway"}, false},
+	{"complex_struct_hit", complexStruct{1, simpleStruct{2, "three"}},
+		complexStruct{1, simpleStruct{2, "three"}}, true},
+}
+
+func TestGet(t *testing.T) {
+	for _, tt := range getTests {
+		lru := New(0)
+		lru.Add(tt.keyToAdd, 1234)
+		val, ok := lru.Get(tt.keyToGet)
+		if ok != tt.expectedOk {
+			t.Fatalf("%s: cache hit = %v; want %v", tt.name, ok, !ok)
+		} else if ok && val != 1234 {
+			t.Fatalf("%s expected get to return 1234 but got %v", tt.name, val)
+		}
+	}
+}
+
+func TestRemove(t *testing.T) {
+	lru := New(0)
+	lru.Add("myKey", 1234)
+	if val, ok := lru.Get("myKey"); !ok {
+		t.Fatal("TestRemove returned no match")
+	} else if val != 1234 {
+		t.Fatalf("TestRemove failed.  Expected %d, got %v", 1234, val)
+	}
+
+	lru.Remove("myKey")
+	if _, ok := lru.Get("myKey"); ok {
+		t.Fatal("TestRemove returned a removed entry")
+	}
+}

build.sh

@@ -18,12 +18,12 @@ build() {
 	${godep} go build -ldflags "-w -X main.Version $version" ./cmd/stcli
 }
 
-prepare() {
-	go run cmd/assets/assets.go gui > auto/gui.files.go
+assets() {
+	godep go run cmd/assets/assets.go gui > auto/gui.files.go
 }
 
 test() {
-	go test -cpu=1,2,4 ./...
+	godep go test -cpu=1,2,4 ./...
 }
 
 sign() {
@@ -56,9 +56,14 @@ zipDist() {
 	rm -rf "$name"
 }
 
+deps() {
+	godep save ./cmd/syncthing ./cmd/assets ./cmd/stcli ./discover/cmd/discosrv
+}
+
 case "$1" in
 	"")
-		build
+		shift
+		build $*
 		;;
 
 	race)
@@ -71,8 +76,8 @@ case "$1" in
 
 	tar)
 		rm -f *.tar.gz *.zip
-		prepare
 		test || exit 1
+		assets
 		build
 
 		eval $(go env)
@@ -83,8 +88,8 @@ case "$1" in
 
 	all)
 		rm -f *.tar.gz *.zip
-		prepare
 		test || exit 1
+		assets
 
 		for os in darwin-amd64 linux-386 linux-amd64 freebsd-amd64 windows-amd64 ; do
 			export GOOS=${os%-*}
@@ -126,6 +131,14 @@ case "$1" in
 		done
 		;;
 
+	deps)
+		deps
+		;;
+
+	assets)
+		assets
+		;;
+
 	*)
 		echo "Unknown build parameter $1"
 		;;

cmd/stcli/main.go

@@ -61,7 +61,7 @@ func connect(target string) {
 
 	remoteID := certID(conn.ConnectionState().PeerCertificates[0].Raw)
 
-	pc = protocol.NewConnection(remoteID, conn, conn, Model{}, nil)
+	pc = protocol.NewConnection(remoteID, conn, conn, Model{})
 
 	select {}
 }
@@ -127,6 +127,11 @@ func (m Model) IndexUpdate(nodeID string, repo string, files []protocol.FileInfo
 	}
 }
 
+func (m Model) ClusterConfig(nodeID string, config protocol.ClusterConfigMessage) {
+	log.Println("Received cluster config")
+	log.Printf("%#v", config)
+}
+
 func (m Model) Request(nodeID, repo string, name string, offset int64, size int) ([]byte, error) {
 	log.Println("Received request")
 	return nil, io.EOF

cmd/syncthing/config.go

@@ -1,19 +1,18 @@
 package main
 
 import (
-	"crypto/sha256"
 	"encoding/xml"
-	"fmt"
 	"io"
 	"reflect"
 	"sort"
 	"strconv"
-	"strings"
 )
 
 type Configuration struct {
-	Version      int                       `xml:"version,attr" default:"1"`
+	Version      int                       `xml:"version,attr" default:"2"`
 	Repositories []RepositoryConfiguration `xml:"repository"`
+	Nodes        []NodeConfiguration       `xml:"node"`
+	GUI          GUIConfiguration          `xml:"gui"`
 	Options      OptionsConfiguration      `xml:"options"`
 	XMLName      xml.Name                  `xml:"configuration" json:"-"`
 }
@@ -22,29 +21,47 @@ type RepositoryConfiguration struct {
 	ID        string              `xml:"id,attr"`
 	Directory string              `xml:"directory,attr"`
 	Nodes     []NodeConfiguration `xml:"node"`
+	ReadOnly  bool                `xml:"ro,attr"`
+	nodeIDs   []string
+}
+
+func (r *RepositoryConfiguration) NodeIDs() []string {
+	if r.nodeIDs == nil {
+		for _, n := range r.Nodes {
+			r.nodeIDs = append(r.nodeIDs, n.NodeID)
+		}
+	}
+	return r.nodeIDs
 }
 
 type NodeConfiguration struct {
 	NodeID    string   `xml:"id,attr"`
-	Name      string   `xml:"name,attr"`
-	Addresses []string `xml:"address"`
+	Name      string   `xml:"name,attr,omitempty"`
+	Addresses []string `xml:"address,omitempty"`
 }
 
 type OptionsConfiguration struct {
-	ListenAddress      []string `xml:"listenAddress" default:":22000" ini:"listen-address"`
-	ReadOnly           bool     `xml:"readOnly" ini:"read-only"`
-	FollowSymlinks     bool     `xml:"followSymlinks" default:"true" ini:"follow-symlinks"`
-	GUIEnabled         bool     `xml:"guiEnabled" default:"true" ini:"gui-enabled"`
-	GUIAddress         string   `xml:"guiAddress" default:"127.0.0.1:8080" ini:"gui-address"`
-	GlobalAnnServer    string   `xml:"globalAnnounceServer" default:"announce.syncthing.net:22025" ini:"global-announce-server"`
-	GlobalAnnEnabled   bool     `xml:"globalAnnounceEnabled" default:"true" ini:"global-announce-enabled"`
-	LocalAnnEnabled    bool     `xml:"localAnnounceEnabled" default:"true" ini:"local-announce-enabled"`
-	ParallelRequests   int      `xml:"parallelRequests" default:"16" ini:"parallel-requests"`
-	MaxSendKbps        int      `xml:"maxSendKbps" ini:"max-send-kbps"`
-	RescanIntervalS    int      `xml:"rescanIntervalS" default:"60" ini:"rescan-interval"`
-	ReconnectIntervalS int      `xml:"reconnectionIntervalS" default:"60" ini:"reconnection-interval"`
-	MaxChangeKbps      int      `xml:"maxChangeKbps" default:"1000" ini:"max-change-bw"`
+	ListenAddress      []string `xml:"listenAddress" default:":22000"`
+	GlobalAnnServer    string   `xml:"globalAnnounceServer" default:"announce.syncthing.net:22025"`
+	GlobalAnnEnabled   bool     `xml:"globalAnnounceEnabled" default:"true"`
+	LocalAnnEnabled    bool     `xml:"localAnnounceEnabled" default:"true"`
+	ParallelRequests   int      `xml:"parallelRequests" default:"16"`
+	MaxSendKbps        int      `xml:"maxSendKbps"`
+	RescanIntervalS    int      `xml:"rescanIntervalS" default:"60"`
+	ReconnectIntervalS int      `xml:"reconnectionIntervalS" default:"60"`
+	MaxChangeKbps      int      `xml:"maxChangeKbps" default:"1000"`
 	StartBrowser       bool     `xml:"startBrowser" default:"true"`
+
+	Deprecated_ReadOnly   bool   `xml:"readOnly,omitempty"`
+	Deprecated_GUIEnabled bool   `xml:"guiEnabled,omitempty"`
+	Deprecated_GUIAddress string `xml:"guiAddress,omitempty"`
+}
+
+type GUIConfiguration struct {
+	Enabled  bool   `xml:"enabled,attr" default:"true"`
+	Address  string `xml:"address" default:"127.0.0.1:8080"`
+	User     string `xml:"user,omitempty"`
+	Password string `xml:"password,omitempty"`
 }
 
 func setDefaults(data interface{}) error {
@@ -108,41 +125,6 @@ func fillNilSlices(data interface{}) error {
 	return nil
 }
 
-func readConfigINI(m map[string]string, data interface{}) error {
-	s := reflect.ValueOf(data).Elem()
-	t := s.Type()
-
-	for i := 0; i < s.NumField(); i++ {
-		f := s.Field(i)
-		tag := t.Field(i).Tag
-
-		name := tag.Get("ini")
-		if len(name) == 0 {
-			name = strings.ToLower(t.Field(i).Name)
-		}
-
-		if v, ok := m[name]; ok {
-			switch f.Interface().(type) {
-			case string:
-				f.SetString(v)
-
-			case int:
-				i, err := strconv.ParseInt(v, 10, 64)
-				if err == nil {
-					f.SetInt(i)
-				}
-
-			case bool:
-				f.SetBool(v == "true")
-
-			default:
-				panic(f.Type())
-			}
-		}
-	}
-	return nil
-}
-
 func writeConfigXML(wr io.Writer, cfg Configuration) error {
 	e := xml.NewEncoder(wr)
 	e.Indent("", "    ")
@@ -173,6 +155,7 @@ func readConfigXML(rd io.Reader) (Configuration, error) {
 
 	setDefaults(&cfg)
 	setDefaults(&cfg.Options)
+	setDefaults(&cfg.GUI)
 
 	var err error
 	if rd != nil {
@@ -196,9 +179,44 @@ func readConfigXML(rd io.Reader) (Configuration, error) {
 		seenRepos[id] = true
 	}
 
+	if cfg.Version == 1 {
+		convertV1V2(&cfg)
+	}
+
 	return cfg, err
 }
 
+func convertV1V2(cfg *Configuration) {
+	// Collect the list of nodes.
+	// Replace node configs inside repositories with only a reference to the nide ID.
+	// Set all repositories to read only if the global read only flag is set.
+	var nodes = map[string]NodeConfiguration{}
+	for i, repo := range cfg.Repositories {
+		cfg.Repositories[i].ReadOnly = cfg.Options.Deprecated_ReadOnly
+		for j, node := range repo.Nodes {
+			if _, ok := nodes[node.NodeID]; !ok {
+				nodes[node.NodeID] = node
+			}
+			cfg.Repositories[i].Nodes[j] = NodeConfiguration{NodeID: node.NodeID}
+		}
+	}
+	cfg.Options.Deprecated_ReadOnly = false
+
+	// Set and sort the list of nodes.
+	for _, node := range nodes {
+		cfg.Nodes = append(cfg.Nodes, node)
+	}
+	sort.Sort(NodeConfigurationList(cfg.Nodes))
+
+	// GUI
+	cfg.GUI.Address = cfg.Options.Deprecated_GUIAddress
+	cfg.GUI.Enabled = cfg.Options.Deprecated_GUIEnabled
+	cfg.Options.Deprecated_GUIEnabled = false
+	cfg.Options.Deprecated_GUIAddress = ""
+
+	cfg.Version = 2
+}
+
 type NodeConfigurationList []NodeConfiguration
 
 func (l NodeConfigurationList) Less(a, b int) bool {
@@ -211,15 +229,6 @@ func (l NodeConfigurationList) Len() int {
 	return len(l)
 }
 
-func clusterHash(nodes []NodeConfiguration) string {
-	sort.Sort(NodeConfigurationList(nodes))
-	h := sha256.New()
-	for _, n := range nodes {
-		h.Write([]byte(n.NodeID))
-	}
-	return fmt.Sprintf("%x", h.Sum(nil))
-}
-
 func cleanNodeList(nodes []NodeConfiguration, myID string) []NodeConfiguration {
 	var myIDExists bool
 	for _, node := range nodes {

cmd/syncthing/config_test.go

@@ -10,10 +10,6 @@ import (
 func TestDefaultValues(t *testing.T) {
 	expected := OptionsConfiguration{
 		ListenAddress:      []string{":22000"},
-		ReadOnly:           false,
-		FollowSymlinks:     true,
-		GUIEnabled:         true,
-		GUIAddress:         "127.0.0.1:8080",
 		GlobalAnnServer:    "announce.syncthing.net:22025",
 		GlobalAnnEnabled:   true,
 		LocalAnnEnabled:    true,
@@ -35,6 +31,81 @@ func TestDefaultValues(t *testing.T) {
 	}
 }
 
+func TestNodeConfig(t *testing.T) {
+	v1data := []byte(`
+<configuration version="1">
+    <repository id="test" directory="~/Sync">
+        <node id="node1" name="node one">
+            <address>a</address>
+        </node>
+        <node id="node2" name="node two">
+            <address>b</address>
+        </node>
+    </repository>
+    <options>
+        <readOnly>true</readOnly>
+    </options>
+</configuration>
+`)
+
+	v2data := []byte(`
+<configuration version="2">
+    <repository id="test" directory="~/Sync" ro="true">
+        <node id="node1"/>
+        <node id="node2"/>
+    </repository>
+    <node id="node1" name="node one">
+        <address>a</address>
+    </node>
+    <node id="node2" name="node two">
+        <address>b</address>
+    </node>
+</configuration>
+`)
+
+	for i, data := range [][]byte{v1data, v2data} {
+		cfg, err := readConfigXML(bytes.NewReader(data))
+		if err != nil {
+			t.Error(err)
+		}
+
+		expectedRepos := []RepositoryConfiguration{
+			{
+				ID:        "test",
+				Directory: "~/Sync",
+				Nodes:     []NodeConfiguration{{NodeID: "node1"}, {NodeID: "node2"}},
+				ReadOnly:  true,
+			},
+		}
+		expectedNodes := []NodeConfiguration{
+			{
+				NodeID:    "node1",
+				Name:      "node one",
+				Addresses: []string{"a"},
+			},
+			{
+				NodeID:    "node2",
+				Name:      "node two",
+				Addresses: []string{"b"},
+			},
+		}
+		expectedNodeIDs := []string{"node1", "node2"}
+
+		if cfg.Version != 2 {
+			t.Errorf("%d: Incorrect version %d != 2", i, cfg.Version)
+		}
+		if !reflect.DeepEqual(cfg.Repositories, expectedRepos) {
+			t.Errorf("%d: Incorrect Repositories\n  A: %#v\n  E: %#v", i, cfg.Repositories, expectedRepos)
+		}
+		if !reflect.DeepEqual(cfg.Nodes, expectedNodes) {
+			t.Errorf("%d: Incorrect Nodes\n  A: %#v\n  E: %#v", i, cfg.Nodes, expectedNodes)
+		}
+		if !reflect.DeepEqual(cfg.Repositories[0].NodeIDs(), expectedNodeIDs) {
+			t.Errorf("%d: Incorrect NodeIDs\n  A: %#v\n  E: %#v", i, cfg.Repositories[0].NodeIDs(), expectedNodeIDs)
+		}
+	}
+}
+
 func TestNoListenAddress(t *testing.T) {
 	data := []byte(`<configuration version="1">
     <repository directory="~/Sync">
@@ -60,7 +131,7 @@ func TestNoListenAddress(t *testing.T) {
 }
 
 func TestOverriddenValues(t *testing.T) {
-	data := []byte(`<configuration version="1">
+	data := []byte(`<configuration version="2">
     <repository directory="~/Sync">
         <node id="..." name="...">
             <address>dynamic</address>
@@ -68,11 +139,7 @@ func TestOverriddenValues(t *testing.T) {
     </repository>
     <options>
        <listenAddress>:23000</listenAddress>
-        <readOnly>true</readOnly>
         <allowDelete>false</allowDelete>
-        <followSymlinks>false</followSymlinks>
-        <guiEnabled>false</guiEnabled>
-        <guiAddress>125.2.2.2:8080</guiAddress>
         <globalAnnounceServer>syncthing.nym.se:22025</globalAnnounceServer>
         <globalAnnounceEnabled>false</globalAnnounceEnabled>
         <localAnnounceEnabled>false</localAnnounceEnabled>
@@ -88,10 +155,6 @@ func TestOverriddenValues(t *testing.T) {
 
 	expected := OptionsConfiguration{
 		ListenAddress:      []string{":23000"},
-		ReadOnly:           true,
-		FollowSymlinks:     false,
-		GUIEnabled:         false,
-		GUIAddress:         "125.2.2.2:8080",
 		GlobalAnnServer:    "syncthing.nym.se:22025",
 		GlobalAnnEnabled:   false,
 		LocalAnnEnabled:    false,

cmd/syncthing/gui.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/calmh/syncthing/scanner"
 	"github.com/codegangsta/martini"
+	"github.com/codegangsta/martini-contrib/auth"
 )
 
 type guiError struct {
@@ -24,15 +25,15 @@ var (
 	guiErrorsMut sync.Mutex
 )
 
-func startGUI(addr string, m *Model) {
+func startGUI(cfg GUIConfiguration, m *Model) {
 	router := martini.NewRouter()
 	router.Get("/", getRoot)
 	router.Get("/rest/version", restGetVersion)
-	router.Get("/rest/model", restGetModel)
+	router.Get("/rest/model/:repo", restGetModel)
 	router.Get("/rest/connections", restGetConnections)
 	router.Get("/rest/config", restGetConfig)
 	router.Get("/rest/config/sync", restGetConfigInSync)
-	router.Get("/rest/need", restGetNeed)
+	router.Get("/rest/need/:repo", restGetNeed)
 	router.Get("/rest/system", restGetSystem)
 	router.Get("/rest/errors", restGetErrors)
 
@@ -43,12 +44,15 @@ func startGUI(addr string, m *Model) {
 
 	go func() {
 		mr := martini.New()
+		if len(cfg.User) > 0 && len(cfg.Password) > 0 {
+			mr.Use(auth.Basic(cfg.User, cfg.Password))
+		}
 		mr.Use(embeddedStatic())
 		mr.Use(martini.Recovery())
 		mr.Use(restMiddleware)
 		mr.Action(router.Handle)
 		mr.Map(m)
-		err := http.ListenAndServe(addr, mr)
+		err := http.ListenAndServe(cfg.Address, mr)
 		if err != nil {
 			warnln("GUI not possible:", err)
 		}
@@ -69,20 +73,22 @@ func restGetVersion() string {
 	return Version
 }
 
-func restGetModel(m *Model, w http.ResponseWriter) {
+func restGetModel(m *Model, w http.ResponseWriter, params martini.Params) {
+	var repo = params["repo"]
 	var res = make(map[string]interface{})
 
-	globalFiles, globalDeleted, globalBytes := m.GlobalSize()
+	globalFiles, globalDeleted, globalBytes := m.GlobalSize(repo)
 	res["globalFiles"], res["globalDeleted"], res["globalBytes"] = globalFiles, globalDeleted, globalBytes
 
-	localFiles, localDeleted, localBytes := m.LocalSize()
+	localFiles, localDeleted, localBytes := m.LocalSize(repo)
 	res["localFiles"], res["localDeleted"], res["localBytes"] = localFiles, localDeleted, localBytes
 
-	inSyncFiles, inSyncBytes := m.InSyncSize()
-	res["inSyncFiles"], res["inSyncBytes"] = inSyncFiles, inSyncBytes
+	needFiles, needBytes := m.NeedSize(repo)
+	res["needFiles"], res["needBytes"] = needFiles, needBytes
 
-	files, total := m.NeedFiles()
-	res["needFiles"], res["needBytes"] = len(files), total
+	res["inSyncFiles"], res["inSyncBytes"] = globalFiles-needFiles, globalBytes-needBytes
+
+	res["state"] = m.State(repo)
 
 	w.Header().Set("Content-Type", "application/json")
 	json.NewEncoder(w).Encode(res)
@@ -138,8 +144,9 @@ func (f guiFile) MarshalJSON() ([]byte, error) {
 	})
 }
 
-func restGetNeed(m *Model, w http.ResponseWriter) {
-	files, _ := m.NeedFiles()
+func restGetNeed(m *Model, w http.ResponseWriter, params martini.Params) {
+	repo := params["repo"]
+	files := m.NeedFilesRepo(repo)
 	gfs := make([]guiFile, len(files))
 	for i, f := range files {
 		gfs[i] = guiFile(f)
@@ -148,7 +155,7 @@ func restGetNeed(m *Model, w http.ResponseWriter) {
 	json.NewEncoder(w).Encode(gfs)
 }
 
-var cpuUsagePercent float64
+var cpuUsagePercent [10]float64 // The last ten seconds
 var cpuUsageLock sync.RWMutex
 
 func restGetSystem(w http.ResponseWriter) {
@@ -161,8 +168,12 @@ func restGetSystem(w http.ResponseWriter) {
 	res["alloc"] = m.Alloc
 	res["sys"] = m.Sys
 	cpuUsageLock.RLock()
-	res["cpuPercent"] = cpuUsagePercent
+	var cpusum float64
+	for _, p := range cpuUsagePercent {
+		cpusum += p
+	}
 	cpuUsageLock.RUnlock()
+	res["cpuPercent"] = cpusum / 10
 
 	w.Header().Set("Content-Type", "application/json")
 	json.NewEncoder(w).Encode(res)

cmd/syncthing/gui_solaris.go

@@ -72,8 +72,7 @@ func trackCPUUsage() {
 	var prevTime = time.Now().UnixNano()
 	var rusage prusage_t
 	var pid = os.Getpid()
-	for {
-		time.Sleep(10 * time.Second)
+	for _ = range time.NewTicker(time.Second).C {
 		err := solarisPrusage(pid, &rusage)
 		if err != nil {
 			warnln(err)
@@ -84,7 +83,8 @@ func trackCPUUsage() {
 		curUsage := rusage.Pr_utime.Nano() + rusage.Pr_stime.Nano()
 		usageDiff := curUsage - prevUsage
 		cpuUsageLock.Lock()
-		cpuUsagePercent = 100 * float64(usageDiff) / float64(timeDiff)
+		copy(cpuUsagePercent[1:], cpuUsagePercent[0:])
+		cpuUsagePercent[0] = 100 * float64(usageDiff) / float64(timeDiff)
 		cpuUsageLock.Unlock()
 		prevTime = curTime
 		prevUsage = curUsage

cmd/syncthing/gui_unix.go

@@ -15,15 +15,15 @@ func trackCPUUsage() {
 	var prevUsage int64
 	var prevTime = time.Now().UnixNano()
 	var rusage syscall.Rusage
-	for {
-		time.Sleep(10 * time.Second)
+	for _ = range time.NewTicker(time.Second).C {
 		syscall.Getrusage(syscall.RUSAGE_SELF, &rusage)
 		curTime := time.Now().UnixNano()
 		timeDiff := curTime - prevTime
 		curUsage := rusage.Utime.Nano() + rusage.Stime.Nano()
 		usageDiff := curUsage - prevUsage
 		cpuUsageLock.Lock()
-		cpuUsagePercent = 100 * float64(usageDiff) / float64(timeDiff)
+		copy(cpuUsagePercent[1:], cpuUsagePercent[0:])
+		cpuUsagePercent[0] = 100 * float64(usageDiff) / float64(timeDiff)
 		cpuUsageLock.Unlock()
 		prevTime = curTime
 		prevUsage = curUsage

cmd/syncthing/main.go

@@ -14,10 +14,10 @@ import (
 	"path/filepath"
 	"runtime"
 	"runtime/debug"
+	"runtime/pprof"
 	"strings"
 	"time"
 
-	"github.com/calmh/ini"
 	"github.com/calmh/syncthing/discover"
 	"github.com/calmh/syncthing/protocol"
 	"github.com/juju/ratelimit"
@@ -32,30 +32,32 @@ var (
 	myID       string
 	confDir    string
 	rateBucket *ratelimit.Bucket
+	stop       = make(chan bool)
 )
 
 const (
 	usage      = "syncthing [options]"
 	extraUsage = `The following enviroment variables are interpreted by syncthing:
 
- STNORESTART  Do not attempt to restart when requested to, instead just exit.
-              Set this variable when running under a service manager such as
-              runit, launchd, etc.
+ STNORESTART   Do not attempt to restart when requested to, instead just exit.
+               Set this variable when running under a service manager such as
+               runit, launchd, etc.
 
- STPROFILER   Set to a listen address such as "127.0.0.1:9090" to start the
-              profiler with HTTP access.
+ STPROFILER    Set to a listen address such as "127.0.0.1:9090" to start the
+               profiler with HTTP access.
 
- STTRACE      A comma separated string of facilities to trace. The valid
-              facility strings:
-              - "discover" (the node discovery package)
-              - "files"    (file set store)
-              - "idx"      (index sending and receiving)
-              - "mc"       (multicast beacon)
-              - "need"     (file need calculations)
-              - "net"      (connecting and disconnecting, network messages)
-              - "pull"     (file pull activity)
-              - "scanner"  (the file change scanner)
-              `
+ STTRACE       A comma separated string of facilities to trace. The valid
+               facility strings:
+               - "discover" (the node discovery package)
+               - "files"    (file set store)
+               - "idx"      (index sending and receiving)
+               - "mc"       (multicast beacon)
+               - "need"     (file need calculations)
+               - "net"      (connecting and disconnecting, network messages)
+               - "pull"     (file pull activity)
+               - "scanner"  (the file change scanner)
+
+ STCPUPROFILE  Write CPU profile to the specified file.`
 )
 
 func main() {
@@ -74,7 +76,7 @@ func main() {
 
 	if showVersion {
 		fmt.Printf("syncthing %s (%s %s-%s)\n", Version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
-		os.Exit(0)
+		return
 	}
 
 	if len(os.Getenv("GOGC")) == 0 {
@@ -120,31 +122,6 @@ func main() {
 			fatalln(err)
 		}
 		cf.Close()
-	} else {
-		// No config.xml, let's try the old syncthing.ini
-		iniFile := filepath.Join(confDir, "syncthing.ini")
-		cf, err := os.Open(iniFile)
-		if err == nil {
-			infoln("Migrating syncthing.ini to config.xml")
-			iniCfg := ini.Parse(cf)
-			cf.Close()
-			Rename(iniFile, filepath.Join(confDir, "migrated_syncthing.ini"))
-
-			cfg, _ = readConfigXML(nil)
-			cfg.Repositories = []RepositoryConfiguration{
-				{Directory: iniCfg.Get("repository", "dir")},
-			}
-			readConfigINI(iniCfg.OptionMap("settings"), &cfg.Options)
-			for name, addrs := range iniCfg.OptionMap("nodes") {
-				n := NodeConfiguration{
-					NodeID:    name,
-					Addresses: strings.Fields(addrs),
-				}
-				cfg.Repositories[0].Nodes = append(cfg.Repositories[0].Nodes, n)
-			}
-
-			saveConfig()
-		}
 	}
 
 	if len(cfg.Repositories) == 0 {
@@ -155,11 +132,12 @@ func main() {
 			{
 				ID:        "default",
 				Directory: filepath.Join(getHomeDir(), "Sync"),
-				Nodes: []NodeConfiguration{
-					{NodeID: myID, Addresses: []string{"dynamic"}},
-				},
+				Nodes:     []NodeConfiguration{{NodeID: myID}},
 			},
 		}
+		cfg.Nodes = []NodeConfiguration{
+			{NodeID: myID, Addresses: []string{"dynamic"}},
+		}
 
 		saveConfig()
 		infof("Edit %s to taste or use the GUI\n", cfgFile)
@@ -167,7 +145,7 @@ func main() {
 
 	if reset {
 		resetRepositories()
-		os.Exit(0)
+		return
 	}
 
 	if profiler := os.Getenv("STPROFILER"); len(profiler) > 0 {
@@ -210,10 +188,10 @@ func main() {
 	}
 
 	// GUI
-	if cfg.Options.GUIEnabled && cfg.Options.GUIAddress != "" {
-		addr, err := net.ResolveTCPAddr("tcp", cfg.Options.GUIAddress)
+	if cfg.GUI.Enabled && cfg.GUI.Address != "" {
+		addr, err := net.ResolveTCPAddr("tcp", cfg.GUI.Address)
 		if err != nil {
-			warnf("Cannot start GUI on %q: %v", cfg.Options.GUIAddress, err)
+			warnf("Cannot start GUI on %q: %v", cfg.GUI.Address, err)
 		} else {
 			var hostOpen, hostShow string
 			switch {
@@ -229,7 +207,7 @@ func main() {
 			}
 
 			infof("Starting web GUI on http://%s:%d/", hostShow, addr.Port)
-			startGUI(cfg.Options.GUIAddress, m)
+			startGUI(cfg.GUI, m)
 			if cfg.Options.StartBrowser && len(os.Getenv("STRESTART")) == 0 {
 				openURL(fmt.Sprintf("http://%s:%d", hostOpen, addr.Port))
 			}
@@ -244,27 +222,32 @@ func main() {
 	m.ScanRepos()
 	m.SaveIndexes(confDir)
 
-	connOpts := map[string]string{
-		"clientId":      "syncthing",
-		"clientVersion": Version,
-		"clusterHash":   clusterHash(cfg.Repositories[0].Nodes),
-	}
-
 	// Routine to connect out to configured nodes
 	disc := discovery()
-	go listenConnect(myID, disc, m, tlsCfg, connOpts)
+	go listenConnect(myID, disc, m, tlsCfg)
 
-	// Routine to pull blocks from other nodes to synchronize the local
-	// repository. Does not run when we are in read only (publish only) mode.
-	if cfg.Options.ReadOnly {
-		okln("Ready to synchronize (read only; no external updates accepted)")
-		m.StartRO()
-	} else {
-		okln("Ready to synchronize (read-write)")
-		m.StartRW(cfg.Options.ParallelRequests)
+	for _, repo := range cfg.Repositories {
+		// Routine to pull blocks from other nodes to synchronize the local
+		// repository. Does not run when we are in read only (publish only) mode.
+		if repo.ReadOnly {
+			okf("Ready to synchronize %s (read only; no external updates accepted)", repo.ID)
+			m.StartRepoRO(repo.ID)
+		} else {
+			okf("Ready to synchronize %s (read-write)", repo.ID)
+			m.StartRepoRW(repo.ID, cfg.Options.ParallelRequests)
+		}
 	}
 
-	select {}
+	if cpuprof := os.Getenv("STCPUPROFILE"); len(cpuprof) > 0 {
+		f, err := os.Create(cpuprof)
+		if err != nil {
+			log.Fatal(err)
+		}
+		pprof.StartCPUProfile(f)
+		defer pprof.StopCPUProfile()
+	}
+
+	<-stop
 }
 
 func resetRepositories() {
@@ -291,7 +274,8 @@ func restart() {
 	if os.Getenv("SMF_FMRI") != "" || os.Getenv("STNORESTART") != "" {
 		// Solaris SMF
 		infoln("Service manager detected; exit instead of restart")
-		os.Exit(0)
+		stop <- true
+		return
 	}
 
 	env := os.Environ()
@@ -311,7 +295,7 @@ func restart() {
 		fatalln(err)
 	}
 	proc.Release()
-	os.Exit(0)
+	stop <- true
 }
 
 var saveConfigCh = make(chan struct{})
@@ -348,7 +332,7 @@ func saveConfig() {
 	saveConfigCh <- struct{}{}
 }
 
-func listenConnect(myID string, disc *discover.Discoverer, m *Model, tlsCfg *tls.Config, connOpts map[string]string) {
+func listenConnect(myID string, disc *discover.Discoverer, m *Model, tlsCfg *tls.Config) {
 	var conns = make(chan *tls.Conn)
 
 	// Listen
@@ -389,13 +373,15 @@ func listenConnect(myID string, disc *discover.Discoverer, m *Model, tlsCfg *tls
 	go func() {
 		for {
 		nextNode:
-			for _, nodeCfg := range cfg.Repositories[0].Nodes {
+			for _, nodeCfg := range cfg.Nodes {
 				if nodeCfg.NodeID == myID {
 					continue
 				}
 				if m.ConnectedTo(nodeCfg.NodeID) {
 					continue
 				}
+
+				var addrs []string
 				for _, addr := range nodeCfg.Addresses {
 					if addr == "dynamic" {
 						if disc != nil {
@@ -403,10 +389,14 @@ func listenConnect(myID string, disc *discover.Discoverer, m *Model, tlsCfg *tls
 							if len(t) == 0 {
 								continue
 							}
-							addr = t[0] //XXX: Handle all of them
+							addrs = append(addrs, t...)
 						}
+					} else {
+						addrs = append(addrs, addr)
 					}
+				}
 
+				for _, addr := range addrs {
 					if debugNet {
 						dlog.Println("dial", nodeCfg.NodeID, addr)
 					}
@@ -429,7 +419,13 @@ func listenConnect(myID string, disc *discover.Discoverer, m *Model, tlsCfg *tls
 
 next:
 	for conn := range conns {
-		remoteID := certID(conn.ConnectionState().PeerCertificates[0].Raw)
+		certs := conn.ConnectionState().PeerCertificates
+		if l := len(certs); l != 1 {
+			warnf("Got peer certificate list of length %d != 1; protocol error", l)
+			conn.Close()
+			continue
+		}
+		remoteID := certID(certs[0].Raw)
 
 		if remoteID == myID {
 			warnf("Connected to myself (%s) - should not happen", remoteID)
@@ -443,13 +439,13 @@ next:
 			continue
 		}
 
-		for _, nodeCfg := range cfg.Repositories[0].Nodes {
+		for _, nodeCfg := range cfg.Nodes {
 			if nodeCfg.NodeID == remoteID {
 				var wr io.Writer = conn
 				if rateBucket != nil {
 					wr = &limitedWriter{conn, rateBucket}
 				}
-				protoConn := protocol.NewConnection(remoteID, conn, wr, m, connOpts)
+				protoConn := protocol.NewConnection(remoteID, conn, wr, m)
 				m.AddConnection(conn, protoConn)
 				continue next
 			}

cmd/syncthing/model.go

@@ -20,17 +20,28 @@ import (
 	"github.com/calmh/syncthing/scanner"
 )
 
+type repoState int
+
+const (
+	RepoIdle repoState = iota
+	RepoScanning
+	RepoSyncing
+	RepoCleaning
+)
+
 type Model struct {
 	repoDirs  map[string]string     // repo -> dir
 	repoFiles map[string]*files.Set // repo -> files
 	repoNodes map[string][]string   // repo -> nodeIDs
 	nodeRepos map[string][]string   // nodeID -> repos
+	repoState map[string]repoState  // repo -> state
 	rmut      sync.RWMutex          // protects the above
 
 	cm *cid.Map
 
 	protoConn map[string]protocol.Connection
 	rawConn   map[string]io.Closer
+	nodeVer   map[string]string
 	pmut      sync.RWMutex // protects protoConn and rawConn
 
 	sup suppressor
@@ -53,9 +64,11 @@ func NewModel(maxChangeBw int) *Model {
 		repoFiles: make(map[string]*files.Set),
 		repoNodes: make(map[string][]string),
 		nodeRepos: make(map[string][]string),
+		repoState: make(map[string]repoState),
 		cm:        cid.NewMap(),
 		protoConn: make(map[string]protocol.Connection),
 		rawConn:   make(map[string]io.Closer),
+		nodeVer:   make(map[string]string),
 		sup:       suppressor{threshold: int64(maxChangeBw)},
 	}
 
@@ -66,15 +79,13 @@ func NewModel(maxChangeBw int) *Model {
 // StartRW starts read/write processing on the current model. When in
 // read/write mode the model will attempt to keep in sync with the cluster by
 // pulling needed files from peer nodes.
-func (m *Model) StartRW(threads int) {
+func (m *Model) StartRepoRW(repo string, threads int) {
 	m.rmut.Lock()
 	defer m.rmut.Unlock()
 
-	if !m.addedRepo {
+	if dir, ok := m.repoDirs[repo]; !ok {
 		panic("cannot start without repo")
-	}
-	m.started = true
-	for repo, dir := range m.repoDirs {
+	} else {
 		newPuller(repo, dir, m, threads)
 	}
 }
@@ -82,23 +93,13 @@ func (m *Model) StartRW(threads int) {
 // StartRO starts read only processing on the current model. When in
 // read only mode the model will announce files to the cluster but not
 // pull in any external changes.
-func (m *Model) StartRO() {
-	m.rmut.Lock()
-	defer m.rmut.Unlock()
-
-	if !m.addedRepo {
-		panic("cannot start without repo")
-	}
-	m.started = true
-	for repo, dir := range m.repoDirs {
-		newPuller(repo, dir, m, 0) // zero threads => read only
-	}
+func (m *Model) StartRepoRO(repo string) {
+	m.StartRepoRW(repo, 0) // zero threads => read only
 }
 
 type ConnectionInfo struct {
 	protocol.Statistics
 	Address       string
-	ClientID      string
 	ClientVersion string
 	Completion    int
 }
@@ -116,8 +117,7 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
 	for node, conn := range m.protoConn {
 		ci := ConnectionInfo{
 			Statistics:    conn.Statistics(),
-			ClientID:      conn.Option("clientId"),
-			ClientVersion: conn.Option("clientVersion"),
+			ClientVersion: m.nodeVer[node],
 		}
 		if nc, ok := m.rawConn[node].(remoteAddrer); ok {
 			ci.Address = nc.RemoteAddr().String()
@@ -169,70 +169,45 @@ func sizeOf(fs []scanner.File) (files, deleted int, bytes int64) {
 
 // GlobalSize returns the number of files, deleted files and total bytes for all
 // files in the global model.
-func (m *Model) GlobalSize() (files, deleted int, bytes int64) {
+func (m *Model) GlobalSize(repo string) (files, deleted int, bytes int64) {
 	m.rmut.RLock()
-	var fs []scanner.File
-	for _, rf := range m.repoFiles {
-		fs = append(fs, rf.Global()...)
+	defer m.rmut.RUnlock()
+	if rf, ok := m.repoFiles[repo]; ok {
+		return sizeOf(rf.Global())
 	}
-	m.rmut.RUnlock()
-	return sizeOf(fs)
+	return 0, 0, 0
 }
 
 // LocalSize returns the number of files, deleted files and total bytes for all
 // files in the local repository.
-func (m *Model) LocalSize() (files, deleted int, bytes int64) {
+func (m *Model) LocalSize(repo string) (files, deleted int, bytes int64) {
 	m.rmut.RLock()
-	var fs []scanner.File
-	for _, rf := range m.repoFiles {
-		fs = append(fs, rf.Have(cid.LocalID)...)
+	defer m.rmut.RUnlock()
+	if rf, ok := m.repoFiles[repo]; ok {
+		return sizeOf(rf.Have(cid.LocalID))
 	}
-	m.rmut.RUnlock()
-	return sizeOf(fs)
-}
-
-// InSyncSize returns the number and total byte size of the local files that
-// are in sync with the global model.
-func (m *Model) InSyncSize() (files int, bytes int64) {
-	var gf []scanner.File
-	var nf []scanner.File
-
-	m.rmut.RLock()
-	for _, rf := range m.repoFiles {
-		gf = append(gf, rf.Global()...)
-		nf = append(nf, rf.Need(cid.LocalID)...)
-	}
-	m.rmut.RUnlock()
-
-	gn, _, gb := sizeOf(gf)
-	nn, _, nb := sizeOf(nf)
-
-	return gn - nn, gb - nb
+	return 0, 0, 0
 }
 
 // NeedFiles returns the list of currently needed files and the total size.
-func (m *Model) NeedFiles() ([]scanner.File, int64) {
-	var nf []scanner.File
-	m.rmut.RLock()
-	for _, rf := range m.repoFiles {
-		nf = append(nf, rf.Need(cid.LocalID)...)
-	}
-	m.rmut.RUnlock()
+func (m *Model) NeedSize(repo string) (files int, bytes int64) {
+	var nf = m.NeedFilesRepo(repo)
 
-	var bytes int64
 	for _, f := range nf {
 		bytes += f.Size
 	}
 
-	return nf, bytes
+	return len(nf), bytes
 }
 
 // NeedFiles returns the list of currently needed files and the total size.
 func (m *Model) NeedFilesRepo(repo string) []scanner.File {
 	m.rmut.RLock()
-	nf := m.repoFiles[repo].Need(cid.LocalID)
-	m.rmut.RUnlock()
-	return nf
+	defer m.rmut.RUnlock()
+	if rf, ok := m.repoFiles[repo]; ok {
+		return rf.Need(cid.LocalID)
+	}
+	return nil
 }
 
 // Index is called when a new node is connected and we receive their full index.
@@ -281,15 +256,37 @@ func (m *Model) IndexUpdate(nodeID string, repo string, fs []protocol.FileInfo)
 	m.rmut.RUnlock()
 }
 
+func (m *Model) ClusterConfig(nodeID string, config protocol.ClusterConfigMessage) {
+	compErr := compareClusterConfig(m.clusterConfig(nodeID), config)
+	if debugNet {
+		dlog.Printf("ClusterConfig: %s: %#v", nodeID, config)
+		dlog.Printf("  ... compare: %s: %v", nodeID, compErr)
+	}
+
+	if compErr != nil {
+		warnf("%s: %v", nodeID, compErr)
+		m.Close(nodeID, compErr)
+	}
+
+	m.pmut.Lock()
+	if config.ClientName == "syncthing" {
+		m.nodeVer[nodeID] = config.ClientVersion
+	} else {
+		m.nodeVer[nodeID] = config.ClientName + " " + config.ClientVersion
+	}
+	m.pmut.Unlock()
+}
+
 // Close removes the peer from the model and closes the underlying connection if possible.
 // Implements the protocol.Model interface.
 func (m *Model) Close(node string, err error) {
 	if debugNet {
 		dlog.Printf("%s: %v", node, err)
 	}
-	if err == protocol.ErrClusterHash {
-		warnf("Connection to %s closed due to mismatched cluster hash. Ensure that the configured cluster members are identical on both nodes.", node)
-	} else if err != io.EOF {
+
+	if err != io.EOF {
+		warnf("Connection to %s closed: %v", node, err)
+	} else if _, ok := err.(ClusterConfigMismatch); ok {
 		warnf("Connection to %s closed: %v", node, err)
 	}
 
@@ -308,6 +305,7 @@ func (m *Model) Close(node string, err error) {
 	}
 	delete(m.protoConn, node)
 	delete(m.rawConn, node)
+	delete(m.nodeVer, node)
 	m.pmut.Unlock()
 }
 
@@ -422,12 +420,17 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection)
 	m.rawConn[nodeID] = rawConn
 	m.pmut.Unlock()
 
+	cm := m.clusterConfig(nodeID)
+	protoConn.ClusterConfig(cm)
+
 	go func() {
 		m.rmut.RLock()
 		repos := m.nodeRepos[nodeID]
 		m.rmut.RUnlock()
 		for _, repo := range repos {
-			idx := m.ProtocolIndex(repo)
+			m.rmut.RLock()
+			idx := m.protocolIndex(repo)
+			m.rmut.RUnlock()
 			if debugNet {
 				dlog.Printf("IDX(out/initial): %s: %q: %d files", nodeID, repo, len(idx))
 			}
@@ -436,14 +439,11 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection)
 	}()
 }
 
-// ProtocolIndex returns the current local index in protocol data types.
-// Must be called with the read lock held.
-func (m *Model) ProtocolIndex(repo string) []protocol.FileInfo {
+// protocolIndex returns the current local index in protocol data types.
+func (m *Model) protocolIndex(repo string) []protocol.FileInfo {
 	var index []protocol.FileInfo
 
-	m.rmut.RLock()
 	fs := m.repoFiles[repo].Have(cid.LocalID)
-	m.rmut.RUnlock()
 
 	for _, f := range fs {
 		mf := fileInfoFromFile(f)
@@ -497,7 +497,7 @@ func (m *Model) broadcastIndexLoop() {
 			}
 			lastChange[repo] = c
 
-			idx := m.ProtocolIndex(repo)
+			idx := m.protocolIndex(repo)
 			m.saveIndex(repo, confDir, idx)
 
 			var indexWg sync.WaitGroup
@@ -546,31 +546,39 @@ func (m *Model) AddRepo(id, dir string, nodes []NodeConfiguration) {
 
 func (m *Model) ScanRepos() {
 	m.rmut.RLock()
+	var repos = make([]string, 0, len(m.repoDirs))
 	for repo := range m.repoDirs {
-		m.ScanRepo(repo)
+		repos = append(repos, repo)
 	}
 	m.rmut.RUnlock()
+
+	for _, repo := range repos {
+		m.ScanRepo(repo)
+	}
 }
 
 func (m *Model) ScanRepo(repo string) {
 	sup := &suppressor{threshold: int64(cfg.Options.MaxChangeKbps)}
+	m.rmut.Lock()
 	w := &scanner.Walker{
-		Dir:            m.repoDirs[repo],
-		IgnoreFile:     ".stignore",
-		FollowSymlinks: cfg.Options.FollowSymlinks,
-		BlockSize:      BlockSize,
-		TempNamer:      defTempNamer,
-		Suppressor:     sup,
-		CurrentFiler:   cFiler{m, repo},
+		Dir:          m.repoDirs[repo],
+		IgnoreFile:   ".stignore",
+		BlockSize:    BlockSize,
+		TempNamer:    defTempNamer,
+		Suppressor:   sup,
+		CurrentFiler: cFiler{m, repo},
 	}
+	m.rmut.Unlock()
+	m.setState(repo, RepoScanning)
 	fs, _ := w.Walk()
 	m.ReplaceLocal(repo, fs)
+	m.setState(repo, RepoIdle)
 }
 
 func (m *Model) SaveIndexes(dir string) {
 	m.rmut.RLock()
 	for repo := range m.repoDirs {
-		fs := m.ProtocolIndex(repo)
+		fs := m.protocolIndex(repo)
 		m.saveIndex(repo, dir, fs)
 	}
 	m.rmut.RUnlock()
@@ -633,46 +641,52 @@ func (m *Model) loadIndex(repo string, dir string) []protocol.FileInfo {
 	return im.Files
 }
 
-func fileFromFileInfo(f protocol.FileInfo) scanner.File {
-	var blocks = make([]scanner.Block, len(f.Blocks))
-	var offset int64
-	for i, b := range f.Blocks {
-		blocks[i] = scanner.Block{
-			Offset: offset,
-			Size:   b.Size,
-			Hash:   b.Hash,
+// clusterConfig returns a ClusterConfigMessage that is correct for the given peer node
+func (m *Model) clusterConfig(node string) protocol.ClusterConfigMessage {
+	cm := protocol.ClusterConfigMessage{
+		ClientName:    "syncthing",
+		ClientVersion: Version,
+	}
+
+	m.rmut.Lock()
+	for _, repo := range m.nodeRepos[node] {
+		cr := protocol.Repository{
+			ID: repo,
 		}
-		offset += int64(b.Size)
-	}
-	return scanner.File{
-		// Name is with native separator and normalization
-		Name:       filepath.FromSlash(f.Name),
-		Size:       offset,
-		Flags:      f.Flags &^ protocol.FlagInvalid,
-		Modified:   f.Modified,
-		Version:    f.Version,
-		Blocks:     blocks,
-		Suppressed: f.Flags&protocol.FlagInvalid != 0,
+		for _, node := range m.repoNodes[repo] {
+			// TODO: Set read only bit when relevant
+			cr.Nodes = append(cr.Nodes, protocol.Node{
+				ID:    node,
+				Flags: protocol.FlagShareTrusted,
+			})
+		}
+		cm.Repositories = append(cm.Repositories, cr)
 	}
+	m.rmut.Unlock()
+
+	return cm
 }
 
-func fileInfoFromFile(f scanner.File) protocol.FileInfo {
-	var blocks = make([]protocol.BlockInfo, len(f.Blocks))
-	for i, b := range f.Blocks {
-		blocks[i] = protocol.BlockInfo{
-			Size: b.Size,
-			Hash: b.Hash,
-		}
-	}
-	pf := protocol.FileInfo{
-		Name:     filepath.ToSlash(f.Name),
-		Flags:    f.Flags,
-		Modified: f.Modified,
-		Version:  f.Version,
-		Blocks:   blocks,
-	}
-	if f.Suppressed {
-		pf.Flags |= protocol.FlagInvalid
-	}
-	return pf
+func (m *Model) setState(repo string, state repoState) {
+	m.rmut.Lock()
+	m.repoState[repo] = state
+	m.rmut.Unlock()
+}
+
+func (m *Model) State(repo string) string {
+	m.rmut.Lock()
+	state := m.repoState[repo]
+	m.rmut.Unlock()
+	switch state {
+	case RepoIdle:
+		return "idle"
+	case RepoScanning:
+		return "scanning"
+	case RepoCleaning:
+		return "cleaning"
+	case RepoSyncing:
+		return "syncing"
+	default:
+		return "unknown"
+	}
 }

cmd/syncthing/model_test.go

@@ -170,6 +170,8 @@ func (f FakeConnection) Request(repo, name string, offset int64, size int) ([]by
 	return f.requestData, nil
 }
 
+func (FakeConnection) ClusterConfig(protocol.ClusterConfigMessage) {}
+
 func (FakeConnection) Ping() bool {
 	return true
 }

cmd/syncthing/puller.go

@@ -127,11 +127,13 @@ func (p *puller) run() {
 		for {
 			select {
 			case res := <-p.requestResults:
+				p.model.setState(p.repo, RepoSyncing)
 				changed = true
 				p.requestSlots <- true
 				p.handleRequestResult(res)
 
 			case b := <-p.blocks:
+				p.model.setState(p.repo, RepoSyncing)
 				changed = true
 				p.handleBlock(b)
 
@@ -155,10 +157,13 @@ func (p *puller) run() {
 		}
 
 		if changed {
+			p.model.setState(p.repo, RepoCleaning)
 			p.fixupDirectories()
 			changed = false
 		}
 
+		p.model.setState(p.repo, RepoIdle)
+
 		// Do a rescan if it's time for it
 		select {
 		case <-walkTicker:

cmd/syncthing/util.go

@@ -3,7 +3,11 @@ package main
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"runtime"
+
+	"github.com/calmh/syncthing/protocol"
+	"github.com/calmh/syncthing/scanner"
 )
 
 func MetricPrefix(n int64) string {
@@ -41,3 +45,99 @@ func Rename(from, to string) error {
 	}
 	return os.Rename(from, to)
 }
+
+func fileFromFileInfo(f protocol.FileInfo) scanner.File {
+	var blocks = make([]scanner.Block, len(f.Blocks))
+	var offset int64
+	for i, b := range f.Blocks {
+		blocks[i] = scanner.Block{
+			Offset: offset,
+			Size:   b.Size,
+			Hash:   b.Hash,
+		}
+		offset += int64(b.Size)
+	}
+	return scanner.File{
+		// Name is with native separator and normalization
+		Name:       filepath.FromSlash(f.Name),
+		Size:       offset,
+		Flags:      f.Flags &^ protocol.FlagInvalid,
+		Modified:   f.Modified,
+		Version:    f.Version,
+		Blocks:     blocks,
+		Suppressed: f.Flags&protocol.FlagInvalid != 0,
+	}
+}
+
+func fileInfoFromFile(f scanner.File) protocol.FileInfo {
+	var blocks = make([]protocol.BlockInfo, len(f.Blocks))
+	for i, b := range f.Blocks {
+		blocks[i] = protocol.BlockInfo{
+			Size: b.Size,
+			Hash: b.Hash,
+		}
+	}
+	pf := protocol.FileInfo{
+		Name:     filepath.ToSlash(f.Name),
+		Flags:    f.Flags,
+		Modified: f.Modified,
+		Version:  f.Version,
+		Blocks:   blocks,
+	}
+	if f.Suppressed {
+		pf.Flags |= protocol.FlagInvalid
+	}
+	return pf
+}
+
+func cmMap(cm protocol.ClusterConfigMessage) map[string]map[string]uint32 {
+	m := make(map[string]map[string]uint32)
+	for _, repo := range cm.Repositories {
+		m[repo.ID] = make(map[string]uint32)
+		for _, node := range repo.Nodes {
+			m[repo.ID][node.ID] = node.Flags
+		}
+	}
+	return m
+}
+
+type ClusterConfigMismatch error
+
+// compareClusterConfig returns nil for two equivalent configurations,
+// otherwise a decriptive error
+func compareClusterConfig(local, remote protocol.ClusterConfigMessage) error {
+	lm := cmMap(local)
+	rm := cmMap(remote)
+
+	for repo, lnodes := range lm {
+		_ = lnodes
+		if rnodes, ok := rm[repo]; ok {
+			for node, lflags := range lnodes {
+				if rflags, ok := rnodes[node]; ok {
+					if lflags&protocol.FlagShareBits != rflags&protocol.FlagShareBits {
+						return ClusterConfigMismatch(fmt.Errorf("remote has different sharing flags for node %q in repository %q", node, repo))
+					}
+				} else {
+					return ClusterConfigMismatch(fmt.Errorf("remote is missing node %q in repository %q", node, repo))
+				}
+			}
+		} else {
+			return ClusterConfigMismatch(fmt.Errorf("remote is missing repository %q", repo))
+		}
+	}
+
+	for repo, rnodes := range rm {
+		if lnodes, ok := lm[repo]; ok {
+			for node := range rnodes {
+				if _, ok := lnodes[node]; !ok {
+					return ClusterConfigMismatch(fmt.Errorf("remote has extra node %q in repository %q", node, repo))
+				}
+			}
+		} else {
+			return ClusterConfigMismatch(fmt.Errorf("remote has extra repository %q", repo))
+		}
+
+	}
+
+	return nil
+}

cmd/syncthing/util_test.go

@@ -0,0 +1,183 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/calmh/syncthing/protocol"
+)
+
+var testcases = []struct {
+	local, remote protocol.ClusterConfigMessage
+	err           string
+}{
+	{
+		local:  protocol.ClusterConfigMessage{},
+		remote: protocol.ClusterConfigMessage{},
+		err:    "",
+	},
+	{
+		local:  protocol.ClusterConfigMessage{ClientName: "a", ClientVersion: "b"},
+		remote: protocol.ClusterConfigMessage{ClientName: "c", ClientVersion: "d"},
+		err:    "",
+	},
+	{
+		local: protocol.ClusterConfigMessage{
+			Repositories: []protocol.Repository{
+				{ID: "foo"},
+			},
+		},
+		remote: protocol.ClusterConfigMessage{ClientName: "c", ClientVersion: "d"},
+		err:    `remote is missing repository "foo"`,
+	},
+	{
+		local: protocol.ClusterConfigMessage{ClientName: "c", ClientVersion: "d"},
+		remote: protocol.ClusterConfigMessage{
+			Repositories: []protocol.Repository{
+				{ID: "foo"},
+			},
+		},
+		err: `remote has extra repository "foo"`,
+	},
+	{
+		local: protocol.ClusterConfigMessage{
+			Repositories: []protocol.Repository{
+				{ID: "foo"},
+				{ID: "bar"},
+			},
+		},
+		remote: protocol.ClusterConfigMessage{
+			Repositories: []protocol.Repository{
+				{ID: "foo"},
+				{ID: "bar"},
+			},
+		},
+		err: "",
+	},
+	{
+		local: protocol.ClusterConfigMessage{
+			Repositories: []protocol.Repository{
+				{ID: "quux"},
+				{ID: "foo"},
+				{ID: "bar"},
+			},
+		},
+		remote: protocol.ClusterConfigMessage{
+			Repositories: []protocol.Repository{
+				{ID: "bar"},
+				{ID: "quux"},
+			},
+		},
+		err: `remote is missing repository "foo"`,
+	},
+	{
+		local: protocol.ClusterConfigMessage{
+			Repositories: []protocol.Repository{
+				{ID: "quux"},
+				{ID: "bar"},
+			},
+		},
+		remote: protocol.ClusterConfigMessage{
+			Repositories: []protocol.Repository{
+				{ID: "bar"},
+				{ID: "foo"},
+				{ID: "quux"},
+			},
+		},
+		err: `remote has extra repository "foo"`,
+	},
+	{
+		local: protocol.ClusterConfigMessage{
+			Repositories: []protocol.Repository{
+				{
+					ID: "foo",
+					Nodes: []protocol.Node{
+						{ID: "a"},
+					},
+				},
+				{ID: "bar"},
+			},
+		},
+		remote: protocol.ClusterConfigMessage{
+			Repositories: []protocol.Repository{
+				{ID: "foo"},
+				{ID: "bar"},
+			},
+		},
+		err: `remote is missing node "a" in repository "foo"`,
+	},
+
+	{
+		local: protocol.ClusterConfigMessage{
+			Repositories: []protocol.Repository{
+				{
+					ID: "foo",
+					Nodes: []protocol.Node{
+						{ID: "a"},
+					},
+				},
+				{ID: "bar"},
+			},
+		},
+		remote: protocol.ClusterConfigMessage{
+			Repositories: []protocol.Repository{
+				{
+					ID: "foo",
+					Nodes: []protocol.Node{
+						{ID: "a"},
+						{ID: "b"},
+					},
+				},
+				{ID: "bar"},
+			},
+		},
+		err: `remote has extra node "b" in repository "foo"`,
+	},
+
+	{
+		local: protocol.ClusterConfigMessage{
+			Repositories: []protocol.Repository{
+				{
+					ID: "foo",
+					Nodes: []protocol.Node{
+						{
+							ID:    "a",
+							Flags: protocol.FlagShareReadOnly,
+						},
+					},
+				},
+				{ID: "bar"},
+			},
+		},
+		remote: protocol.ClusterConfigMessage{
+			Repositories: []protocol.Repository{
+				{
+					ID: "foo",
+					Nodes: []protocol.Node{
+						{
+							ID:    "a",
+							Flags: protocol.FlagShareTrusted,
+						},
+					},
+				},
+				{ID: "bar"},
+			},
+		},
+		err: `remote has different sharing flags for node "a" in repository "foo"`,
+	},
+}
+
+func TestCompareClusterConfig(t *testing.T) {
+	for i, tc := range testcases {
+		err := compareClusterConfig(tc.local, tc.remote)
+		switch {
+		case tc.err == "" && err != nil:
+			t.Errorf("#%d: unexpected error: %v", i, err)
+
+		case tc.err != "" && err == nil:
+			t.Errorf("#%d: unexpected nil error", i)
+
+		case tc.err != "" && err != nil && tc.err != err.Error():
+			t.Errorf("#%d: incorrect error: %q != %q", i, err, tc.err)
+		}
+	}
+}

files/set.go

@@ -2,7 +2,6 @@
 package files
 
 import (
-	"crypto/md5"
 	"sync"
 
 	"github.com/calmh/syncthing/cid"
@@ -11,48 +10,14 @@ import (
 	"github.com/calmh/syncthing/scanner"
 )
 
-type key struct {
-	Name     string
-	Version  uint64
-	Modified int64
-	Hash     [md5.Size]byte
-}
-
 type fileRecord struct {
-	Usage int
-	File  scanner.File
+	File   scanner.File
+	Usage  int
+	Global bool
 }
 
 type bitset uint64
 
-func keyFor(f scanner.File) key {
-	h := md5.New()
-	for _, b := range f.Blocks {
-		h.Write(b.Hash)
-	}
-	return key{
-		Name:     f.Name,
-		Version:  f.Version,
-		Modified: f.Modified,
-		Hash:     md5.Sum(nil),
-	}
-}
-
-func (a key) newerThan(b key) bool {
-	if a.Version != b.Version {
-		return a.Version > b.Version
-	}
-	if a.Modified != b.Modified {
-		return a.Modified > b.Modified
-	}
-	for i := 0; i < md5.Size; i++ {
-		if a.Hash[i] != b.Hash[i] {
-			return a.Hash[i] > b.Hash[i]
-		}
-	}
-	return false
-}
-
 type Set struct {
 	sync.Mutex
 	files              map[key]fileRecord
@@ -143,15 +108,20 @@ func (m *Set) Need(id uint) []scanner.File {
 	if debug {
 		dlog.Printf("Need(%d)", id)
 	}
-	var fs []scanner.File
+	var fs = make([]scanner.File, 0, len(m.globalKey)/2) // Just a guess, but avoids too many reallocations
 	m.Lock()
 	rkID := m.remoteKey[id]
-	for name, gk := range m.globalKey {
-		if gk.newerThan(rkID[name]) {
-			if m.files[gk].File.Flags&protocol.FlagDirectory == 0 || // Regular file
-				m.files[gk].File.Flags&(protocol.FlagDirectory|protocol.FlagDeleted) == protocol.FlagDirectory { // Non-deleted directory
-				fs = append(fs, m.files[gk].File)
-			}
+	for gk, gf := range m.files {
+		if !gf.Global {
+			continue
+		}
+
+		file := gf.File
+		switch {
+		case file.Flags&protocol.FlagDirectory == 0 && gk.newerThan(rkID[gk.Name]):
+			fs = append(fs, file)
+		case file.Flags&(protocol.FlagDirectory|protocol.FlagDeleted) == protocol.FlagDirectory && gk.newerThan(rkID[gk.Name]):
+			fs = append(fs, file)
 		}
 	}
 	m.Unlock()
@@ -162,7 +132,7 @@ func (m *Set) Have(id uint) []scanner.File {
 	if debug {
 		dlog.Printf("Have(%d)", id)
 	}
-	var fs []scanner.File
+	var fs = make([]scanner.File, 0, len(m.remoteKey[id]))
 	m.Lock()
 	for _, rk := range m.remoteKey[id] {
 		fs = append(fs, m.files[rk].File)
@@ -175,10 +145,12 @@ func (m *Set) Global() []scanner.File {
 	if debug {
 		dlog.Printf("Global()")
 	}
-	var fs []scanner.File
+	var fs = make([]scanner.File, 0, len(m.globalKey))
 	m.Lock()
-	for _, rk := range m.globalKey {
-		fs = append(fs, m.files[rk].File)
+	for _, file := range m.files {
+		if file.Global {
+			fs = append(fs, file.File)
+		}
 	}
 	m.Unlock()
 	return fs
@@ -272,6 +244,14 @@ func (m *Set) update(cid uint, fs []scanner.File) {
 			av |= 1 << cid
 			m.globalAvailability[n] = av
 		case fk.newerThan(gk):
+			if ok {
+				f := m.files[gk]
+				f.Global = false
+				m.files[gk] = f
+			}
+			f := m.files[fk]
+			f.Global = true
+			m.files[fk] = f
 			m.globalKey[n] = fk
 			m.globalAvailability[n] = 1 << cid
 		}

files/set_anal.go

@@ -0,0 +1,44 @@
+//+build anal
+
+package files
+
+import (
+	"crypto/md5"
+
+	"github.com/calmh/syncthing/scanner"
+)
+
+type key struct {
+	Name     string
+	Version  uint64
+	Modified int64
+	Hash     [md5.Size]byte
+}
+
+func keyFor(f scanner.File) key {
+	h := md5.New()
+	for _, b := range f.Blocks {
+		h.Write(b.Hash)
+	}
+	return key{
+		Name:     f.Name,
+		Version:  f.Version,
+		Modified: f.Modified,
+		Hash:     md5.Sum(nil),
+	}
+}
+
+func (a key) newerThan(b key) bool {
+	if a.Version != b.Version {
+		return a.Version > b.Version
+	}
+	if a.Modified != b.Modified {
+		return a.Modified > b.Modified
+	}
+	for i := 0; i < md5.Size; i++ {
+		if a.Hash[i] != b.Hash[i] {
+			return a.Hash[i] > b.Hash[i]
+		}
+	}
+	return false
+}

files/set_fast.go

@@ -0,0 +1,21 @@
+//+build !anal
+
+package files
+
+import "github.com/calmh/syncthing/scanner"
+
+type key struct {
+	Name    string
+	Version uint64
+}
+
+func keyFor(f scanner.File) key {
+	return key{
+		Name:    f.Name,
+		Version: f.Version,
+	}
+}
+
+func (a key) newerThan(b key) bool {
+	return a.Version > b.Version
+}

files/set_test.go

@@ -82,10 +82,24 @@ func TestLocalDeleted(t *testing.T) {
 
 	m.ReplaceWithDelete(cid.LocalID, local1)
 
-	local2 := []scanner.File{
+	m.ReplaceWithDelete(cid.LocalID, []scanner.File{
+		local1[0],
+		// [1] removed
+		local1[2],
+		local1[3],
+		local1[4],
+	})
+	m.ReplaceWithDelete(cid.LocalID, []scanner.File{
 		local1[0],
 		local1[2],
-	}
+		// [3] removed
+		local1[4],
+	})
+	m.ReplaceWithDelete(cid.LocalID, []scanner.File{
+		local1[0],
+		local1[2],
+		// [4] removed
+	})
 
 	expectedGlobal1 := []scanner.File{
 		local1[0],
@@ -95,7 +109,6 @@ func TestLocalDeleted(t *testing.T) {
 		scanner.File{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory},
 	}
 
-	m.ReplaceWithDelete(cid.LocalID, local2)
 	g := m.Global()
 	sort.Sort(fileList(g))
 	sort.Sort(fileList(expectedGlobal1))
@@ -104,9 +117,10 @@ func TestLocalDeleted(t *testing.T) {
 		t.Errorf("Global incorrect;\n A: %v !=\n E: %v", g, expectedGlobal1)
 	}
 
-	local3 := []scanner.File{
+	m.ReplaceWithDelete(cid.LocalID, []scanner.File{
 		local1[0],
-	}
+		// [2] removed
+	})
 
 	expectedGlobal2 := []scanner.File{
 		local1[0],
@@ -116,7 +130,6 @@ func TestLocalDeleted(t *testing.T) {
 		scanner.File{Name: "z", Version: 1003, Flags: protocol.FlagDeleted | protocol.FlagDirectory},
 	}
 
-	m.ReplaceWithDelete(cid.LocalID, local3)
 	g = m.Global()
 	sort.Sort(fileList(g))
 	sort.Sort(fileList(expectedGlobal2))
@@ -126,62 +139,33 @@ func TestLocalDeleted(t *testing.T) {
 	}
 }
 
-func BenchmarkSetLocal10k(b *testing.B) {
-	m := NewSet()
-
+func Benchmark10kReplace(b *testing.B) {
 	var local []scanner.File
 	for i := 0; i < 10000; i++ {
-		local = append(local, scanner.File{Name: fmt.Sprintf("file%d"), Version: 1000})
+		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 
-	var remote []scanner.File
-	for i := 0; i < 10000; i++ {
-		remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d"), Version: 1000})
-	}
-
-	m.Replace(1, remote)
-
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
+		m := NewSet()
 		m.ReplaceWithDelete(cid.LocalID, local)
 	}
 }
 
-func BenchmarkSetLocal10(b *testing.B) {
-	m := NewSet()
-
-	var local []scanner.File
-	for i := 0; i < 10; i++ {
-		local = append(local, scanner.File{Name: fmt.Sprintf("file%d"), Version: 1000})
-	}
-
+func Benchmark10kUpdateChg(b *testing.B) {
 	var remote []scanner.File
 	for i := 0; i < 10000; i++ {
-		remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d"), Version: 1000})
+		remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 
-	m.Replace(1, remote)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		m.ReplaceWithDelete(cid.LocalID, local)
-	}
-}
-
-func BenchmarkAddLocal10k(b *testing.B) {
 	m := NewSet()
+	m.Replace(1, remote)
 
 	var local []scanner.File
 	for i := 0; i < 10000; i++ {
-		local = append(local, scanner.File{Name: fmt.Sprintf("file%d"), Version: 1000})
+		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 
-	var remote []scanner.File
-	for i := 0; i < 10000; i++ {
-		remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d"), Version: 1000})
-	}
-
-	m.Replace(1, remote)
 	m.ReplaceWithDelete(cid.LocalID, local)
 
 	b.ResetTimer()
@@ -195,31 +179,112 @@ func BenchmarkAddLocal10k(b *testing.B) {
 	}
 }
 
-func BenchmarkAddLocal10(b *testing.B) {
-	m := NewSet()
-
-	var local []scanner.File
-	for i := 0; i < 10; i++ {
-		local = append(local, scanner.File{Name: fmt.Sprintf("file%d"), Version: 1000})
-	}
-
+func Benchmark10kUpdateSme(b *testing.B) {
 	var remote []scanner.File
 	for i := 0; i < 10000; i++ {
-		remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d"), Version: 1000})
+		remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
 	}
 
+	m := NewSet()
 	m.Replace(1, remote)
+
+	var local []scanner.File
+	for i := 0; i < 10000; i++ {
+		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
+	}
+
 	m.ReplaceWithDelete(cid.LocalID, local)
 
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		for j := range local {
-			local[j].Version++
-		}
 		m.Update(cid.LocalID, local)
 	}
 }
 
+func Benchmark10kNeed2k(b *testing.B) {
+	var remote []scanner.File
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
+	}
+
+	m := NewSet()
+	m.Replace(cid.LocalID+1, remote)
+
+	var local []scanner.File
+	for i := 0; i < 8000; i++ {
+		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
+	}
+	for i := 8000; i < 10000; i++ {
+		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 980})
+	}
+
+	m.ReplaceWithDelete(cid.LocalID, local)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		fs := m.Need(cid.LocalID)
+		if l := len(fs); l != 2000 {
+			b.Errorf("wrong length %d != 2k", l)
+		}
+	}
+}
+
+func Benchmark10kHave(b *testing.B) {
+	var remote []scanner.File
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
+	}
+
+	m := NewSet()
+	m.Replace(cid.LocalID+1, remote)
+
+	var local []scanner.File
+	for i := 0; i < 2000; i++ {
+		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
+	}
+	for i := 2000; i < 10000; i++ {
+		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 980})
+	}
+
+	m.ReplaceWithDelete(cid.LocalID, local)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		fs := m.Have(cid.LocalID)
+		if l := len(fs); l != 10000 {
+			b.Errorf("wrong length %d != 10k", l)
+		}
+	}
+}
+
+func Benchmark10kGlobal(b *testing.B) {
+	var remote []scanner.File
+	for i := 0; i < 10000; i++ {
+		remote = append(remote, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
+	}
+
+	m := NewSet()
+	m.Replace(cid.LocalID+1, remote)
+
+	var local []scanner.File
+	for i := 0; i < 2000; i++ {
+		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 1000})
+	}
+	for i := 2000; i < 10000; i++ {
+		local = append(local, scanner.File{Name: fmt.Sprintf("file%d", i), Version: 980})
+	}
+
+	m.ReplaceWithDelete(cid.LocalID, local)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		fs := m.Global()
+		if l := len(fs); l != 10000 {
+			b.Errorf("wrong length %d != 10k", l)
+		}
+	}
+}
+
 func TestGlobalReset(t *testing.T) {
 	m := NewSet()
 
@@ -284,6 +349,10 @@ func TestNeed(t *testing.T) {
 	m.Replace(1, remote)
 
 	need := m.Need(0)
+
+	sort.Sort(fileList(need))
+	sort.Sort(fileList(shouldNeed))
+
 	if !reflect.DeepEqual(need, shouldNeed) {
 		t.Errorf("Need incorrect;\n%v !=\n%v", need, shouldNeed)
 	}

gui/app.js

@@ -7,7 +7,7 @@ var syncthing = angular.module('syncthing', []);
 
 syncthing.controller('SyncthingCtrl', function ($scope, $http) {
     var prevDate = 0,
-        modelGetOK = true;
+    getOK = true;
 
     $scope.connections = {};
     $scope.config = {};
@@ -16,35 +16,40 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
     $scope.configInSync = true;
     $scope.errors = [];
     $scope.seenError = '';
+    $scope.model = {};
+    $scope.repos = [];
 
     // Strings before bools look better
     $scope.settings = [
-        {id: 'ListenStr', descr: 'Sync Protocol Listen Addresses', type: 'text', restart: true},
-        {id: 'GUIAddress', descr: 'GUI Listen Address', type: 'text', restart: true},
-        {id: 'MaxSendKbps', descr: 'Outgoing Rate Limit (KBps)', type: 'number', restart: true},
-        {id: 'RescanIntervalS', descr: 'Rescan Interval (s)', type: 'number', restart: true},
-        {id: 'ReconnectIntervalS', descr: 'Reconnect Interval (s)', type: 'number', restart: true},
-        {id: 'ParallelRequests', descr: 'Max Outstanding Requests', type: 'number', restart: true},
-        {id: 'MaxChangeKbps', descr: 'Max File Change Rate (KBps)', type: 'number', restart: true},
+    {id: 'ListenStr', descr: 'Sync Protocol Listen Addresses', type: 'text', restart: true},
+    {id: 'MaxSendKbps', descr: 'Outgoing Rate Limit (KBps)', type: 'number', restart: true},
+    {id: 'RescanIntervalS', descr: 'Rescan Interval (s)', type: 'number', restart: true},
+    {id: 'ReconnectIntervalS', descr: 'Reconnect Interval (s)', type: 'number', restart: true},
+    {id: 'ParallelRequests', descr: 'Max Outstanding Requests', type: 'number', restart: true},
+    {id: 'MaxChangeKbps', descr: 'Max File Change Rate (KBps)', type: 'number', restart: true},
 
-        {id: 'ReadOnly', descr: 'Read Only', type: 'bool', restart: true},
-        {id: 'FollowSymlinks', descr: 'Follow Symlinks', type: 'bool', restart: true},
-        {id: 'GlobalAnnEnabled', descr: 'Global Announce', type: 'bool', restart: true},
-        {id: 'LocalAnnEnabled', descr: 'Local Announce', type: 'bool', restart: true},
-        {id: 'StartBrowser', descr: 'Start Browser', type: 'bool'},
+    {id: 'GlobalAnnEnabled', descr: 'Global Announce', type: 'bool', restart: true},
+    {id: 'LocalAnnEnabled', descr: 'Local Announce', type: 'bool', restart: true},
+    {id: 'StartBrowser', descr: 'Start Browser', type: 'bool'},
     ];
 
-    function modelGetSucceeded() {
-        if (!modelGetOK) {
+    $scope.guiSettings = [
+    {id: 'Address', descr: 'GUI Listen Addresses', type: 'text', restart: true},
+    {id: 'User', descr: 'GUI Authentication User', type: 'text', restart: true},
+    {id: 'Password', descr: 'GUI Authentication Password', type: 'password', restart: true},
+    ];
+
+    function getSucceeded() {
+        if (!getOK) {
             $('#networkError').modal('hide');
-            modelGetOK = true;
+            getOK = true;
         }
     }
 
-    function modelGetFailed() {
-        if (modelGetOK) {
+    function getFailed() {
+        if (getOK) {
             $('#networkError').modal({backdrop: 'static', keyboard: false});
-            modelGetOK = false;
+            getOK = false;
         }
     }
 
@@ -61,40 +66,22 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         return a.NodeID > b.NodeID;
     }
 
-    $http.get('/rest/version').success(function (data) {
-        $scope.version = data;
-    });
-    $http.get('/rest/system').success(function (data) {
-        $scope.system = data;
-        $scope.myID = data.myID;
-
-        $http.get('/rest/config').success(function (data) {
-            $scope.config = data;
-            $scope.config.Options.ListenStr = $scope.config.Options.ListenAddress.join(', ');
-
-            var nodes = $scope.config.Repositories[0].Nodes;
-            nodes.sort(nodeCompare);
-            $scope.nodes = nodes;
-        });
-        $http.get('/rest/config/sync').success(function (data) {
-            $scope.configInSync = data.configInSync;
-        });
-    });
-
     $scope.refresh = function () {
         $http.get('/rest/system').success(function (data) {
+            getSucceeded();
             $scope.system = data;
-        });
-        $http.get('/rest/model').success(function (data) {
-            $scope.model = data;
-            modelGetSucceeded();
         }).error(function () {
-            modelGetFailed();
+            getFailed();
+        });
+        $scope.repos.forEach(function (repo) {
+            $http.get('/rest/model/' + repo.ID).success(function (data) {
+                $scope.model[repo.ID] = data;
+            });
         });
         $http.get('/rest/connections').success(function (data) {
             var now = Date.now(),
-                td = (now - prevDate) / 1000,
-                id;
+            td = (now - prevDate) / 1000,
+            id;
 
             prevDate = now;
             $scope.inbps = 0;
@@ -116,28 +103,53 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
             }
             $scope.connections = data;
         });
-        $http.get('/rest/need').success(function (data) {
-            var i, name;
-            for (i = 0; i < data.length; i++) {
-                name = data[i].Name.split('/');
-                data[i].ShortName = name[name.length - 1];
-            }
-            data.sort(function (a, b) {
-                if (a.ShortName < b.ShortName) {
-                    return -1;
-                }
-                if (a.ShortName > b.ShortName) {
-                    return 1;
-                }
-                return 0;
-            });
-            $scope.need = data;
-        });
         $http.get('/rest/errors').success(function (data) {
             $scope.errors = data;
         });
     };
 
+    $scope.repoStatus = function (repo) {
+        if (typeof $scope.model[repo] === 'undefined') {
+            return 'Unknown';
+        }
+
+        var state = '' + $scope.model[repo].state;
+        state = state[0].toUpperCase() + state.substr(1);
+
+        if (state == "Syncing" || state == "Idle") {
+            state += " (" + $scope.syncPercentage(repo) + "%)";
+        }
+
+        return state;
+    }
+
+    $scope.repoClass = function (repo) {
+        if (typeof $scope.model[repo] === 'undefined') {
+            return 'text-info';
+        }
+
+        var state = '' + $scope.model[repo].state;
+        if (state == 'idle') {
+            return 'text-success';
+        }
+        if (state == 'syncing') {
+            return 'text-primary';
+        }
+        return 'text-info';
+    }
+
+    $scope.syncPercentage = function (repo) {
+        if (typeof $scope.model[repo] === 'undefined') {
+            return 100;
+        }
+        if ($scope.model[repo].globalBytes === 0) {
+            return 100;
+        }
+
+        var pct = 100 * $scope.model[repo].inSyncBytes / $scope.model[repo].globalBytes;
+        return Math.ceil(pct);
+    };
+
     $scope.nodeStatus = function (nodeCfg) {
         var conn = $scope.connections[nodeCfg.NodeID];
         if (conn) {
@@ -182,7 +194,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         if (conn) {
             return conn.Address;
         }
-        return '(unknown address)';
+        return '?';
     };
 
     $scope.nodeCompletion = function (nodeCfg) {
@@ -201,7 +213,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         if (conn) {
             return conn.ClientVersion;
         }
-        return '(unknown version)';
+        return '?';
     };
 
     $scope.nodeName = function (nodeCfg) {
@@ -211,11 +223,15 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         return nodeCfg.NodeID.substr(0, 6);
     };
 
+    $scope.editSettings = function () {
+        $('#settings').modal({backdrop: 'static', keyboard: true});
+    }
+
     $scope.saveSettings = function () {
         $scope.configInSync = false;
         $scope.config.Options.ListenAddress = $scope.config.Options.ListenStr.split(',').map(function (x) { return x.trim(); });
         $http.post('/rest/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
-        $('#settingsTable').collapse('hide');
+        $('#settings').modal("hide");
     };
 
     $scope.restart = function () {
@@ -224,34 +240,34 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
     };
 
     $scope.editNode = function (nodeCfg) {
-        $scope.currentNode = nodeCfg;
+        $scope.currentNode = $.extend({}, nodeCfg);
         $scope.editingExisting = true;
         $scope.currentNode.AddressesStr = nodeCfg.Addresses.join(', ');
-        $('#editNode').modal({backdrop: 'static', keyboard: false});
+        $('#editNode').modal({backdrop: 'static', keyboard: true});
     };
 
     $scope.addNode = function () {
-        $scope.currentNode = {NodeID: '', AddressesStr: 'dynamic'};
+        $scope.currentNode = {AddressesStr: 'dynamic'};
         $scope.editingExisting = false;
-        $('#editNode').modal({backdrop: 'static', keyboard: false});
+        $('#editNode').modal({backdrop: 'static', keyboard: true});
     };
 
     $scope.deleteNode = function () {
-        var newNodes = [], i;
-
         $('#editNode').modal('hide');
         if (!$scope.editingExisting) {
             return;
         }
 
-        for (i = 0; i < $scope.nodes.length; i++) {
-            if ($scope.nodes[i].NodeID !== $scope.currentNode.NodeID) {
-                newNodes.push($scope.nodes[i]);
-            }
-        }
+        $scope.nodes = $scope.nodes.filter(function (n) {
+            return n.NodeID !== $scope.currentNode.NodeID;
+        });
+        $scope.config.Nodes = $scope.nodes;
 
-        $scope.nodes = newNodes;
-        $scope.config.Repositories[0].Nodes = newNodes;
+        for (var i = 0; i < $scope.repos.length; i++) {
+            $scope.repos[i].Nodes = $scope.repos[i].Nodes.filter(function (n) {
+                return n.NodeID !== $scope.currentNode.NodeID;
+            });
+        }
 
         $scope.configInSync = false;
         $http.post('/rest/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
@@ -279,21 +295,15 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         }
 
         $scope.nodes.sort(nodeCompare);
-        $scope.config.Repositories[0].Nodes = $scope.nodes;
+        $scope.config.Nodes = $scope.nodes;
 
         $http.post('/rest/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
     };
 
     $scope.otherNodes = function () {
-        var nodes = [], i, n;
-
-        for (i = 0; i < $scope.nodes.length; i++) {
-            n = $scope.nodes[i];
-            if (n.NodeID !== $scope.myID) {
-                nodes.push(n);
-            }
-        }
-        return nodes;
+        return $scope.nodes.filter(function (n){
+            return n.NodeID !== $scope.myID;
+        });
     };
 
     $scope.thisNode = function () {
@@ -308,14 +318,9 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
     };
 
     $scope.errorList = function () {
-        var errors = [];
-        for (var i = 0; i < $scope.errors.length; i++) {
-            var e = $scope.errors[i];
-            if (e.Time > $scope.seenError) {
-                errors.push(e);
-            }
-        }
-        return errors;
+        return $scope.errors.filter(function (e) {
+            return e.Time > $scope.seenError;
+        });
     };
 
     $scope.clearErrors = function () {
@@ -330,7 +335,96 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         return str;
     };
 
-    $scope.refresh();
+    $scope.editRepo = function (nodeCfg) {
+        $scope.currentRepo = $.extend({selectedNodes: {}}, nodeCfg);
+        $scope.currentRepo.Nodes.forEach(function (n) {
+            $scope.currentRepo.selectedNodes[n.NodeID] = true;
+        });
+        $scope.editingExisting = true;
+        $('#editRepo').modal({backdrop: 'static', keyboard: true});
+    };
+
+    $scope.addRepo = function () {
+        $scope.currentRepo = {selectedNodes: {}};
+        $scope.editingExisting = false;
+        $('#editRepo').modal({backdrop: 'static', keyboard: true});
+    };
+
+    $scope.saveRepo = function () {
+        var repoCfg, done, i;
+
+        $scope.configInSync = false;
+        $('#editRepo').modal('hide');
+        repoCfg = $scope.currentRepo;
+        repoCfg.Nodes = [];
+        repoCfg.selectedNodes[$scope.myID] = true;
+        for (var nodeID in repoCfg.selectedNodes) {
+            if (repoCfg.selectedNodes[nodeID] === true) {
+                repoCfg.Nodes.push({NodeID: nodeID});
+            }
+        }
+        delete repoCfg.selectedNodes;
+
+        done = false;
+        for (i = 0; i < $scope.repos.length; i++) {
+            if ($scope.repos[i].ID === repoCfg.ID) {
+                $scope.repos[i] = repoCfg;
+                done = true;
+                break;
+            }
+        }
+
+        if (!done) {
+            $scope.repos.push(repoCfg);
+        }
+
+        $scope.config.Repositories = $scope.repos;
+
+        $http.post('/rest/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
+    };
+
+    $scope.deleteRepo = function () {
+        $('#editRepo').modal('hide');
+        if (!$scope.editingExisting) {
+            return;
+        }
+
+        $scope.repos = $scope.repos.filter(function (r) {
+            return r.ID !== $scope.currentRepo.ID;
+        });
+
+        $scope.config.Repositories = $scope.repos;
+
+        $scope.configInSync = false;
+        $http.post('/rest/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
+    };
+
+    $http.get('/rest/version').success(function (data) {
+        $scope.version = data;
+    });
+
+    $http.get('/rest/system').success(function (data) {
+        $scope.system = data;
+        $scope.myID = data.myID;
+    });
+
+    $http.get('/rest/config').success(function (data) {
+        $scope.config = data;
+        $scope.config.Options.ListenStr = $scope.config.Options.ListenAddress.join(', ');
+
+        var nodes = $scope.config.Nodes;
+        nodes.sort(nodeCompare);
+        $scope.nodes = nodes;
+
+        $scope.repos = $scope.config.Repositories;
+
+        $scope.refresh();
+    });
+
+    $http.get('/rest/config/sync').success(function (data) {
+        $scope.configInSync = data.configInSync;
+    });
+
     setInterval($scope.refresh, 10000);
 });
 

gui/index.html

@@ -1,281 +1,469 @@
 <!DOCTYPE html>
 <html lang="en" ng-app="syncthing">
 <head>
-<meta charset="utf-8">
-<meta http-equiv="X-UA-Compatible" content="IE=edge">
-<meta name="viewport" content="width=device-width, initial-scale=1.0">
-<meta name="description" content="">
-<meta name="author" content="">
-<link rel="shortcut icon" href="favicon.png">
+  <meta charset="utf-8">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <meta name="description" content="">
+  <meta name="author" content="">
+  <link rel="shortcut icon" href="favicon.png">
 
-<title>syncthing</title>
-<link href="bootstrap/css/bootstrap.min.css" rel="stylesheet">
-<style type="text/css">
+  <title>syncthing</title>
+  <link href="bootstrap/css/bootstrap.min.css" rel="stylesheet">
+  <style type="text/css">
+    body {
+      padding-bottom: 70px;
+      font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+    }
 
-body {
-    padding-top: 70px;
-    padding-bottom: 70px;
-}
+    h1, h2, h3, h4, h5, h6 {
+      font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
+      font-weight: 300;
+    }
 
-.text-monospace {
-    font-family: monospace;
-}
+    a.btn {
+      text-decoration: none;
+    }
 
-.table-condensed>thead>tr>th, .table-condensed>tbody>tr>th, .table-condensed>tfoot>tr>th, .table-condensed>thead>tr>td, .table-condensed>tbody>tr>td, .table-condensed>tfoot>tr>td {
-    border-top: none;
-}
+    ul+h5 {
+      margin-top: 1.5em;
+    }
 
-thead tr th {
-    text-align: center;
-}
+    .text-monospace {
+      font-family: monospace;
+    }
 
-.logo {
-    margin: 0;
-    padding: 0;
-    top: -5px;
-    position: relative;
-}
+    .table-condensed>thead>tr>th, .table-condensed>tbody>tr>th, .table-condensed>tfoot>tr>th, .table-condensed>thead>tr>td, .table-condensed>tbody>tr>td, .table-condensed>tfoot>tr>td {
+      border-top: none;
+    }
 
-</style>
+    thead tr th {
+      text-align: center;
+    }
+
+    .logo {
+      margin: 0;
+      padding: 0;
+      top: -5px;
+      position: relative;
+    }
+
+    .progress {
+      height: 21px;
+      margin-bottom: inherit;
+    }
+
+    .progress .progress-bar {
+      line-height: 21px;
+      font-size: 12px;
+    }
+
+    .collapsed-visible {
+      display: none;
+    }
+    .collapsed .collapsed-visible {
+      display: inline;
+    }
+
+    .list-no-bullet {
+      list-style-type: none
+    }
+
+    .li-column {
+      display: inline-block;
+      min-width: 7em;
+      margin-right: 1em;
+      background-color: rgb(236, 240, 241);
+      border-radius: 3px;
+      padding: 1px 4px;
+      margin: 2px 2px;
+    }
+    .li-column span.data {
+      margin-left: 0.5em;
+      min-width: 10em;
+      text-align: right;
+      display: inline-block;
+    }
+  </style>
 </head>
 
 <body ng-controller="SyncthingCtrl">
-<div class="navbar navbar-fixed-top navbar-default">
-    <div class="container">
-        <a class="navbar-brand"><img class="logo" src="st-logo-128.png" width="32" height="32"> Syncthing</a>
-        <div ng-if="!configInSync">
-            <form class="navbar-form navbar-right">
-                <button type="button" class="btn btn-primary" ng-click="restart()">Restart Now</button>
-            </form>
-            <p class="navbar-text navbar-right">The configuration has been changed but not activated. Syncthing must restart to activate the new configuration.</p>
-        </div>
-    </div>
-</div>
 
-<div class="container">
-    <div class="row">
-        <div class="col-md-12">
-            <div ng-if="errorList().length > 0" class="alert alert-warning">
-                <p ng-repeat="err in errorList()"><small>{{err.Time | date:"hh:mm:ss.sss"}}:</small> {{friendlyNodes(err.Error)}}</p>
-                    <button type="button" class="pull-right btn btn-warning" ng-click="clearErrors()">OK</button>
+  <!-- Top bar -->
+
+  <nav class="navbar navbar-top navbar-default" role="navigation">
+    <div class="container">
+      <span class="navbar-brand"><img class="logo" src="st-logo-128.png" width="32" height="32"> Syncthing</span>
+      <button type="button" class="btn btn-default btn-sm pull-right navbar-btn" ng-click="editSettings()"><span class="glyphicon glyphicon-cog"></span> Settings</button>
+    </div>
+  </nav>
+
+  <div class="container">
+
+    <!-- First row, only shown if necessary; Restart warning -->
+
+    <div ng-if="!configInSync" class="row">
+      <div class="col-md-12">
+        <div class="panel panel-warning">
+          <div class="panel-heading"><h3 class="panel-title">Restart Needed</h3></div>
+          <div class="panel-body">
+            <p>The configuration has been saved but not activated. Syncthing must restart to activate the new configuration.</p>
+          </div>
+          <div class="panel-footer">
+            <button type="button" class="btn btn-sm btn-default pull-right" ng-click="restart()"><span class="glyphicon glyphicon-off"></span> Restart Now</button>
             <div class="clearfix"></div>
-            </div>
-
-            <div class="panel panel-info">
-                <div class="panel-heading"><h3 class="panel-title">Cluster</h3></div>
-                <table class="table table-condensed">
-                    <tbody>
-                    <!-- myself -->
-                    <tr class="text-muted" ng-repeat="nodeCfg in thisNode()">
-                        <td style="width:12%">
-                            <span class="label label-default">
-                                <span class="glyphicon glyphicon-ok"></span> This node
-                            </span>
-                        </td>
-                        <td style="width:10%">
-                            <span class="text-monospace">{{nodeName(nodeCfg)}}</span>
-                        </td>
-                        <td style="width:20%">{{version}}</td>
-                        <td style="width:25%">(this node)</td>
-                        <td style="width:9%" class="text-right">
-                            {{inbps | metric}}bps
-                            <span class="text-muted glyphicon glyphicon-chevron-down"></span>
-                        </td>
-                        <td style="width:9%" class="text-right">
-                            {{outbps | metric}}bps
-                            <span class="text-muted glyphicon glyphicon-chevron-up"></span>
-                        </td>
-                        <td style="width:7%" class="text-right">
-                            <button type="button" ng-click="editNode(nodeCfg)" class="btn btn-default btn-xs"><span class="glyphicon glyphicon-pencil"></span> Edit</button>
-                        </td>
-                    </tr>
-                    <!-- all other nodes -->
-                    <tr ng-repeat="nodeCfg in otherNodes()">
-                        <td>
-                            <span class="label label-{{nodeClass(nodeCfg)}}">
-                                <span class="glyphicon glyphicon-{{nodeIcon(nodeCfg)}}"></span> {{nodeStatus(nodeCfg)}}
-                            </span>
-                        </td>
-                        <td>
-                            <span class="text-monospace">{{nodeName(nodeCfg)}}</span>
-                        </td>
-                        <td>{{nodeVer(nodeCfg)}}</td>
-                        <td>{{nodeAddr(nodeCfg)}}</td>
-                        <td class="text-right">
-                            <abbr title="{{connections[nodeCfg.NodeID].InBytesTotal | binary}}B">{{connections[nodeCfg.NodeID].inbps | metric}}bps</abbr>
-                            <span class="text-muted glyphicon glyphicon-chevron-down"></span>
-                        </td>
-                        <td class="text-right">
-                            <abbr title="{{connections[nodeCfg.NodeID].OutBytesTotal | binary}}B">{{connections[nodeCfg.NodeID].outbps | metric}}bps</abbr>
-                            <span class="text-muted glyphicon glyphicon-chevron-up"></span>
-                        </td>
-                        <td class="text-right">
-                            <button type="button" ng-click="editNode(nodeCfg)" class="btn btn-default btn-xs"><span class="glyphicon glyphicon-pencil"></span> Edit</button>
-                        </td>
-                    </tr>
-                    <tr>
-                        <td></td>
-                        <td></td>
-                        <td></td>
-                        <td></td>
-                        <td></td>
-                        <td></td>
-                        <td class="text-right">
-                            <button type="button" class="btn btn-default btn-xs" ng-click="addNode()"><span class="glyphicon glyphicon-plus"></span> Add</button>
-                        </td>
-                    </tr>
-                    </tbody>
-                </table>
-            </div>
+          </div>
         </div>
+      </div>
     </div>
 
+    <!-- First regular row -->
+
     <div class="row">
-        <div class="col-md-6">
-            <div class="panel panel-info">
-                <div class="panel-heading"><h3 class="panel-title">Repository</h3></div>
-                <div class="panel-body">
-                    <p>Cluster contains {{model.globalFiles | alwaysNumber}} files, {{model.globalBytes | binary}}B
-                    <span class="text-muted">(+{{model.globalDeleted | alwaysNumber}} delete records)</span></p>
 
-                    <p>Local repository has {{model.localFiles | alwaysNumber}} files, {{model.localBytes | binary}}B
-                    <span class="text-muted">(+{{model.localDeleted | alwaysNumber}} delete records)</span></p>
-                </div>
-            </div>
-        </div>
-        <div class="col-md-6">
-            <div class="panel" ng-class="{'panel-success': model.needBytes === 0, 'panel-primary': model.needBytes !== 0}">
-                <div class="panel-heading"><h3 class="panel-title">Synchronization</h3></div>
-                <div class="panel-body">
-                    <div class="progress">
-                        <div class="progress-bar" role="progressbar" aria-valuenow="60" aria-valuemin="0" aria-valuemax="100"
-                            ng-class="{'progress-bar-success': model.needBytes === 0, 'progress-bar-info': model.needBytes !== 0}"
-                            ng-style="{width: (100 * model.inSyncBytes / model.globalBytes) + '%'}">
-                            {{100 * model.inSyncBytes / model.globalBytes | alwaysNumber | number:0}}%
-                        </div>
+      <!-- Repository list (top left) -->
+
+      <div class="col-md-6">
+        <div class="panel panel-default">
+          <div class="panel-heading"><h3 class="panel-title">Repositories</h3></div>
+          <div class="panel-body">
+            <ul class="list-unstyled" ng-repeat="repo in repos">
+              <li>
+                <span class="text-monospace">{{repo.Directory}}</span>
+                <ul class="list-no-bullet">
+                  <li>
+                    <div class="li-column">
+                      <span class="text-muted glyphicon glyphicon-tag"></span>
+                      <span class="data">{{repo.ID}}</span>
                     </div>
-                    <p ng-show="model.needBytes > 0">Need {{model.needFiles | alwaysNumber}} files, {{model.needBytes | binary}}B</p>
-                </div>
-            </div>
+                    <div class="li-column">
+                      <span class="text-muted glyphicon glyphicon-comment"></span>
+                      <span class="data" ng-class="repoClass(repo.ID)">{{repoStatus(repo.ID)}}</span>
+                    </div>
+                  </li>
+                  <li>
+                    <div class="li-column">
+                      <span class="text-muted glyphicon glyphicon-globe"></span>
+                      <span class="data">{{model[repo.ID].globalFiles | alwaysNumber}} files, {{model[repo.ID].globalBytes | binary}}B</span>
+                    </div>
+                    <div class="li-column">
+                      <span class="text-muted glyphicon glyphicon-home"></span>
+                      <span class="data">{{model[repo.ID].localFiles | alwaysNumber}} files, {{model[repo.ID].localBytes | binary}}B</span>
+                    </div>
+                  </li>
+                  <li>
+                    <div class="li-column">
+                      <span class="text-muted glyphicon glyphicon-cloud-download"></span>
+                      <span class="data">{{model[repo.ID].needFiles | alwaysNumber}} files, {{model[repo.ID].needBytes | binary}}B</span>
+                    </div>
+                    <div class="li-column">
+                      <span class="text-muted glyphicon glyphicon-cog"></span>
+                      <span class="data"><a href="" ng-click="editRepo(repo)"><span class="glyphicon glyphicon-pencil"></span> Edit</a></span>
+                    </div>
+                  </li>
+                </ul>
+              </li>
+            </ul>
+          </div>
+          <div class="panel-footer">
+            <button type="button" class="pull-right btn btn-sm btn-default" ng-click="addRepo()"><span class="glyphicon glyphicon-plus"></span> Add Repository</button>
+            <div class="clearfix"></div>
+          </div>
         </div>
+      </div>
+
+      <!-- Node list (top right) -->
+
+      <div class="col-md-6">
+        <div class="panel panel-default">
+          <div class="panel-heading"><h3 class="panel-title">Nodes</h3></div>
+          <div class="panel-body">
+
+            <h5>Peer Nodes</h5>
+            <ul class="list-unstyled" ng-repeat="nodeCfg in otherNodes()">
+              <li>
+                <span class="text-monospace">{{nodeName(nodeCfg)}}</span>
+                <ul class="list-no-bullet">
+                  <li>
+                    <div class="li-column">
+                      <span class="text-muted glyphicon glyphicon-link"></span>
+                      <span class="data">{{nodeAddr(nodeCfg)}}</span>
+                    </div>
+                    <div class="li-column">
+                      <span class="text-muted glyphicon glyphicon-comment"></span>
+                      <span class="data text-{{nodeClass(nodeCfg)}}">{{nodeStatus(nodeCfg)}}</span>
+                    </div>
+                  </li>
+                  <li>
+                    <div class="li-column">
+                      <span class="text-muted glyphicon glyphicon-cloud-download"></span>
+                      <span class="data">{{connections[nodeCfg.NodeID].inbps | metric}}bps</span>
+                    </div>
+                    <div class="li-column">
+                      <span class="text-muted glyphicon glyphicon-cloud-upload"></span>
+                      <span class="data">{{connections[nodeCfg.NodeID].outbps | metric}}bps</span>
+                    </div>
+                  </li>
+                  <li>
+                    <div class="li-column">
+                      <span class="text-muted glyphicon glyphicon-tag"></span>
+                      <span class="data">{{nodeVer(nodeCfg)}}</span>
+                    </div>
+                    <div class="li-column">
+                      <span class="text-muted glyphicon glyphicon-cog"></span>
+                      <span class="data"><a href="" ng-click="editNode(nodeCfg)"><span class="glyphicon glyphicon-pencil"></span> Edit</a></span>
+                    </div>
+                  </li>
+                </ul>
+              </li>
+            </ul>
+
+            <h5>This Node</h5>
+            <ul class="list-unstyled" ng-repeat="nodeCfg in thisNode()">
+              <li>
+                <span class="text-monospace">{{nodeName(nodeCfg)}}</span>&emsp;
+                <ul class="list-no-bullet">
+                  <li>
+                    <div class="li-column">
+                      <span class="text-muted glyphicon glyphicon-th"></span>
+                      <span class="data">{{system.sys | binary}}B RAM</span>
+                    </div>
+                    <div class="li-column">
+                      <span class="text-muted glyphicon glyphicon-tasks"></span>
+                      <span class="data">{{system.cpuPercent | alwaysNumber | natural:1}}% CPU</span>
+                    </div>
+                  </li>
+                  <li>
+                    <div class="li-column">
+                      <span class="text-muted glyphicon glyphicon-cloud-download"></span>
+                      <span class="data">{{inbps | metric}}bps</span>
+                    </div>
+                    <div class="li-column">
+                      <span class="text-muted glyphicon glyphicon-cloud-upload"></span>
+                      <span class="data">{{outbps | metric}}bps</span>
+                    </div>
+                  </li>
+                  <li>
+                    <div class="li-column">
+                      <span class="text-muted glyphicon glyphicon-cog"></span>
+                      <span class="data"><a href="" ng-click="editNode(nodeCfg)"><span class="glyphicon glyphicon-pencil"></span> Edit</a></span>
+                    </div>
+                  </li>
+                </ul>
+              </li>
+            </ul>
+          </div>
+          <div class="panel-footer">
+            <button type="button" class="pull-right btn btn-sm btn-default" ng-click="addNode()"><span class="glyphicon glyphicon-plus"></span> Add Node</button>
+            <div class="clearfix"></div>
+          </div>
+        </div>
+      </div>
+
+    </div> <!-- /row -->
+
+    <!-- Errors -->
+
+    <div ng-if="errorList().length > 0" class="row">
+      <div class="col-md-12">
+        <div class="panel panel-warning">
+          <div class="panel-heading"><h3 class="panel-title">Notice</h3></div>
+          <div class="panel-body">
+            <p ng-repeat="err in errorList()"><small>{{err.Time | date:"hh:mm:ss.sss"}}:</small> {{friendlyNodes(err.Error)}}</p>
+          </div>
+          <div class="panel-footer">
+            <button type="button" class="pull-right btn btn-sm btn-default" ng-click="clearErrors()">OK</button>
+            <div class="clearfix"></div>
+          </div>
+        </div>
+      </div>
     </div>
 
-    <div class="row">
-        <div class="col-md-6">
-            <div class="panel panel-info">
-                <div class="panel-heading"><h3 class="panel-title"><a href="" data-toggle="collapse" data-target="#system">System</a></h3></div>
-                <div id="system" class="panel-collapse collapse">
-                    <div class="panel-body">
-                        <p>{{system.sys | binary}}B RAM allocated, {{system.alloc | binary}}B in use</p>
-                        <p>{{system.cpuPercent | alwaysNumber | natural:1}}% CPU, {{system.goroutines | alwaysNumber}} goroutines</p>
-                    </div>
-                </div>
-            </div>
-        </div>
-        <div class="col-md-6">
-            <div class="panel panel-info">
-                <div class="panel-heading"><h3 class="panel-title"><a href="" data-toggle="collapse" data-target="#settingsTable">Settings</a></h3></div>
-                <div id="settingsTable" class="panel-collapse collapse">
-                    <div class="panel-body">
-                        <form role="form">
-                            <div class="form-group" ng-repeat="setting in settings">
-                                <div ng-if="setting.type == 'text' || setting.type == 'number'">
-                                    <label for="{{setting.id}}">{{setting.descr}}</label>
-                                    <input id="{{setting.id}}" class="form-control" type="{{setting.type}}" ng-model="config.Options[setting.id]"></input>
-                                </div>
-                                <div class="checkbox" ng-if="setting.type == 'bool'">
-                                    <label>
-                                        {{setting.descr}} <input id="{{setting.id}}" type="checkbox" ng-model="config.Options[setting.id]"></input>
-                                    </label>
-                                </div>
-                            </div>
-                        </form>
-                    </div>
-                    <div class="panel-footer">
-                        <button type="button" class="btn btn-sm btn-default" ng-click="saveSettings()">Save</button>
-                        <small><span class="text-muted">Changes take effect when restarting syncthing.</span></small>
-                    </div>
-                </div>
-            </div>
-        </div>
-    </div>
-</div>
+  </div> <!-- /container -->
 
-<div class="navbar navbar-default navbar-fixed-bottom">
+  <!-- Bottom bar -->
+
+  <nav class="navbar navbar-default navbar-fixed-bottom hidden-xs">
     <div class="container">
-        <p class="navbar-text">{{version}}</p>
-        <ul class="nav navbar-nav navbar-right">
-            <li><a class="navbar-link" href="http://discourse.syncthing.net/">Support / Forum</a></li>
-            <li><a class="navbar-link" href="https://github.com/calmh/syncthing/releases">Latest Release</a></li>
-            <li><a class="navbar-link" href="https://github.com/calmh/syncthing/wiki">Documentation</a></li>
-            <li><a class="navbar-link" href="https://github.com/calmh/syncthing/issues">Bugs</a></li>
-            <li><a class="navbar-link" href="https://github.com/calmh/syncthing">Source Code</a></li>
-        </ul>
-        </p>
+      <p class="navbar-text">{{version}}</p>
+      <ul class="nav navbar-nav navbar-right">
+        <li><a class="navbar-link" href="http://discourse.syncthing.net/">Support / Forum</a></li>
+        <li><a class="navbar-link hidden-sm" href="https://github.com/calmh/syncthing/releases">Latest Release</a></li>
+        <li><a class="navbar-link" href="https://github.com/calmh/syncthing/wiki">Documentation</a></li>
+        <li><a class="navbar-link hidden-sm" href="https://github.com/calmh/syncthing/issues">Bugs</a></li>
+        <li><a class="navbar-link hidden-sm" href="https://github.com/calmh/syncthing">Source Code</a></li>
+      </ul>
     </div>
-</div>
+  </nav>
 
-<div id="networkError" class="modal fade">
+  <!-- Network error modal -->
+
+  <div id="networkError" class="modal fade">
     <div class="modal-dialog">
-        <div class="modal-content">
-            <div class="modal-header alert alert-danger">
-                <h4 class="modal-title">
-                    <span class="glyphicon glyphicon-exclamation-sign"></span>
-                    Connection Error
-                </h4>
-            </div>
-            <div class="modal-body">
-                <p>
-                Syncthing seems to be down, or there is a problem with your Internet connection.
-                Retrying&hellip;
-                </p>
-            </div>
+      <div class="modal-content">
+        <div class="modal-header alert alert-danger">
+          <h4 class="modal-title">
+            <span class="glyphicon glyphicon-exclamation-sign"></span>
+            Connection Error
+          </h4>
         </div>
+        <div class="modal-body">
+          <p>
+            Syncthing seems to be down, or there is a problem with your Internet connection.
+            Retrying&hellip;
+          </p>
+        </div>
+      </div>
     </div>
-</div>
+  </div>
 
-<div id="editNode" class="modal fade">
+  <!-- Node editor modal -->
+
+  <div id="editNode" class="modal fade">
     <div class="modal-dialog modal-lg">
-        <div class="modal-content">
-            <div class="modal-header">
-                <button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
-                <h4 class="modal-title">Edit Node</h4>
-            </div>
-            <div class="modal-body">
-                <form role="form">
-                    <div class="form-group">
-                        <label for="nodeID">Node ID</label>
-                        <input placeholder="YUFJOUDPORCMA..." ng-disabled="editingExisting" id="nodeID" class="form-control" type="text" ng-model="currentNode.NodeID"></input>
-                        <p class="help-block">The node ID can be found in the logs or in the "Add Node" dialog on the other node.</p>
-                    </div>
-                    <div class="form-group">
-                        <label for="name">Name</label>
-                        <input placeholder="Home Server" id="name" class="form-control" type="text" ng-model="currentNode.Name"></input>
-                        <p class="help-block">Shown instead of Node ID in the cluster status.</p>
-                    </div>
-                    <div class="form-group">
-                        <label for="addresses">Addresses</label>
-                        <input placeholder="dynamic" ng-disabled="currentNode.NodeID == myID" id="addresses" class="form-control" type="text" ng-model="currentNode.AddressesStr"></input>
-                        <p class="help-block">Enter comma separated <span class="text-monospace">ip:port</span> addresses or <span class="text-monospace">dynamic</span> to perform automatic discovery of the address.</p>
-                    </div>
-                </form>
-                <div ng-show="!editingExisting">
-                    When adding a new node, keep in mind that <em>this node</em> must be added on the other side too. The Node ID of this node is:
-                    <pre>{{myID}}</pre>
-                </div>
-            </div>
-            <div class="modal-footer">
-                <button type="button" class="btn btn-primary" ng-click="saveNode()">Save</button>
-                <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
-                <button ng-if="editingExisting" type="button" class="btn btn-danger pull-left" ng-click="deleteNode()">Delete</button>
-            </div>
+      <div class="modal-content">
+        <div class="modal-header">
+          <button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
+          <h4 ng-show="!editingExisting" class="modal-title">Add Node</h4>
+          <h4 ng-show="editingExisting" class="modal-title">Edit Node</h4>
         </div>
+        <div class="modal-body">
+          <form role="form">
+            <div class="form-group">
+              <label for="nodeID">Node ID</label>
+              <input ng-disabled="editingExisting" id="nodeID" class="form-control" type="text" ng-model="currentNode.NodeID"></input>
+              <p class="help-block">The node ID can be found in the logs or in the "Add Node" dialog on the other node.</p>
+            </div>
+            <div class="form-group">
+              <label for="name">Name</label>
+              <input placeholder="Home Server" id="name" class="form-control" type="text" ng-model="currentNode.Name"></input>
+              <p class="help-block">Shown instead of Node ID in the cluster status.</p>
+            </div>
+            <div class="form-group">
+              <label for="addresses">Addresses</label>
+              <input placeholder="dynamic" ng-disabled="currentNode.NodeID == myID" id="addresses" class="form-control" type="text" ng-model="currentNode.AddressesStr"></input>
+              <p class="help-block">Enter comma separated <span class="text-monospace">ip:port</span> addresses or <span class="text-monospace">dynamic</span> to perform automatic discovery of the address.</p>
+            </div>
+          </form>
+          <div ng-show="!editingExisting">
+            When adding a new node, keep in mind that <em>this node</em> must be added on the other side too. The Node ID of this node is:
+            <pre>{{myID}}</pre>
+          </div>
+        </div>
+        <div class="modal-footer">
+          <button type="button" class="btn btn-primary" ng-click="saveNode()">Save</button>
+          <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
+          <button ng-if="editingExisting" type="button" class="btn btn-danger pull-left" ng-click="deleteNode()">Delete</button>
+        </div>
+      </div>
     </div>
-</div>
+  </div>
 
-<script src="angular.min.js"></script>
-<script src="jquery-2.0.3.min.js"></script>
-<script src="bootstrap/js/bootstrap.min.js"></script>
-<script src="app.js"></script>
+  <!-- Repo editor modal -->
+
+  <div id="editRepo" class="modal fade">
+    <div class="modal-dialog modal-lg">
+      <div class="modal-content">
+        <div class="modal-header">
+          <button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
+          <h4 ng-show="!editingExisting" class="modal-title">Add Repository</h4>
+          <h4 ng-show="editingExisting" class="modal-title">Edit Repository</h4>
+        </div>
+        <div class="modal-body">
+          <form role="form">
+            <div class="form-group">
+              <label for="repoID">Repository ID</label>
+              <input placeholder="documents" ng-disabled="editingExisting" id="repoID" class="form-control" type="text" ng-model="currentRepo.ID"></input>
+              <p class="help-block">Short, unique identifier for the repository. Must be the same on all cluster nodes.</p>
+            </div>
+            <div class="form-group">
+              <label for="repoPath">Repository Path</label>
+              <input placeholder="~/Documents" id="repoPath" class="form-control" type="text" ng-model="currentRepo.Directory"></input>
+              <p class="help-block">Path to the repository on the local computer. Will be created if it does not exist.</p>
+            </div>
+            <div class="form-group">
+              <div class="checkbox">
+                <label>
+                  <input type="checkbox" ng-model="currentRepo.ReadOnly"> Read Only
+                </label>
+              </div>
+            </div>
+            <div class="form-group">
+              <label for="nodes">Nodes</label>
+              <div class="checkbox" ng-repeat="node in otherNodes()">
+                <label>
+                  <input type="checkbox" ng-model="currentRepo.selectedNodes[node.NodeID]"> {{nodeName(node)}}
+                </label>
+              </div>
+              <p class="help-block">Select the nodes to share this repository with.</p>
+            </div>
+          </form>
+          <div ng-show="!editingExisting">
+            When adding a new repository, keep in mind that the Repository ID is used to tie repositories together between nodes. They are case sensitive and must match exactly between all nodes.
+          </div>
+        </div>
+        <div class="modal-footer">
+          <button type="button" class="btn btn-primary" ng-click="saveRepo()">Save</button>
+          <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
+          <button ng-if="editingExisting" type="button" class="btn btn-danger pull-left" ng-click="deleteRepo()">Delete</button>
+        </div>
+      </div>
+    </div>
+  </div>
+
+  <!-- Settings modal -->
+
+  <div id="settings" class="modal fade">
+    <div class="modal-dialog modal-lg">
+      <div class="modal-content">
+        <div class="modal-header">
+          <button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
+          <h4 class="modal-title"> Settings</h4>
+        </div>
+        <div class="modal-body">
+          <form role="form">
+            <div class="row">
+              <div class="col-md-6">
+                <div class="form-group" ng-repeat="setting in settings">
+                  <div ng-if="setting.type == 'text' || setting.type == 'number'">
+                    <label for="{{setting.id}}">{{setting.descr}}</label>
+                    <input id="{{setting.id}}" class="form-control" type="{{setting.type}}" ng-model="config.Options[setting.id]"></input>
+                  </div>
+                  <div class="checkbox" ng-if="setting.type == 'bool'">
+                    <label>
+                      {{setting.descr}} <input id="{{setting.id}}" type="checkbox" ng-model="config.Options[setting.id]"></input>
+                    </label>
+                  </div>
+                </div>
+              </div>
+              <div class="col-md-6">
+                <div class="form-group" ng-repeat="setting in guiSettings">
+                  <div ng-if="setting.type == 'text' || setting.type == 'number' || setting.type == 'password'">
+                    <label for="{{setting.id}}">{{setting.descr}}</label>
+                    <input id="{{setting.id}}" class="form-control" type="{{setting.type}}" ng-model="config.GUI[setting.id]"></input>
+                  </div>
+                  <div class="checkbox" ng-if="setting.type == 'bool'">
+                    <label>
+                      {{setting.descr}} <input id="{{setting.id}}" type="checkbox" ng-model="config.GUI[setting.id]"></input>
+                    </label>
+                  </div>
+                </div>
+              </div>
+            </div>
+          </form>
+        </div>
+        <div class="modal-footer">
+          <button type="button" class="btn btn-primary" ng-click="saveSettings()">Save</button>
+          <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
+        </div>
+      </div>
+    </div>
+  </div>
+
+
+  <script src="angular.min.js"></script>
+  <script src="jquery-2.0.3.min.js"></script>
+  <script src="bootstrap/js/bootstrap.min.js"></script>
+  <script src="app.js"></script>
 </body>
 </html>

integration/h1/config.xml

@@ -30,7 +30,7 @@
         <localAnnounceEnabled>true</localAnnounceEnabled>
         <parallelRequests>16</parallelRequests>
         <maxSendKbps>0</maxSendKbps>
-        <rescanIntervalS>60</rescanIntervalS>
+        <rescanIntervalS>10</rescanIntervalS>
         <reconnectionIntervalS>5</reconnectionIntervalS>
         <maxChangeKbps>1000</maxChangeKbps>
         <startBrowser>false</startBrowser>

integration/h2/config.xml

@@ -38,7 +38,7 @@
         <localAnnounceEnabled>true</localAnnounceEnabled>
         <parallelRequests>16</parallelRequests>
         <maxSendKbps>0</maxSendKbps>
-        <rescanIntervalS>60</rescanIntervalS>
+        <rescanIntervalS>15</rescanIntervalS>
         <reconnectionIntervalS>5</reconnectionIntervalS>
         <maxChangeKbps>1000</maxChangeKbps>
         <startBrowser>false</startBrowser>

integration/h3/config.xml

@@ -30,7 +30,7 @@
         <localAnnounceEnabled>true</localAnnounceEnabled>
         <parallelRequests>16</parallelRequests>
         <maxSendKbps>0</maxSendKbps>
-        <rescanIntervalS>60</rescanIntervalS>
+        <rescanIntervalS>20</rescanIntervalS>
         <reconnectionIntervalS>5</reconnectionIntervalS>
         <maxChangeKbps>1000</maxChangeKbps>
         <startBrowser>false</startBrowser>

integration/test.sh

@@ -2,7 +2,7 @@
 
 export STNORESTART=1
 
-iterations=5
+iterations=${1:-5}
 
 id1=I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA
 id2=JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ
@@ -12,12 +12,14 @@ go build genfiles.go
 go build md5r.go
 go build json.go
 
-testConvergence() {
+start() {
 	echo "Starting..."
 	for i in 1 2 3 ; do
 		STPROFILER=":909$i" syncthing -home "h$i" &
 	done
+}
 
+testConvergence() {
 	while true ; do
 		sleep 5
 		s1comp=$(curl -s "http://localhost:8082/rest/connections" | ./json "$id1/Completion")
@@ -29,8 +31,6 @@ testConvergence() {
 		tot=$(($s1comp + $s2comp + $s3comp))
 		echo $tot / 300
 		if [[ $tot == 300 ]] ; then
-			echo "Stopping..."
-			pkill syncthing
 			break
 		fi
 	done
@@ -72,10 +72,37 @@ testConvergence() {
 		fi
 	done
 	if [[ $ok != 7 ]] ; then
+		pkill syncthing
 		exit 1
 	fi
 }
 
+alterFiles() {
+	pkill -STOP syncthing
+	for i in 1 2 3 12-1 12-2 23-2 23-3 ; do
+		pushd "s$i" >/dev/null
+
+		nfiles=$(find . -type f | wc -l)
+		if [[ $nfiles > 2000 ]] ; then
+			todelete=$(( $nfiles - 2000 ))
+			echo "Deleting $todelete files..."
+			find . -type f \
+				| grep -v large \
+				| sort -k 1.16 \
+				| head -n "$todelete" \
+				| xargs rm -f
+		fi
+
+		../genfiles -maxexp 22 -files 600
+		echo "  $i: append to large file"
+		dd if=/dev/urandom bs=1024k count=4 >> large-$i 2>/dev/null
+		../md5r -l > ../md5-tmp
+		(grep -v large ../md5-tmp ; grep "large-$i" ../md5-tmp) | grep -v '/.syncthing.' > ../md5-$i
+		popd >/dev/null
+	done
+	pkill -CONT syncthing
+}
+
 echo "Setting up files..."
 for i in 1 2 3 12-1 12-2 23-2 23-3; do
 	rm -f h$i/*.idx.gz
@@ -98,21 +125,16 @@ for i in 1 2 3 12-1 12-2 23-2 23-3 ; do
 	popd >/dev/null
 done
 
+start
 testConvergence
 
-for ((t = 0; t < $iterations; t++)) ; do
-	echo "Add and remove random files ($((t+1)) / $iterations)..."
-	for i in 1 2 3 12-1 12-2 23-2 23-3 ; do
-		pushd "s$i" >/dev/null
-		rm -rf */?[02468ace]
-		../genfiles -maxexp 22 -files 600
-		echo "  $i: append to large file"
-		dd if=/dev/urandom bs=1024k count=4 >> large-$i 2>/dev/null
-		../md5r -l > ../md5-tmp
-		(grep -v large ../md5-tmp ; grep "large-$i" ../md5-tmp) > ../md5-$i
-		popd >/dev/null
-	done
+for ((t = 1; t <= $iterations; t++)) ; do
+	echo "Add and remove random files ($t / $iterations)..."
+	alterFiles
 
+	echo "Waiting..."
+	sleep 30
 	testConvergence
 done
 
+pkill syncthing

protocol/PROTOCOL.md

@@ -79,10 +79,14 @@ version, type and ID.
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 For BEP v1 the Version field is set to zero. Future versions with
-incompatible message formats will increment the Version field.
+incompatible message formats will increment the Version field. A message
+with an unknown version is a protocol error and MUST result in the
+connection being terminated. A client supporting multiple versions MAY
+retry with a different protcol version upon disconnection.
 
 The Type field indicates the type of data following the message header
-and is one of the integers defined below.
+and is one of the integers defined below. A message of an unknown type
+is a protocol error and MUST result in the connection being terminated.
 
 The Message ID is set to a unique value for each transmitted message. In
 request messages the Reply To is set to zero. In response messages it is
@@ -110,6 +114,183 @@ Opaque data should not be interpreted but can be compared bytewise to
 other opaque data. All strings MUST use the Unicode UTF-8 encoding,
 normalization form C.
 
+### Cluster Config (Type = 0)
+
+This informational message provides information about the cluster
+configuration, as it pertains to the current connection. It is sent by
+both sides after connection establishment.
+
+#### Graphical Representation
+
+    ClusterConfigMessage Structure:
+
+     0                   1                   2                   3
+     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                     Length of ClientName                      |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \                 ClientName (variable length)                  \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                    Length of ClientVersion                    |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \                ClientVersion (variable length)                \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                    Number of Repositories                     |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \              Zero or more Repository Structures               \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                       Number of Options                       |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \                Zero or more Option Structures                 \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+    Repository Structure:
+
+     0                   1                   2                   3
+     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                         Length of ID                          |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \                     ID (variable length)                      \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                        Number of Nodes                        |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \                 Zero or more Node Structures                  \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+    Node Structure:
+
+     0                   1                   2                   3
+     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                         Length of ID                          |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \                     ID (variable length)                      \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                             Flags                             |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+    Option Structure:
+
+     0                   1                   2                   3
+     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                         Length of Key                         |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \                     Key (variable length)                     \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                        Length of Value                        |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    /                                                               /
+    \                    Value (variable length)                    \
+    /                                                               /
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+#### Fields
+
+The ClientName and ClientVersion fields identify the implementation. The
+values SHOULD be simple strings identifying the implementation name, as
+a user would expect to see it, and the version string in the same
+manner. An example ClientName is "syncthing" and an example
+ClientVersion is "v0.7.2". The ClientVersion field SHOULD follow the
+patterns laid out in the [Semantic Versioning](http://semver.org/)
+standard.
+
+The Repositories field lists all repositories that will be synchronized
+over the current connection. Each repository has a list of participating
+Nodes. Each node has an associated Flags field to indicate the sharing
+mode of that node for the repository in question. See the discussion on
+Sharing Modes.
+
+The Node Flags field contains the following single bit flags:
+
+     0                   1                   2                   3
+     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |          Reserved         |Pri|          Reserved         |R|T|
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+ - Bit 31 ("T", Trusted) is set for nodes that participate in trusted
+   mode.
+
+ - Bit 30 ("R", Read Only) is set for nodes that participate in read
+   only mode.
+
+ - Bits 16 through 28 are reserved and MUST be set to zero.
+
+ - Bits 14-15 ("Pri) indicate the node's upload priority for this
+   repository. Possible values are:
+
+    - 00: The default. Normal priority.
+
+    - 01: High priority. Other nodes SHOULD favour requesting files from
+      this node over nodes with normal or low priority.
+
+    - 10: Low priority. Other nodes SHOULD avoid requesting files from
+      this node when they are available from other nodes.
+
+    - 11: Sharing disabled. Other nodes SHOULD NOT request files from
+      this node.
+
+ - Bits 0 through 14 are reserved and MUST be set to zero.
+
+Exactly one of the T, R or S bits MUST be set.
+
+The Options field contain option values to be used in an implementation
+specific manner. The options list is conceptually a map of Key => Value
+items, although it is transmitted in the form of a list of (Key, Value)
+pairs, both of string type. Key ID:s are implementation specific. An
+implementation MUST ignore unknown keys. An implementation MAY impose
+limits on the length keys and values. The options list may be used to
+inform nodes of relevant local configuration options such as rate
+limiting or make recommendations about request parallellism, node
+priorities, etc. An empty options list is valid for nodes not having any
+such information to share. Nodes MAY NOT make any assumptions about
+peers acting in a specific manner as a result of sent options.
+
+#### XDR
+
+    struct ClusterConfigMessage {
+        string ClientName<>;
+        string ClientVersion<>;
+        Repository Repositories<>;
+        Option Options<>;
+    }
+
+    struct Repository {
+        string ID<>;
+        Node Nodes<>;
+    }
+
+    struct Node {
+        string ID<>;
+        unsigned int Flags;
+    }
+
+    struct Option {
+        string Key<>;
+        string Value<>;
+    }
+
 ### Index (Type = 1)
 
 The Index message defines the contents of the senders repository. An
@@ -356,65 +537,32 @@ model, the Index Update merely amends it with new or updated file
 information. Any files not mentioned in an Index Update are left
 unchanged.
 
-### Options (Type = 7)
+Sharing Modes
+-------------
 
-This informational message provides information about the client
-configuration, version, etc. It is sent at connection initiation and,
-optionally, when any of the sent parameters have changed. The message is
-in the form of a list of (key, value) pairs, both of string type.
+### Trusted
 
-Key ID:s apart from the well known ones are implementation specific. An
-implementation is expected to ignore unknown keys. An implementation may
-impose limits on key and value size.
+Trusted mode is the default sharing mode. Updates are exchanged in both
+directions.
 
-Well known keys:
+    +------------+     Updates      /---------\
+    |            |  ----------->   /           \
+    |    Node    |                 |  Cluster  |
+    |            |  <-----------   \           /
+    +------------+     Updates      \---------/
 
-  - "clientId" -- The name of the implementation. Example: "syncthing".
+### Read Only
 
-  - "clientVersion" -- The version of the client. Example: "v1.0.33-47".
-    The Following the SemVer 2.0 specification for version strings is
-    encouraged but not enforced.
+In read only mode a node does not synchronize the local repository to
+the cluster, but publishes changes to it's local repository contents as
+usual. The local repository can be seen as a "master copy" that is never
+affected by the actions of other cluster nodes.
 
-#### Graphical Representation
-
-     0                   1                   2                   3
-     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    |                       Number of Options                       |
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    /                                                               /
-    \               Zero or more KeyValue Structures                \
-    /                                                               /
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-    KeyValue Structure:
-
-     0                   1                   2                   3
-     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    |                         Length of Key                         |
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    /                                                               /
-    \                     Key (variable length)                     \
-    /                                                               /
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    |                        Length of Value                        |
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    /                                                               /
-    \                    Value (variable length)                    \
-    /                                                               /
-    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
-#### XDR
-
-    struct OptionsMessage {
-        KeyValue Options<>;
-    }
-
-    struct KeyValue {
-        string Key<>;
-        string Value<>;
-    }
+    +------------+     Updates      /---------\
+    |            |  ----------->   /           \
+    |    Node    |                 |  Cluster  |
+    |            |                 \           /
+    +------------+                  \---------/
 
 Message Limits
 --------------

protocol/common_test.go

@@ -38,6 +38,9 @@ func (t *TestModel) Close(nodeID string, err error) {
 	close(t.closedCh)
 }
 
+func (t *TestModel) ClusterConfig(nodeID string, config ClusterConfigMessage) {
+}
+
 func (t *TestModel) isClosed() bool {
 	select {
 	case <-t.closedCh:

protocol/message_types.go

@@ -25,8 +25,21 @@ type RequestMessage struct {
 	Size       uint32
 }
 
-type OptionsMessage struct {
-	Options []Option // max:64
+type ClusterConfigMessage struct {
+	ClientName    string       // max:64
+	ClientVersion string       // max:64
+	Repositories  []Repository // max:64
+	Options       []Option     // max:64
+}
+
+type Repository struct {
+	ID    string // max:64
+	Nodes []Node // max:64
+}
+
+type Node struct {
+	ID    string // max:64
+	Flags uint32
 }
 
 type Option struct {

protocol/message_xdr.go

@@ -198,19 +198,34 @@ func (o *RequestMessage) decodeXDR(xr *xdr.Reader) error {
 	return xr.Error()
 }
 
-func (o OptionsMessage) EncodeXDR(w io.Writer) (int, error) {
+func (o ClusterConfigMessage) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
 	return o.encodeXDR(xw)
 }
 
-func (o OptionsMessage) MarshalXDR() []byte {
+func (o ClusterConfigMessage) MarshalXDR() []byte {
 	var buf bytes.Buffer
 	var xw = xdr.NewWriter(&buf)
 	o.encodeXDR(xw)
 	return buf.Bytes()
 }
 
-func (o OptionsMessage) encodeXDR(xw *xdr.Writer) (int, error) {
+func (o ClusterConfigMessage) encodeXDR(xw *xdr.Writer) (int, error) {
+	if len(o.ClientName) > 64 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteString(o.ClientName)
+	if len(o.ClientVersion) > 64 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteString(o.ClientVersion)
+	if len(o.Repositories) > 64 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteUint32(uint32(len(o.Repositories)))
+	for i := range o.Repositories {
+		o.Repositories[i].encodeXDR(xw)
+	}
 	if len(o.Options) > 64 {
 		return xw.Tot(), xdr.ErrElementSizeExceeded
 	}
@@ -221,18 +236,28 @@ func (o OptionsMessage) encodeXDR(xw *xdr.Writer) (int, error) {
 	return xw.Tot(), xw.Error()
 }
 
-func (o *OptionsMessage) DecodeXDR(r io.Reader) error {
+func (o *ClusterConfigMessage) DecodeXDR(r io.Reader) error {
 	xr := xdr.NewReader(r)
 	return o.decodeXDR(xr)
 }
 
-func (o *OptionsMessage) UnmarshalXDR(bs []byte) error {
+func (o *ClusterConfigMessage) UnmarshalXDR(bs []byte) error {
 	var buf = bytes.NewBuffer(bs)
 	var xr = xdr.NewReader(buf)
 	return o.decodeXDR(xr)
 }
 
-func (o *OptionsMessage) decodeXDR(xr *xdr.Reader) error {
+func (o *ClusterConfigMessage) decodeXDR(xr *xdr.Reader) error {
+	o.ClientName = xr.ReadStringMax(64)
+	o.ClientVersion = xr.ReadStringMax(64)
+	_RepositoriesSize := int(xr.ReadUint32())
+	if _RepositoriesSize > 64 {
+		return xdr.ErrElementSizeExceeded
+	}
+	o.Repositories = make([]Repository, _RepositoriesSize)
+	for i := range o.Repositories {
+		(&o.Repositories[i]).decodeXDR(xr)
+	}
 	_OptionsSize := int(xr.ReadUint32())
 	if _OptionsSize > 64 {
 		return xdr.ErrElementSizeExceeded
@@ -244,6 +269,95 @@ func (o *OptionsMessage) decodeXDR(xr *xdr.Reader) error {
 	return xr.Error()
 }
 
+func (o Repository) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o Repository) MarshalXDR() []byte {
+	var buf bytes.Buffer
+	var xw = xdr.NewWriter(&buf)
+	o.encodeXDR(xw)
+	return buf.Bytes()
+}
+
+func (o Repository) encodeXDR(xw *xdr.Writer) (int, error) {
+	if len(o.ID) > 64 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteString(o.ID)
+	if len(o.Nodes) > 64 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteUint32(uint32(len(o.Nodes)))
+	for i := range o.Nodes {
+		o.Nodes[i].encodeXDR(xw)
+	}
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Repository) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *Repository) UnmarshalXDR(bs []byte) error {
+	var buf = bytes.NewBuffer(bs)
+	var xr = xdr.NewReader(buf)
+	return o.decodeXDR(xr)
+}
+
+func (o *Repository) decodeXDR(xr *xdr.Reader) error {
+	o.ID = xr.ReadStringMax(64)
+	_NodesSize := int(xr.ReadUint32())
+	if _NodesSize > 64 {
+		return xdr.ErrElementSizeExceeded
+	}
+	o.Nodes = make([]Node, _NodesSize)
+	for i := range o.Nodes {
+		(&o.Nodes[i]).decodeXDR(xr)
+	}
+	return xr.Error()
+}
+
+func (o Node) EncodeXDR(w io.Writer) (int, error) {
+	var xw = xdr.NewWriter(w)
+	return o.encodeXDR(xw)
+}
+
+func (o Node) MarshalXDR() []byte {
+	var buf bytes.Buffer
+	var xw = xdr.NewWriter(&buf)
+	o.encodeXDR(xw)
+	return buf.Bytes()
+}
+
+func (o Node) encodeXDR(xw *xdr.Writer) (int, error) {
+	if len(o.ID) > 64 {
+		return xw.Tot(), xdr.ErrElementSizeExceeded
+	}
+	xw.WriteString(o.ID)
+	xw.WriteUint32(o.Flags)
+	return xw.Tot(), xw.Error()
+}
+
+func (o *Node) DecodeXDR(r io.Reader) error {
+	xr := xdr.NewReader(r)
+	return o.decodeXDR(xr)
+}
+
+func (o *Node) UnmarshalXDR(bs []byte) error {
+	var buf = bytes.NewBuffer(bs)
+	var xr = xdr.NewReader(buf)
+	return o.decodeXDR(xr)
+}
+
+func (o *Node) decodeXDR(xr *xdr.Reader) error {
+	o.ID = xr.ReadStringMax(64)
+	o.Flags = xr.ReadUint32()
+	return xr.Error()
+}
+
 func (o Option) EncodeXDR(w io.Writer) (int, error) {
 	var xw = xdr.NewWriter(w)
 	return o.encodeXDR(xw)

protocol/nativemodel_darwin.go

@@ -29,6 +29,10 @@ func (m nativeModel) Request(nodeID, repo string, name string, offset int64, siz
 	return m.next.Request(nodeID, repo, name, offset, size)
 }
 
+func (m nativeModel) ClusterConfig(nodeID string, config ClusterConfigMessage) {
+	m.next.ClusterConfig(nodeID, config)
+}
+
 func (m nativeModel) Close(nodeID string, err error) {
 	m.next.Close(nodeID, err)
 }

protocol/nativemodel_unix.go

@@ -20,6 +20,10 @@ func (m nativeModel) Request(nodeID, repo string, name string, offset int64, siz
 	return m.next.Request(nodeID, repo, name, offset, size)
 }
 
+func (m nativeModel) ClusterConfig(nodeID string, config ClusterConfigMessage) {
+	m.next.ClusterConfig(nodeID, config)
+}
+
 func (m nativeModel) Close(nodeID string, err error) {
 	m.next.Close(nodeID, err)
 }

protocol/nativemodel_windows.go

@@ -29,6 +29,10 @@ func (m nativeModel) Request(nodeID, repo string, name string, offset int64, siz
 	return m.next.Request(nodeID, repo, name, offset, size)
 }
 
+func (m nativeModel) ClusterConfig(nodeID string, config ClusterConfigMessage) {
+	m.next.ClusterConfig(nodeID, config)
+}
+
 func (m nativeModel) Close(nodeID string, err error) {
 	m.next.Close(nodeID, err)
 }

protocol/protocol.go

@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"log"
 	"sync"
 	"time"
 
@@ -17,13 +16,13 @@ import (
 const BlockSize = 128 * 1024
 
 const (
-	messageTypeIndex       = 1
-	messageTypeRequest     = 2
-	messageTypeResponse    = 3
-	messageTypePing        = 4
-	messageTypePong        = 5
-	messageTypeIndexUpdate = 6
-	messageTypeOptions     = 7
+	messageTypeClusterConfig = 0
+	messageTypeIndex         = 1
+	messageTypeRequest       = 2
+	messageTypeResponse      = 3
+	messageTypePing          = 4
+	messageTypePong          = 5
+	messageTypeIndexUpdate   = 6
 )
 
 const (
@@ -32,6 +31,12 @@ const (
 	FlagDirectory        = 1 << 14
 )
 
+const (
+	FlagShareTrusted  uint32 = 1 << 0
+	FlagShareReadOnly        = 1 << 1
+	FlagShareBits            = 0x000000ff
+)
+
 var (
 	ErrClusterHash = fmt.Errorf("configuration error: mismatched cluster hash")
 	ErrClosed      = errors.New("connection closed")
@@ -44,6 +49,8 @@ type Model interface {
 	IndexUpdate(nodeID string, repo string, files []FileInfo)
 	// A request was made by the peer node
 	Request(nodeID string, repo string, name string, offset int64, size int) ([]byte, error)
+	// A cluster configuration message was received
+	ClusterConfig(nodeID string, config ClusterConfigMessage)
 	// The peer node closed the connection
 	Close(nodeID string, err error)
 }
@@ -52,27 +59,24 @@ type Connection interface {
 	ID() string
 	Index(repo string, files []FileInfo)
 	Request(repo string, name string, offset int64, size int) ([]byte, error)
+	ClusterConfig(config ClusterConfigMessage)
 	Statistics() Statistics
-	Option(key string) string
 }
 
 type rawConnection struct {
 	sync.RWMutex
 
-	id          string
-	receiver    Model
-	reader      io.ReadCloser
-	xr          *xdr.Reader
-	writer      io.WriteCloser
-	wb          *bufio.Writer
-	xw          *xdr.Writer
-	closed      chan struct{}
-	awaiting    map[int]chan asyncResult
-	nextID      int
-	indexSent   map[string]map[string][2]int64
-	peerOptions map[string]string
-	myOptions   map[string]string
-	optionsLock sync.Mutex
+	id        string
+	receiver  Model
+	reader    io.ReadCloser
+	xr        *xdr.Reader
+	writer    io.WriteCloser
+	wb        *bufio.Writer
+	xw        *xdr.Writer
+	closed    chan struct{}
+	awaiting  map[int]chan asyncResult
+	nextID    int
+	indexSent map[string]map[string][2]int64
 
 	hasSentIndex  bool
 	hasRecvdIndex bool
@@ -88,7 +92,7 @@ const (
 	pingIdleTime = 5 * time.Minute
 )
 
-func NewConnection(nodeID string, reader io.Reader, writer io.Writer, receiver Model, options map[string]string) Connection {
+func NewConnection(nodeID string, reader io.Reader, writer io.Writer, receiver Model) Connection {
 	flrd := flate.NewReader(reader)
 	flwr, err := flate.NewWriter(writer, flate.BestSpeed)
 	if err != nil {
@@ -112,28 +116,6 @@ func NewConnection(nodeID string, reader io.Reader, writer io.Writer, receiver M
 	go c.readerLoop()
 	go c.pingerLoop()
 
-	if options != nil {
-		c.myOptions = options
-		go func() {
-			c.Lock()
-			header{0, c.nextID, messageTypeOptions}.encodeXDR(c.xw)
-			var om OptionsMessage
-			for k, v := range options {
-				om.Options = append(om.Options, Option{k, v})
-			}
-			om.encodeXDR(c.xw)
-			err := c.xw.Error()
-			if err == nil {
-				err = c.flush()
-			}
-			if err != nil {
-				log.Println("Warning: Write error during initial handshake:", err)
-			}
-			c.nextID++
-			c.Unlock()
-		}()
-	}
-
 	return wireFormatConnection{&c}
 }
 
@@ -217,6 +199,27 @@ func (c *rawConnection) Request(repo string, name string, offset int64, size int
 	return res.val, res.err
 }
 
+// ClusterConfig send the cluster configuration message to the peer and returns any error
+func (c *rawConnection) ClusterConfig(config ClusterConfigMessage) {
+	c.Lock()
+	defer c.Unlock()
+
+	if c.isClosed() {
+		return
+	}
+
+	header{0, c.nextID, messageTypeClusterConfig}.encodeXDR(c.xw)
+	c.nextID = (c.nextID + 1) & 0xfff
+
+	_, err := config.encodeXDR(c.xw)
+	if err == nil {
+		err = c.flush()
+	}
+	if err != nil {
+		c.close(err)
+	}
+}
+
 func (c *rawConnection) ping() bool {
 	c.Lock()
 	if c.isClosed() {
@@ -386,24 +389,14 @@ loop:
 				c.Unlock()
 			}
 
-		case messageTypeOptions:
-			var om OptionsMessage
-			om.decodeXDR(c.xr)
+		case messageTypeClusterConfig:
+			var cm ClusterConfigMessage
+			cm.decodeXDR(c.xr)
 			if c.xr.Error() != nil {
 				c.close(c.xr.Error())
 				break loop
-			}
-
-			c.optionsLock.Lock()
-			c.peerOptions = make(map[string]string, len(om.Options))
-			for _, opt := range om.Options {
-				c.peerOptions[opt.Key] = opt.Value
-			}
-			c.optionsLock.Unlock()
-
-			if mh, rh := c.myOptions["clusterHash"], c.peerOptions["clusterHash"]; len(mh) > 0 && len(rh) > 0 && mh != rh {
-				c.close(ErrClusterHash)
-				break loop
+			} else {
+				go c.receiver.ClusterConfig(c.id, cm)
 			}
 
 		default:
@@ -472,9 +465,3 @@ func (c *rawConnection) Statistics() Statistics {
 		OutBytesTotal: int(c.xw.Tot()),
 	}
 }
-
-func (c *rawConnection) Option(key string) string {
-	c.optionsLock.Lock()
-	defer c.optionsLock.Unlock()
-	return c.peerOptions[key]
-}

protocol/protocol_test.go

@@ -25,8 +25,8 @@ func TestPing(t *testing.T) {
 	ar, aw := io.Pipe()
 	br, bw := io.Pipe()
 
-	c0 := NewConnection("c0", ar, bw, nil, nil).(wireFormatConnection).next.(*rawConnection)
-	c1 := NewConnection("c1", br, aw, nil, nil).(wireFormatConnection).next.(*rawConnection)
+	c0 := NewConnection("c0", ar, bw, nil).(wireFormatConnection).next.(*rawConnection)
+	c1 := NewConnection("c1", br, aw, nil).(wireFormatConnection).next.(*rawConnection)
 
 	if ok := c0.ping(); !ok {
 		t.Error("c0 ping failed")
@@ -49,8 +49,8 @@ func TestPingErr(t *testing.T) {
 			eaw := &ErrPipe{PipeWriter: *aw, max: i, err: e}
 			ebw := &ErrPipe{PipeWriter: *bw, max: j, err: e}
 
-			c0 := NewConnection("c0", ar, ebw, m0, nil).(wireFormatConnection).next.(*rawConnection)
-			NewConnection("c1", br, eaw, m1, nil)
+			c0 := NewConnection("c0", ar, ebw, m0).(wireFormatConnection).next.(*rawConnection)
+			NewConnection("c1", br, eaw, m1)
 
 			res := c0.ping()
 			if (i < 4 || j < 4) && res {
@@ -125,8 +125,8 @@ func TestVersionErr(t *testing.T) {
 	ar, aw := io.Pipe()
 	br, bw := io.Pipe()
 
-	c0 := NewConnection("c0", ar, bw, m0, nil).(wireFormatConnection).next.(*rawConnection)
-	NewConnection("c1", br, aw, m1, nil)
+	c0 := NewConnection("c0", ar, bw, m0).(wireFormatConnection).next.(*rawConnection)
+	NewConnection("c1", br, aw, m1)
 
 	c0.xw.WriteUint32(encodeHeader(header{
 		version: 2,
@@ -147,8 +147,8 @@ func TestTypeErr(t *testing.T) {
 	ar, aw := io.Pipe()
 	br, bw := io.Pipe()
 
-	c0 := NewConnection("c0", ar, bw, m0, nil).(wireFormatConnection).next.(*rawConnection)
-	NewConnection("c1", br, aw, m1, nil)
+	c0 := NewConnection("c0", ar, bw, m0).(wireFormatConnection).next.(*rawConnection)
+	NewConnection("c1", br, aw, m1)
 
 	c0.xw.WriteUint32(encodeHeader(header{
 		version: 0,
@@ -169,8 +169,8 @@ func TestClose(t *testing.T) {
 	ar, aw := io.Pipe()
 	br, bw := io.Pipe()
 
-	c0 := NewConnection("c0", ar, bw, m0, nil).(wireFormatConnection).next.(*rawConnection)
-	NewConnection("c1", br, aw, m1, nil)
+	c0 := NewConnection("c0", ar, bw, m0).(wireFormatConnection).next.(*rawConnection)
+	NewConnection("c1", br, aw, m1)
 
 	c0.close(nil)
 

protocol/wireformat.go

@@ -30,10 +30,10 @@ func (c wireFormatConnection) Request(repo, name string, offset int64, size int)
 	return c.next.Request(repo, name, offset, size)
 }
 
+func (c wireFormatConnection) ClusterConfig(config ClusterConfigMessage) {
+	c.next.ClusterConfig(config)
+}
+
 func (c wireFormatConnection) Statistics() Statistics {
 	return c.next.Statistics()
 }
-
-func (c wireFormatConnection) Option(key string) string {
-	return c.next.Option(key)
-}

scanner/walk.go

@@ -16,9 +16,6 @@ import (
 type Walker struct {
 	// Dir is the base directory for the walk
 	Dir string
-	// If FollowSymlinks is true, symbolic links directly under Dir will be followed.
-	// Symbolic links at deeper levels are never followed regardless of this flag.
-	FollowSymlinks bool
 	// BlockSize controls the size of the block used when hashing.
 	BlockSize int
 	// If IgnoreFile is not empty, it is the name used for the file that holds ignore patterns.
@@ -58,7 +55,7 @@ func (w *Walker) Walk() (files []File, ignore map[string][]string) {
 	w.lazyInit()
 
 	if debug {
-		dlog.Println("Walk", w.Dir, w.FollowSymlinks, w.BlockSize, w.IgnoreFile)
+		dlog.Println("Walk", w.Dir, w.BlockSize, w.IgnoreFile)
 	}
 	t0 := time.Now()
 
@@ -68,27 +65,6 @@ func (w *Walker) Walk() (files []File, ignore map[string][]string) {
 	filepath.Walk(w.Dir, w.loadIgnoreFiles(w.Dir, ignore))
 	filepath.Walk(w.Dir, hashFiles)
 
-	if w.FollowSymlinks {
-		d, err := os.Open(w.Dir)
-		if err != nil {
-			return
-		}
-		defer d.Close()
-
-		fis, err := d.Readdir(-1)
-		if err != nil {
-			return
-		}
-
-		for _, info := range fis {
-			if info.Mode()&os.ModeSymlink != 0 {
-				dir := filepath.Join(w.Dir, info.Name()) + "/"
-				filepath.Walk(dir, w.loadIgnoreFiles(dir, ignore))
-				filepath.Walk(dir, hashFiles)
-			}
-		}
-	}
-
 	if debug {
 		t1 := time.Now()
 		d := t1.Sub(t0).Seconds()

xdr/xdr_test.go

@@ -55,3 +55,52 @@ func TestBytesGiven(t *testing.T) {
 		t.Error(err)
 	}
 }
+
+func TestReadBytesMaxInto(t *testing.T) {
+	var max = 64
+	for tot := 32; tot < 128; tot++ {
+		for diff := -32; diff <= 32; diff++ {
+			var b = new(bytes.Buffer)
+			var r = NewReader(b)
+			var w = NewWriter(b)
+
+			var toWrite = make([]byte, tot)
+			w.WriteBytes(toWrite)
+
+			var buf = make([]byte, tot+diff)
+			var bs = r.ReadBytesMaxInto(max, buf)
+
+			if tot <= max {
+				if read := len(bs); read != tot {
+					t.Errorf("Incorrect read bytes, wrote=%d, buf=%d, max=%d, read=%d", tot, tot+diff, max, read)
+				}
+			} else if r.err != ErrElementSizeExceeded {
+				t.Errorf("Unexpected non-ErrElementSizeExceeded error for wrote=%d, max=%d: %v", tot, max, r.err)
+			}
+		}
+	}
+}
+
+func TestReadBytesMaxIntoNil(t *testing.T) {
+	for tot := 42; tot < 72; tot++ {
+		for max := 0; max < 128; max++ {
+			var b = new(bytes.Buffer)
+			var r = NewReader(b)
+			var w = NewWriter(b)
+
+			var toWrite = make([]byte, tot)
+			w.WriteBytes(toWrite)
+
+			var bs = r.ReadBytesMaxInto(max, nil)
+			var read = len(bs)
+
+			if max == 0 || tot <= max {
+				if read != tot {
+					t.Errorf("Incorrect read bytes, wrote=%d, max=%d, read=%d", tot, max, read)
+				}
+			} else if r.err != ErrElementSizeExceeded {
+				t.Errorf("Unexpected non-ErrElementSizeExceeded error for wrote=%d, max=%d, read=%d: %v", tot, max, read, r.err)
+			}
+		}
+	}
+}