Fix upgrade non-support on Windows

Fix typo in header name

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 Please do contribute! If you want to contribute but are unsure where to
 start, the [Contributions Needed
-page](https://github.com/calmh/syncthing/wiki/Contributions-Needed)
+topic](http://discourse.syncthing.net/t/contributions-needed/49)
 lists areas in need of attention.
 
 ## Licensing
@@ -15,7 +15,8 @@ will ensure that you are added to the CONTRIBUTORS file.
 
 ## Building
 
-[See the wiki](https://github.com/calmh/syncthing/wiki/Building)
+[See the
+documentation](http://discourse.syncthing.net/t/building-syncthing/44)
 
 ## Branches
 
@@ -46,7 +47,7 @@ Yes please!
 
 ## Documentation
 
-[Hack it here](https://github.com/calmh/syncthing/wiki)
+[Over here!](http://discourse.syncthing.net/category/documentation)
 
 ## License
 

CONTRIBUTORS

@@ -1,4 +1,5 @@
 Aaron Bieber <qbit@deftly.net>
+Andrew Dunham <andrew@du.nham.ca>
 Brandon Philips <brandon@ifup.org>
 James Patterson <jamespatterson@operamail.com>
 Philippe Schommers <philippe@schommers.be>

Godeps/Godeps.json

@@ -8,6 +8,11 @@
 		"./discover/cmd/discosrv"
 	],
 	"Deps": [
+		{
+			"ImportPath": "bitbucket.org/kardianos/osext",
+			"Comment": "null-9",
+			"Rev": "364fb577de68fb646c4cb39cc0e09c887ee16376"
+		},
 		{
 			"ImportPath": "code.google.com/p/go.crypto/bcrypt",
 			"Comment": "null-185",

Godeps/_workspace/src/bitbucket.org/kardianos/osext/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Daniel Theophanes
+
+This software is provided 'as-is', without any express or implied
+warranty. In no event will the authors be held liable for any damages
+arising from the use of this software.
+
+Permission is granted to anyone to use this software for any purpose,
+including commercial applications, and to alter it and redistribute it
+freely, subject to the following restrictions:
+
+   1. The origin of this software must not be misrepresented; you must not
+   claim that you wrote the original software. If you use this software
+   in a product, an acknowledgment in the product documentation would be
+   appreciated but is not required.
+
+   2. Altered source versions must be plainly marked as such, and must not be
+   misrepresented as being the original software.
+
+   3. This notice may not be removed or altered from any source
+   distribution.

Godeps/_workspace/src/bitbucket.org/kardianos/osext/osext.go

@@ -0,0 +1,32 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Extensions to the standard "os" package.
+package osext
+
+import "path/filepath"
+
+// Executable returns an absolute path that can be used to
+// re-invoke the current program.
+// It may not be valid after the current program exits.
+func Executable() (string, error) {
+	p, err := executable()
+	return filepath.Clean(p), err
+}
+
+// Returns same path as Executable, returns just the folder
+// path. Excludes the executable name.
+func ExecutableFolder() (string, error) {
+	p, err := Executable()
+	if err != nil {
+		return "", err
+	}
+	folder, _ := filepath.Split(p)
+	return folder, nil
+}
+
+// Depricated. Same as Executable().
+func GetExePath() (exePath string, err error) {
+	return Executable()
+}

Godeps/_workspace/src/bitbucket.org/kardianos/osext/osext_plan9.go

@@ -0,0 +1,16 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package osext
+
+import "syscall"
+
+func executable() (string, error) {
+	f, err := Open("/proc/" + itoa(Getpid()) + "/text")
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	return syscall.Fd2path(int(f.Fd()))
+}

Godeps/_workspace/src/bitbucket.org/kardianos/osext/osext_procfs.go

@@ -0,0 +1,25 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux netbsd openbsd
+
+package osext
+
+import (
+	"errors"
+	"os"
+	"runtime"
+)
+
+func executable() (string, error) {
+	switch runtime.GOOS {
+	case "linux":
+		return os.Readlink("/proc/self/exe")
+	case "netbsd":
+		return os.Readlink("/proc/curproc/exe")
+	case "openbsd":
+		return os.Readlink("/proc/curproc/file")
+	}
+	return "", errors.New("ExecPath not implemented for " + runtime.GOOS)
+}

Godeps/_workspace/src/bitbucket.org/kardianos/osext/osext_sysctl.go

@@ -0,0 +1,82 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin freebsd
+
+package osext
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"syscall"
+	"unsafe"
+)
+
+var startUpcwd, getwdError = os.Getwd()
+
+func executable() (string, error) {
+	var mib [4]int32
+	switch runtime.GOOS {
+	case "freebsd":
+		mib = [4]int32{1 /* CTL_KERN */, 14 /* KERN_PROC */, 12 /* KERN_PROC_PATHNAME */, -1}
+	case "darwin":
+		mib = [4]int32{1 /* CTL_KERN */, 38 /* KERN_PROCARGS */, int32(os.Getpid()), -1}
+	}
+
+	n := uintptr(0)
+	// get length
+	_, _, err := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, 0, uintptr(unsafe.Pointer(&n)), 0, 0)
+	if err != 0 {
+		return "", err
+	}
+	if n == 0 { // shouldn't happen
+		return "", nil
+	}
+	buf := make([]byte, n)
+	_, _, err = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, uintptr(unsafe.Pointer(&buf[0])), uintptr(unsafe.Pointer(&n)), 0, 0)
+	if err != 0 {
+		return "", err
+	}
+	if n == 0 { // shouldn't happen
+		return "", nil
+	}
+	for i, v := range buf {
+		if v == 0 {
+			buf = buf[:i]
+			break
+		}
+	}
+	var strpath string
+	if buf[0] != '/' {
+		var e error
+		if strpath, e = getAbs(buf); e != nil {
+			return strpath, e
+		}
+	} else {
+		strpath = string(buf)
+	}
+	// darwin KERN_PROCARGS may return the path to a symlink rather than the
+	// actual executable
+	if runtime.GOOS == "darwin" {
+		if strpath, err := filepath.EvalSymlinks(strpath); err != nil {
+			return strpath, err
+		}
+	}
+	return strpath, nil
+}
+
+func getAbs(buf []byte) (string, error) {
+	if getwdError != nil {
+		return string(buf), getwdError
+	} else {
+		if buf[0] == '.' {
+			buf = buf[1:]
+		}
+		if startUpcwd[len(startUpcwd)-1] != '/' && buf[0] != '/' {
+			return startUpcwd + "/" + string(buf), nil
+		}
+		return startUpcwd + string(buf), nil
+	}
+}

Godeps/_workspace/src/bitbucket.org/kardianos/osext/osext_test.go

@@ -0,0 +1,79 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build darwin linux freebsd netbsd windows
+
+package osext
+
+import (
+	"fmt"
+	"os"
+	oexec "os/exec"
+	"path/filepath"
+	"runtime"
+	"testing"
+)
+
+const execPath_EnvVar = "OSTEST_OUTPUT_EXECPATH"
+
+func TestExecPath(t *testing.T) {
+	ep, err := Executable()
+	if err != nil {
+		t.Fatalf("ExecPath failed: %v", err)
+	}
+	// we want fn to be of the form "dir/prog"
+	dir := filepath.Dir(filepath.Dir(ep))
+	fn, err := filepath.Rel(dir, ep)
+	if err != nil {
+		t.Fatalf("filepath.Rel: %v", err)
+	}
+	cmd := &oexec.Cmd{}
+	// make child start with a relative program path
+	cmd.Dir = dir
+	cmd.Path = fn
+	// forge argv[0] for child, so that we can verify we could correctly
+	// get real path of the executable without influenced by argv[0].
+	cmd.Args = []string{"-", "-test.run=XXXX"}
+	cmd.Env = []string{fmt.Sprintf("%s=1", execPath_EnvVar)}
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		t.Fatalf("exec(self) failed: %v", err)
+	}
+	outs := string(out)
+	if !filepath.IsAbs(outs) {
+		t.Fatalf("Child returned %q, want an absolute path", out)
+	}
+	if !sameFile(outs, ep) {
+		t.Fatalf("Child returned %q, not the same file as %q", out, ep)
+	}
+}
+
+func sameFile(fn1, fn2 string) bool {
+	fi1, err := os.Stat(fn1)
+	if err != nil {
+		return false
+	}
+	fi2, err := os.Stat(fn2)
+	if err != nil {
+		return false
+	}
+	return os.SameFile(fi1, fi2)
+}
+
+func init() {
+	if e := os.Getenv(execPath_EnvVar); e != "" {
+		// first chdir to another path
+		dir := "/"
+		if runtime.GOOS == "windows" {
+			dir = filepath.VolumeName(".")
+		}
+		os.Chdir(dir)
+		if ep, err := Executable(); err != nil {
+			fmt.Fprint(os.Stderr, "ERROR: ", err)
+		} else {
+			fmt.Fprint(os.Stderr, ep)
+		}
+		os.Exit(0)
+	}
+}

Godeps/_workspace/src/bitbucket.org/kardianos/osext/osext_windows.go

@@ -0,0 +1,34 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package osext
+
+import (
+	"syscall"
+	"unicode/utf16"
+	"unsafe"
+)
+
+var (
+	kernel                = syscall.MustLoadDLL("kernel32.dll")
+	getModuleFileNameProc = kernel.MustFindProc("GetModuleFileNameW")
+)
+
+// GetModuleFileName() with hModule = NULL
+func executable() (exePath string, err error) {
+	return getModuleFileName()
+}
+
+func getModuleFileName() (string, error) {
+	var n uint32
+	b := make([]uint16, syscall.MAX_PATH)
+	size := uint32(len(b))
+
+	r0, _, e1 := getModuleFileNameProc.Call(0, uintptr(unsafe.Pointer(&b[0])), uintptr(size))
+	n = uint32(r0)
+	if n == 0 {
+		return "", e1
+	}
+	return string(utf16.Decode(b[0:n])), nil
+}

README.md

@@ -1,4 +1,4 @@
-syncthing [![Build Status](https://drone.io/github.com/calmh/syncthing/status.png)](https://drone.io/github.com/calmh/syncthing/latest)
+syncthing
 =========
 
 This is the `syncthing` project. The following are the project goals:
@@ -25,6 +25,11 @@ making sure large swarms of selfish agents behave and somehow work
 towards a common goal. Here we have a much smaller swarm of cooperative
 agents and a simpler approach will suffice.
 
+Getting Started
+---------------
+
+Take a look at the [getting started guide](http://discourse.syncthing.net/t/getting-started/46).
+
 Signed Releases
 ---------------
 
@@ -35,8 +40,9 @@ normal release bundle as `syncthing.asc` or `syncthing.exe.asc`.
 Documentation
 =============
 
-The syncthing documentation is kept on the
-[GitHub Wiki](https://github.com/calmh/syncthing/wiki).
+The [syncthing
+documentation](http://discourse.syncthing.net/category/documentation) is
+on the discourse site.
 
 License
 =======

cmd/syncthing/blockqueue.go

@@ -1,6 +1,10 @@
 package main
 
-import "github.com/calmh/syncthing/scanner"
+import (
+	"sync/atomic"
+
+	"github.com/calmh/syncthing/scanner"
+)
 
 type bqAdd struct {
 	file scanner.File
@@ -20,6 +24,7 @@ type blockQueue struct {
 	outbox chan bqBlock
 
 	queued []bqBlock
+	qlen   uint32
 }
 
 func newBlockQueue() *blockQueue {
@@ -77,6 +82,7 @@ func (q *blockQueue) run() {
 				q.queued = q.queued[1:]
 			}
 		}
+		atomic.StoreUint32(&q.qlen, uint32(len(q.queued)))
 	}
 }
 
@@ -89,6 +95,7 @@ func (q *blockQueue) get() bqBlock {
 }
 
 func (q *blockQueue) empty() bool {
-	// There is a race condition here. We're only mostly sure the queue is empty if the expression below is true.
-	return len(q.queued) == 0 && len(q.inbox) == 0 && len(q.outbox) == 0
+	var l uint32
+	atomic.LoadUint32(&l)
+	return l == 0
 }

cmd/syncthing/config.go

@@ -25,6 +25,7 @@ type RepositoryConfiguration struct {
 	Directory string              `xml:"directory,attr"`
 	Nodes     []NodeConfiguration `xml:"node"`
 	ReadOnly  bool                `xml:"ro,attr"`
+	Invalid   string              `xml:"-"` // Set at runtime when there is an error, not saved
 	nodeIDs   []string
 }
 
@@ -171,17 +172,21 @@ func readConfigXML(rd io.Reader, myID string) (Configuration, error) {
 	cfg.Options.ListenAddress = uniqueStrings(cfg.Options.ListenAddress)
 
 	// Check for missing or duplicate repository ID:s
-	var seenRepos = map[string]bool{}
+	var seenRepos = map[string]*RepositoryConfiguration{}
 	for i := range cfg.Repositories {
-		if cfg.Repositories[i].ID == "" {
-			cfg.Repositories[i].ID = "default"
+		repo := &cfg.Repositories[i]
+
+		if repo.ID == "" {
+			repo.ID = "default"
 		}
 
-		id := cfg.Repositories[i].ID
-		if seenRepos[id] {
-			panic("duplicate repository ID " + id)
+		if seen, ok := seenRepos[repo.ID]; ok {
+			seen.Invalid = "duplicate repository ID"
+			repo.Invalid = "duplicate repository ID"
+			warnf("Multiple repositories with ID %q; disabling", repo.ID)
+		} else {
+			seenRepos[repo.ID] = repo
 		}
-		seenRepos[id] = true
 	}
 
 	// Upgrade to v2 configuration if appropriate

cmd/syncthing/gui.go

@@ -5,7 +5,9 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"io/ioutil"
+	"log"
 	"math/rand"
+	"net"
 	"net/http"
 	"runtime"
 	"sync"
@@ -25,13 +27,20 @@ var (
 	configInSync = true
 	guiErrors    = []guiError{}
 	guiErrorsMut sync.Mutex
+	static       = embeddedStatic()
+	staticFunc   = static.(func(http.ResponseWriter, *http.Request, *log.Logger))
 )
 
 const (
 	unchangedPassword = "--password-unchanged--"
 )
 
-func startGUI(cfg GUIConfiguration, m *Model) {
+func startGUI(cfg GUIConfiguration, m *Model) error {
+	l, err := net.Listen("tcp", cfg.Address)
+	if err != nil {
+		return err
+	}
+
 	router := martini.NewRouter()
 	router.Get("/", getRoot)
 	router.Get("/rest/version", restGetVersion)
@@ -48,25 +57,24 @@ func startGUI(cfg GUIConfiguration, m *Model) {
 	router.Post("/rest/error", restPostError)
 	router.Post("/rest/error/clear", restClearErrors)
 
-	go func() {
-		mr := martini.New()
-		if len(cfg.User) > 0 && len(cfg.Password) > 0 {
-			mr.Use(basic(cfg.User, cfg.Password))
-		}
-		mr.Use(embeddedStatic())
-		mr.Use(martini.Recovery())
-		mr.Use(restMiddleware)
-		mr.Action(router.Handle)
-		mr.Map(m)
-		err := http.ListenAndServe(cfg.Address, mr)
-		if err != nil {
-			warnln("GUI not possible:", err)
-		}
-	}()
+	mr := martini.New()
+	if len(cfg.User) > 0 && len(cfg.Password) > 0 {
+		mr.Use(basic(cfg.User, cfg.Password))
+	}
+	mr.Use(static)
+	mr.Use(martini.Recovery())
+	mr.Use(restMiddleware)
+	mr.Action(router.Handle)
+	mr.Map(m)
+
+	go http.Serve(l, mr)
+
+	return nil
 }
 
 func getRoot(w http.ResponseWriter, r *http.Request) {
-	http.Redirect(w, r, "/index.html", 302)
+	r.URL.Path = "/index.html"
+	staticFunc(w, r, nil)
 }
 
 func restMiddleware(w http.ResponseWriter, r *http.Request) {
@@ -84,6 +92,13 @@ func restGetModel(m *Model, w http.ResponseWriter, r *http.Request) {
 	var repo = qs.Get("repo")
 	var res = make(map[string]interface{})
 
+	for _, cr := range cfg.Repositories {
+		if cr.ID == repo {
+			res["invalid"] = cr.Invalid
+			break
+		}
+	}
+
 	globalFiles, globalDeleted, globalBytes := m.GlobalSize(repo)
 	res["globalFiles"], res["globalDeleted"], res["globalBytes"] = globalFiles, globalDeleted, globalBytes
 
@@ -180,7 +195,7 @@ func restGetSystem(w http.ResponseWriter) {
 	res["goroutines"] = runtime.NumGoroutine()
 	res["alloc"] = m.Alloc
 	res["sys"] = m.Sys
-	if discoverer != nil {
+	if cfg.Options.GlobalAnnEnabled && discoverer != nil {
 		res["extAnnounceOK"] = discoverer.ExtAnnounceOK()
 	}
 	cpuUsageLock.RLock()

cmd/syncthing/gui_embedded.go

@@ -32,7 +32,7 @@ func embeddedStatic() interface{} {
 		if len(mtype) != 0 {
 			res.Header().Set("Content-Type", mtype)
 		}
-		res.Header().Set("Content-Size", fmt.Sprintf("%d", len(bs)))
+		res.Header().Set("Content-Length", fmt.Sprintf("%d", len(bs)))
 		res.Header().Set("Last-Modified", modt)
 
 		res.Write(bs)

cmd/syncthing/main.go

@@ -83,9 +83,11 @@ const (
 func main() {
 	var reset bool
 	var showVersion bool
+	var doUpgrade bool
 	flag.StringVar(&confDir, "home", getDefaultConfDir(), "Set configuration directory")
 	flag.BoolVar(&reset, "reset", false, "Prepare to resync from cluster")
 	flag.BoolVar(&showVersion, "version", false, "Show version")
+	flag.BoolVar(&doUpgrade, "upgrade", false, "Perform upgrade")
 	flag.Usage = usageFor(flag.CommandLine, usage, extraUsage)
 	flag.Parse()
 
@@ -99,6 +101,14 @@ func main() {
 		return
 	}
 
+	if doUpgrade {
+		err := upgrade()
+		if err != nil {
+			fatalln(err)
+		}
+		return
+	}
+
 	if len(os.Getenv("GOGC")) == 0 {
 		debug.SetGCPercent(25)
 	}
@@ -222,6 +232,9 @@ func main() {
 	m := NewModel(cfg.Options.MaxChangeKbps * 1000)
 
 	for _, repo := range cfg.Repositories {
+		if repo.Invalid != "" {
+			continue
+		}
 		dir := expandTilde(repo.Directory)
 		m.AddRepo(repo.ID, dir, repo.Nodes)
 	}
@@ -230,7 +243,7 @@ func main() {
 	if cfg.GUI.Enabled && cfg.GUI.Address != "" {
 		addr, err := net.ResolveTCPAddr("tcp", cfg.GUI.Address)
 		if err != nil {
-			warnf("Cannot start GUI on %q: %v", cfg.GUI.Address, err)
+			fatalf("Cannot start GUI on %q: %v", cfg.GUI.Address, err)
 		} else {
 			var hostOpen, hostShow string
 			switch {
@@ -246,7 +259,10 @@ func main() {
 			}
 
 			infof("Starting web GUI on http://%s:%d/", hostShow, addr.Port)
-			startGUI(cfg.GUI, m)
+			err := startGUI(cfg.GUI, m)
+			if err != nil {
+				fatalln("Cannot start GUI:", err)
+			}
 			if cfg.Options.StartBrowser && len(os.Getenv("STRESTART")) == 0 {
 				openURL(fmt.Sprintf("http://%s:%d", hostOpen, addr.Port))
 			}
@@ -260,6 +276,10 @@ func main() {
 	m.LoadIndexes(confDir)
 
 	for _, repo := range cfg.Repositories {
+		if repo.Invalid != "" {
+			continue
+		}
+
 		dir := expandTilde(repo.Directory)
 
 		// Safety check. If the cached index contains files but the repository
@@ -295,6 +315,10 @@ func main() {
 	go listenConnect(myID, m, tlsCfg)
 
 	for _, repo := range cfg.Repositories {
+		if repo.Invalid != "" {
+			continue
+		}
+
 		// Routine to pull blocks from other nodes to synchronize the local
 		// repository. Does not run when we are in read only (publish only) mode.
 		if repo.ReadOnly {

cmd/syncthing/model.go

@@ -80,8 +80,8 @@ func NewModel(maxChangeBw int) *Model {
 // read/write mode the model will attempt to keep in sync with the cluster by
 // pulling needed files from peer nodes.
 func (m *Model) StartRepoRW(repo string, threads int) {
-	m.rmut.Lock()
-	defer m.rmut.Unlock()
+	m.rmut.RLock()
+	defer m.rmut.RUnlock()
 
 	if dir, ok := m.repoDirs[repo]; !ok {
 		panic("cannot start without repo")
@@ -423,14 +423,16 @@ func (m *Model) AddConnection(rawConn io.Closer, protoConn protocol.Connection)
 	cm := m.clusterConfig(nodeID)
 	protoConn.ClusterConfig(cm)
 
+	var idxToSend = make(map[string][]protocol.FileInfo)
+
+	m.rmut.RLock()
+	for _, repo := range m.nodeRepos[nodeID] {
+		idxToSend[repo] = m.protocolIndex(repo)
+	}
+	m.rmut.RUnlock()
+
 	go func() {
-		m.rmut.RLock()
-		repos := m.nodeRepos[nodeID]
-		m.rmut.RUnlock()
-		for _, repo := range repos {
-			m.rmut.RLock()
-			idx := m.protocolIndex(repo)
-			m.rmut.RUnlock()
+		for repo, idx := range idxToSend {
 			if debugNet {
 				dlog.Printf("IDX(out/initial): %s: %q: %d files", nodeID, repo, len(idx))
 			}
@@ -559,7 +561,7 @@ func (m *Model) ScanRepos() {
 
 func (m *Model) ScanRepo(repo string) {
 	sup := &suppressor{threshold: int64(cfg.Options.MaxChangeKbps)}
-	m.rmut.Lock()
+	m.rmut.RLock()
 	w := &scanner.Walker{
 		Dir:          m.repoDirs[repo],
 		IgnoreFile:   ".stignore",
@@ -568,7 +570,7 @@ func (m *Model) ScanRepo(repo string) {
 		Suppressor:   sup,
 		CurrentFiler: cFiler{m, repo},
 	}
-	m.rmut.Unlock()
+	m.rmut.RUnlock()
 	m.setState(repo, RepoScanning)
 	fs, _ := w.Walk()
 	m.ReplaceLocal(repo, fs)
@@ -648,7 +650,7 @@ func (m *Model) clusterConfig(node string) protocol.ClusterConfigMessage {
 		ClientVersion: Version,
 	}
 
-	m.rmut.Lock()
+	m.rmut.RLock()
 	for _, repo := range m.nodeRepos[node] {
 		cr := protocol.Repository{
 			ID: repo,
@@ -662,7 +664,7 @@ func (m *Model) clusterConfig(node string) protocol.ClusterConfigMessage {
 		}
 		cm.Repositories = append(cm.Repositories, cr)
 	}
-	m.rmut.Unlock()
+	m.rmut.RUnlock()
 
 	return cm
 }
@@ -674,9 +676,9 @@ func (m *Model) setState(repo string, state repoState) {
 }
 
 func (m *Model) State(repo string) string {
-	m.rmut.Lock()
+	m.rmut.RLock()
 	state := m.repoState[repo]
-	m.rmut.Unlock()
+	m.rmut.RUnlock()
 	switch state {
 	case RepoIdle:
 		return "idle"

cmd/syncthing/openurl.go

@@ -1,34 +0,0 @@
-/*
-Copyright 2011 Google Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package main
-
-import (
-	"os/exec"
-	"runtime"
-)
-
-func openURL(url string) error {
-	if runtime.GOOS == "windows" {
-		return exec.Command("cmd.exe", "/C", "start "+url).Run()
-	}
-
-	if runtime.GOOS == "darwin" {
-		return exec.Command("open", url).Run()
-	}
-
-	return exec.Command("xdg-open", url).Run()
-}

cmd/syncthing/openurl_unix.go

@@ -0,0 +1,23 @@
+// +build !windows
+
+package main
+
+import (
+	"os/exec"
+	"runtime"
+	"syscall"
+)
+
+func openURL(url string) error {
+	switch runtime.GOOS {
+	case "darwin":
+		return exec.Command("open", url).Run()
+
+	default:
+		cmd := exec.Command("xdg-open", url)
+		cmd.SysProcAttr = &syscall.SysProcAttr{
+			Setpgid: true,
+		}
+		return cmd.Run()
+	}
+}

cmd/syncthing/openurl_windows.go

@@ -0,0 +1,9 @@
+// +build windows
+
+package main
+
+import "os/exec"
+
+func openURL(url string) error {
+	return exec.Command("cmd.exe", "/C", "start "+url).Run()
+}

cmd/syncthing/puller.go

@@ -135,7 +135,10 @@ func (p *puller) run() {
 			case b := <-p.blocks:
 				p.model.setState(p.repo, RepoSyncing)
 				changed = true
-				p.handleBlock(b)
+				if p.handleBlock(b) {
+					// Block was fully handled, free up the slot
+					p.requestSlots <- true
+				}
 
 			case <-timeout:
 				if len(p.openFiles) == 0 && p.bq.empty() {
@@ -193,7 +196,7 @@ func (p *puller) runRO() {
 
 func (p *puller) fixupDirectories() {
 	var deleteDirs []string
-	fn := func(path string, info os.FileInfo, err error) error {
+	filepath.Walk(p.dir, func(path string, info os.FileInfo, err error) error {
 		if !info.IsDir() {
 			return nil
 		}
@@ -242,8 +245,7 @@ func (p *puller) fixupDirectories() {
 		}
 
 		return nil
-	}
-	filepath.Walk(p.dir, fn)
+	})
 
 	// Delete any queued directories
 	for i := len(deleteDirs) - 1; i >= 0; i-- {
@@ -278,54 +280,14 @@ func (p *puller) handleRequestResult(res requestResult) {
 	}
 
 	if of.done && of.outstanding == 0 {
-		if debugPull {
-			dlog.Printf("pull: closing %q / %q", p.repo, f.Name)
-		}
-		of.file.Close()
-		defer os.Remove(of.temp)
-
-		delete(p.openFiles, f.Name)
-
-		fd, err := os.Open(of.temp)
-		if err != nil {
-			if debugPull {
-				dlog.Printf("pull: error: %q / %q: %v", p.repo, f.Name, err)
-			}
-			return
-		}
-		hb, _ := scanner.Blocks(fd, BlockSize)
-		fd.Close()
-
-		if l0, l1 := len(hb), len(f.Blocks); l0 != l1 {
-			if debugPull {
-				dlog.Printf("pull: %q / %q: nblocks %d != %d", p.repo, f.Name, l0, l1)
-			}
-			return
-		}
-
-		for i := range hb {
-			if bytes.Compare(hb[i].Hash, f.Blocks[i].Hash) != 0 {
-				dlog.Printf("pull: %q / %q: block %d hash mismatch", p.repo, f.Name, i)
-				return
-			}
-		}
-
-		t := time.Unix(f.Modified, 0)
-		os.Chtimes(of.temp, t, t)
-		os.Chmod(of.temp, os.FileMode(f.Flags&0777))
-		defTempNamer.Show(of.temp)
-		if debugPull {
-			dlog.Printf("pull: rename %q / %q: %q", p.repo, f.Name, of.filepath)
-		}
-		if err := Rename(of.temp, of.filepath); err == nil {
-			p.model.updateLocal(p.repo, f)
-		} else {
-			dlog.Printf("pull: error: %q / %q: %v", p.repo, f.Name, err)
-		}
+		p.closeFile(f)
 	}
 }
 
-func (p *puller) handleBlock(b bqBlock) {
+// handleBlock fulfills the block request by copying, ignoring or fetching
+// from the network. Returns true if the block was fully handled
+// synchronously, i.e. if the slot can be reused.
+func (p *puller) handleBlock(b bqBlock) bool {
 	f := b.file
 
 	// For directories, simply making sure they exist is enough
@@ -336,8 +298,7 @@ func (p *puller) handleBlock(b bqBlock) {
 			os.MkdirAll(path, 0777)
 		}
 		p.model.updateLocal(p.repo, f)
-		p.requestSlots <- true
-		return
+		return true
 	}
 
 	of, ok := p.openFiles[f.Name]
@@ -369,8 +330,7 @@ func (p *puller) handleBlock(b bqBlock) {
 			if !b.last {
 				p.openFiles[f.Name] = of
 			}
-			p.requestSlots <- true
-			return
+			return true
 		}
 		defTempNamer.Hide(of.temp)
 	}
@@ -385,8 +345,7 @@ func (p *puller) handleBlock(b bqBlock) {
 			delete(p.openFiles, f.Name)
 		}
 
-		p.requestSlots <- true
-		return
+		return true
 	}
 
 	p.openFiles[f.Name] = of
@@ -394,15 +353,14 @@ func (p *puller) handleBlock(b bqBlock) {
 	switch {
 	case len(b.copy) > 0:
 		p.handleCopyBlock(b)
-		p.requestSlots <- true
+		return true
 
 	case b.block.Size > 0:
-		p.handleRequestBlock(b)
-		// Request slot gets freed in <-p.blocks case
+		return p.handleRequestBlock(b)
 
 	default:
 		p.handleEmptyBlock(b)
-		p.requestSlots <- true
+		return true
 	}
 }
 
@@ -450,11 +408,15 @@ func (p *puller) handleCopyBlock(b bqBlock) {
 	}
 }
 
-func (p *puller) handleRequestBlock(b bqBlock) {
-	// We have a block to get from the network
-
+// handleRequestBlock tries to pull a block from the network. Returns true if
+// the block could _not_ be fetched (i.e. it was fully handled, matching the
+// return criteria of handleBlock)
+func (p *puller) handleRequestBlock(b bqBlock) bool {
 	f := b.file
-	of := p.openFiles[f.Name]
+	of, ok := p.openFiles[f.Name]
+	if !ok {
+		panic("bug: request for non-open file")
+	}
 
 	node := p.oustandingPerNode.leastBusyNode(of.availability, p.model.cm)
 	if len(node) == 0 {
@@ -469,8 +431,7 @@ func (p *puller) handleRequestBlock(b bqBlock) {
 		} else {
 			p.openFiles[f.Name] = of
 		}
-		p.requestSlots <- true
-		return
+		return true
 	}
 
 	of.outstanding++
@@ -491,6 +452,8 @@ func (p *puller) handleRequestBlock(b bqBlock) {
 			err:      err,
 		}
 	}(node, b)
+
+	return false
 }
 
 func (p *puller) handleEmptyBlock(b bqBlock) {
@@ -542,3 +505,52 @@ func (p *puller) queueNeededBlocks() {
 		dlog.Printf("%q: queued %d blocks", p.repo, queued)
 	}
 }
+
+func (p *puller) closeFile(f scanner.File) {
+	if debugPull {
+		dlog.Printf("pull: closing %q / %q", p.repo, f.Name)
+	}
+
+	of := p.openFiles[f.Name]
+	of.file.Close()
+	defer os.Remove(of.temp)
+
+	delete(p.openFiles, f.Name)
+
+	fd, err := os.Open(of.temp)
+	if err != nil {
+		if debugPull {
+			dlog.Printf("pull: error: %q / %q: %v", p.repo, f.Name, err)
+		}
+		return
+	}
+	hb, _ := scanner.Blocks(fd, BlockSize)
+	fd.Close()
+
+	if l0, l1 := len(hb), len(f.Blocks); l0 != l1 {
+		if debugPull {
+			dlog.Printf("pull: %q / %q: nblocks %d != %d", p.repo, f.Name, l0, l1)
+		}
+		return
+	}
+
+	for i := range hb {
+		if bytes.Compare(hb[i].Hash, f.Blocks[i].Hash) != 0 {
+			dlog.Printf("pull: %q / %q: block %d hash mismatch", p.repo, f.Name, i)
+			return
+		}
+	}
+
+	t := time.Unix(f.Modified, 0)
+	os.Chtimes(of.temp, t, t)
+	os.Chmod(of.temp, os.FileMode(f.Flags&0777))
+	defTempNamer.Show(of.temp)
+	if debugPull {
+		dlog.Printf("pull: rename %q / %q: %q", p.repo, f.Name, of.filepath)
+	}
+	if err := Rename(of.temp, of.filepath); err == nil {
+		p.model.updateLocal(p.repo, f)
+	} else {
+		dlog.Printf("pull: error: %q / %q: %v", p.repo, f.Name, err)
+	}
+}

cmd/syncthing/upgrade_unix.go

@@ -0,0 +1,146 @@
+// +build !windows
+
+package main
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"bitbucket.org/kardianos/osext"
+)
+
+type githubRelease struct {
+	Tag      string        `json:"tag_name"`
+	Prelease bool          `json:"prerelease"`
+	Assets   []githubAsset `json:"assets"`
+}
+
+type githubAsset struct {
+	URL  string `json:"url"`
+	Name string `json:"name"`
+}
+
+func upgrade() error {
+	path, err := osext.Executable()
+	if err != nil {
+		return err
+	}
+
+	resp, err := http.Get("https://api.github.com/repos/calmh/syncthing/releases?per_page=1")
+	if err != nil {
+		return err
+	}
+
+	var rels []githubRelease
+	json.NewDecoder(resp.Body).Decode(&rels)
+	resp.Body.Close()
+
+	if len(rels) != 1 {
+		return fmt.Errorf("Unexpected number of releases: %d", len(rels))
+	}
+	rel := rels[0]
+
+	if rel.Tag > Version {
+		infof("Attempting upgrade to %s...", rel.Tag)
+	} else if rel.Tag == Version {
+		okf("Already running the latest version, %s. Not upgrading.", Version)
+		return nil
+	} else {
+		okf("Current version %s is newer than latest release %s. Not upgrading.", Version, rel.Tag)
+		return nil
+	}
+
+	expectedRelease := fmt.Sprintf("syncthing-%s-%s-%s.", runtime.GOOS, runtime.GOARCH, rel.Tag)
+	for _, asset := range rel.Assets {
+		if strings.HasPrefix(asset.Name, expectedRelease) {
+			if strings.HasSuffix(asset.Name, ".tar.gz") {
+				infof("Downloading %s...", asset.Name)
+				fname, err := readTarGZ(asset.URL, filepath.Dir(path))
+				if err != nil {
+					return err
+				}
+
+				old := path + "." + Version
+				err = os.Rename(path, old)
+				if err != nil {
+					return err
+				}
+				err = os.Rename(fname, path)
+				if err != nil {
+					return err
+				}
+
+				okf("Upgraded %q to %s.", path, rel.Tag)
+				okf("Previous version saved in %q.", old)
+
+				return nil
+			}
+		}
+	}
+
+	return nil
+}
+
+func readTarGZ(url string, dir string) (string, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return "", err
+	}
+
+	req.Header.Add("Accept", "application/octet-stream")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	gr, err := gzip.NewReader(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	tr := tar.NewReader(gr)
+	if err != nil {
+		return "", err
+	}
+
+	// Iterate through the files in the archive.
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			// end of tar archive
+			break
+		}
+		if err != nil {
+			return "", err
+		}
+
+		if path.Base(hdr.Name) == "syncthing" {
+			of, err := ioutil.TempFile(dir, "syncthing")
+			if err != nil {
+				return "", err
+			}
+			io.Copy(of, tr)
+			err = of.Close()
+			if err != nil {
+				os.Remove(of.Name())
+				return "", err
+			}
+
+			os.Chmod(of.Name(), os.FileMode(hdr.Mode))
+			return of.Name(), nil
+		}
+	}
+
+	return "", fmt.Errorf("No upgrade found")
+}

cmd/syncthing/upgrade_windows.go

@@ -0,0 +1,9 @@
+// +build windows
+
+package main
+
+import "errors"
+
+func upgrade() error {
+	return errors.New("Upgrade currently unsupported on Windows")
+}

discover/discover.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"log"
 	"net"
-	"strings"
 	"sync"
 	"time"
 
@@ -76,6 +75,20 @@ func (d *Discoverer) ExtAnnounceOK() bool {
 	return d.extAnnounceOK
 }
 
+func (d *Discoverer) Lookup(node string) []string {
+	d.registryLock.Lock()
+	addr, ok := d.registry[node]
+	d.registryLock.Unlock()
+
+	if ok {
+		return addr
+	} else if len(d.extServer) != 0 {
+		// We might want to cache this, but not permanently so it needs some intelligence
+		return d.externalLookup(node)
+	}
+	return nil
+}
+
 func (d *Discoverer) announcementPkt() []byte {
 	var addrs []Address
 	for _, astr := range d.listenAddrs {
@@ -203,7 +216,7 @@ func (d *Discoverer) recvAnnouncements() {
 			for _, a := range pkt.Addresses {
 				var nodeAddr string
 				if len(a.IP) > 0 {
-					nodeAddr = fmt.Sprintf("%s:%d", ipStr(a.IP), a.Port)
+					nodeAddr = fmt.Sprintf("%s:%d", net.IP(a.IP), a.Port)
 				} else {
 					ua := addr.(*net.UDPAddr)
 					ua.Port = int(a.Port)
@@ -285,39 +298,8 @@ func (d *Discoverer) externalLookup(node string) []string {
 
 	var addrs []string
 	for _, a := range pkt.Addresses {
-		var nodeAddr string
-		if len(a.IP) > 0 {
-			nodeAddr = fmt.Sprintf("%s:%d", ipStr(a.IP), a.Port)
-		}
+		nodeAddr := fmt.Sprintf("%s:%d", net.IP(a.IP), a.Port)
 		addrs = append(addrs, nodeAddr)
 	}
 	return addrs
 }
-
-func (d *Discoverer) Lookup(node string) []string {
-	d.registryLock.Lock()
-	addr, ok := d.registry[node]
-	d.registryLock.Unlock()
-
-	if ok {
-		return addr
-	} else if len(d.extServer) != 0 {
-		// We might want to cache this, but not permanently so it needs some intelligence
-		return d.externalLookup(node)
-	}
-	return nil
-}
-
-func ipStr(ip []byte) string {
-	var f = "%d"
-	var s = "."
-	if len(ip) > 4 {
-		f = "%x"
-		s = ":"
-	}
-	var ss = make([]string, len(ip))
-	for i := range ip {
-		ss[i] = fmt.Sprintf(f, ip[i])
-	}
-	return strings.Join(ss, s)
-}

files/set.go

@@ -108,8 +108,8 @@ func (m *Set) Need(id uint) []scanner.File {
 	if debug {
 		dlog.Printf("Need(%d)", id)
 	}
-	var fs = make([]scanner.File, 0, len(m.globalKey)/2) // Just a guess, but avoids too many reallocations
 	m.Lock()
+	var fs = make([]scanner.File, 0, len(m.globalKey)/2) // Just a guess, but avoids too many reallocations
 	rkID := m.remoteKey[id]
 	for gk, gf := range m.files {
 		if !gf.Global {
@@ -145,8 +145,8 @@ func (m *Set) Global() []scanner.File {
 	if debug {
 		dlog.Printf("Global()")
 	}
-	var fs = make([]scanner.File, 0, len(m.globalKey))
 	m.Lock()
+	var fs = make([]scanner.File, 0, len(m.globalKey))
 	for _, file := range m.files {
 		if file.Global {
 			fs = append(fs, file.File)

gui/app.js

@@ -4,6 +4,7 @@
 'use strict';
 
 var syncthing = angular.module('syncthing', []);
+var urlbase = 'rest';
 
 syncthing.controller('SyncthingCtrl', function ($scope, $http) {
     var prevDate = 0;
@@ -75,18 +76,18 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
     }
 
     $scope.refresh = function () {
-        $http.get('/rest/system').success(function (data) {
+        $http.get(urlbase + '/system').success(function (data) {
             getSucceeded();
             $scope.system = data;
         }).error(function () {
             getFailed();
         });
         $scope.repos.forEach(function (repo) {
-            $http.get('/rest/model?repo=' + encodeURIComponent(repo.ID)).success(function (data) {
+            $http.get(urlbase + '/model?repo=' + encodeURIComponent(repo.ID)).success(function (data) {
                 $scope.model[repo.ID] = data;
             });
         });
-        $http.get('/rest/connections').success(function (data) {
+        $http.get(urlbase + '/connections').success(function (data) {
             var now = Date.now(),
             td = (now - prevDate) / 1000,
             id;
@@ -111,7 +112,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
             }
             $scope.connections = data;
         });
-        $http.get('/rest/errors').success(function (data) {
+        $http.get(urlbase + '/errors').success(function (data) {
             $scope.errors = data;
         });
     };
@@ -121,6 +122,10 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
             return 'Unknown';
         }
 
+        if ($scope.model[repo].invalid !== '') {
+            return 'Stopped';
+        }
+
         var state = '' + $scope.model[repo].state;
         state = state[0].toUpperCase() + state.substr(1);
 
@@ -136,6 +141,10 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
             return 'text-info';
         }
 
+        if ($scope.model[repo].invalid !== '') {
+            return 'text-warning';
+        }
+
         var state = '' + $scope.model[repo].state;
         if (state == 'idle') {
             return 'text-success';
@@ -238,14 +247,14 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
     $scope.saveSettings = function () {
         $scope.configInSync = false;
         $scope.config.Options.ListenAddress = $scope.config.Options.ListenStr.split(',').map(function (x) { return x.trim(); });
-        $http.post('/rest/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
+        $http.post(urlbase + '/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
         $('#settings').modal("hide");
     };
 
     $scope.restart = function () {
         restarting = true;
         $('#restarting').modal('show');
-        $http.post('/rest/restart');
+        $http.post(urlbase + '/restart');
         $scope.configInSync = true;
     };
 
@@ -282,7 +291,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         }
 
         $scope.configInSync = false;
-        $http.post('/rest/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
+        $http.post(urlbase + '/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
     };
 
     $scope.saveNode = function () {
@@ -309,7 +318,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         $scope.nodes.sort(nodeCompare);
         $scope.config.Nodes = $scope.nodes;
 
-        $http.post('/rest/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
+        $http.post(urlbase + '/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
     };
 
     $scope.otherNodes = function () {
@@ -337,7 +346,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
 
     $scope.clearErrors = function () {
         $scope.seenError = $scope.errors[$scope.errors.length - 1].Time;
-        $http.post('/rest/error/clear');
+        $http.post(urlbase + '/error/clear');
     };
 
     $scope.friendlyNodes = function (str) {
@@ -393,7 +402,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
 
         $scope.config.Repositories = $scope.repos;
 
-        $http.post('/rest/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
+        $http.post(urlbase + '/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
     };
 
     $scope.deleteRepo = function () {
@@ -409,19 +418,19 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         $scope.config.Repositories = $scope.repos;
 
         $scope.configInSync = false;
-        $http.post('/rest/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
+        $http.post(urlbase + '/config', JSON.stringify($scope.config), {headers: {'Content-Type': 'application/json'}});
     };
 
-    $http.get('/rest/version').success(function (data) {
+    $http.get(urlbase + '/version').success(function (data) {
         $scope.version = data;
     });
 
-    $http.get('/rest/system').success(function (data) {
+    $http.get(urlbase + '/system').success(function (data) {
         $scope.system = data;
         $scope.myID = data.myID;
     });
 
-    $http.get('/rest/config').success(function (data) {
+    $http.get(urlbase + '/config').success(function (data) {
         $scope.config = data;
         $scope.config.Options.ListenStr = $scope.config.Options.ListenAddress.join(', ');
 
@@ -434,7 +443,7 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http) {
         $scope.refresh();
     });
 
-    $http.get('/rest/config/sync').success(function (data) {
+    $http.get(urlbase + '/config/sync').success(function (data) {
         $scope.configInSync = data.configInSync;
     });
 

gui/index.html

@@ -134,6 +134,7 @@
             <ul class="list-unstyled" ng-repeat="repo in repos">
               <li>
                 <span class="text-monospace">{{repo.Directory}}</span>
+                <span ng-if="repo.Invalid" class="label label-danger">Invalid: {{repo.Invalid}}</span>
                 <ul class="list-no-bullet">
                   <li>
                     <div class="li-column" title="Repository ID">
@@ -360,8 +361,9 @@
           <form role="form">
             <div class="form-group">
               <label for="nodeID">Node ID</label>
-              <input ng-disabled="editingExisting" id="nodeID" class="form-control" type="text" ng-model="currentNode.NodeID"></input>
-              <p class="help-block">The node ID can be found in the logs or in the "Add Node" dialog on the other node.</p>
+              <input ng-if="!editingExisting" id="nodeID" class="form-control" type="text" ng-model="currentNode.NodeID"></input>
+              <div ng-if="editingExisting" class="well well-sm">{{currentNode.NodeID}}</div>
+              <p class="help-block">The node ID can be found in the "Add Node" dialog on the other node.</p>
             </div>
             <div class="form-group">
               <label for="name">Name</label>

mc/beacon.go

@@ -45,6 +45,9 @@ func (b *Beacon) run() {
 	if err != nil {
 		log.Fatal(err)
 	}
+	if debug {
+		dlog.Printf("trying %d interfaces", len(intfs))
+	}
 
 	for _, intf := range intfs {
 		intf := intf
@@ -55,10 +58,13 @@ func (b *Beacon) run() {
 		conn, err := net.ListenMulticastUDP("udp4", &intf, group)
 		if err != nil {
 			if debug {
-				dlog.Printf("listen for multicast group on %q: %v", intf.Name, err)
+				dlog.Printf("failed to listen for multicast group on %q: %v", intf.Name, err)
 			}
 		} else {
 			b.conns = append(b.conns, conn)
+			if debug {
+				dlog.Printf("listening for multicast group on %q", intf.Name)
+			}
 		}
 	}
 
@@ -72,6 +78,9 @@ func (b *Beacon) run() {
 					dlog.Println(err)
 					return
 				}
+				if debug {
+					dlog.Printf("recv %d bytes from %s on %v", n, addr, conn)
+				}
 				b.outbox <- recv{bs[:n], addr}
 			}
 		}()
@@ -85,6 +94,9 @@ func (b *Beacon) run() {
 					dlog.Println(err)
 					return
 				}
+				if debug {
+					dlog.Printf("sent %d bytes to %s on %v", len(bs), group, conn)
+				}
 			}
 		}
 	}()

protocol/protocol.go

@@ -64,24 +64,26 @@ type Connection interface {
 }
 
 type rawConnection struct {
-	sync.RWMutex
+	id       string
+	receiver Model
+
+	reader io.ReadCloser
+	cr     *countingReader
+	xr     *xdr.Reader
+	writer io.WriteCloser
+
+	cw   *countingWriter
+	wb   *bufio.Writer
+	xw   *xdr.Writer
+	wmut sync.Mutex
+
+	close  chan error
+	closed chan struct{}
 
-	id        string
-	receiver  Model
-	reader    io.ReadCloser
-	cr        *countingReader
-	xr        *xdr.Reader
-	writer    io.WriteCloser
-	cw        *countingWriter
-	wb        *bufio.Writer
-	xw        *xdr.Writer
-	closed    chan struct{}
 	awaiting  map[int]chan asyncResult
 	nextID    int
 	indexSent map[string]map[string][2]int64
-
-	hasSentIndex  bool
-	hasRecvdIndex bool
+	imut      sync.Mutex
 }
 
 type asyncResult struct {
@@ -115,11 +117,13 @@ func NewConnection(nodeID string, reader io.Reader, writer io.Writer, receiver M
 		cw:        cw,
 		wb:        wb,
 		xw:        xdr.NewWriter(wb),
+		close:     make(chan error),
 		closed:    make(chan struct{}),
 		awaiting:  make(map[int]chan asyncResult),
 		indexSent: make(map[string]map[string][2]int64),
 	}
 
+	go c.closer()
 	go c.readerLoop()
 	go c.pingerLoop()
 
@@ -132,11 +136,11 @@ func (c *rawConnection) ID() string {
 
 // Index writes the list of file information to the connected peer node
 func (c *rawConnection) Index(repo string, idx []FileInfo) {
-	c.Lock()
 	if c.isClosed() {
-		c.Unlock()
 		return
 	}
+
+	c.imut.Lock()
 	var msgType int
 	if c.indexSent[repo] == nil {
 		// This is the first time we send an index.
@@ -159,45 +163,48 @@ func (c *rawConnection) Index(repo string, idx []FileInfo) {
 		idx = diff
 	}
 
-	header{0, c.nextID, msgType}.encodeXDR(c.xw)
-	_, err := IndexMessage{repo, idx}.encodeXDR(c.xw)
-	if err == nil {
-		err = c.flush()
-	}
+	id := c.nextID
 	c.nextID = (c.nextID + 1) & 0xfff
-	c.hasSentIndex = true
-	c.Unlock()
+	c.imut.Unlock()
+
+	c.wmut.Lock()
+	header{0, id, msgType}.encodeXDR(c.xw)
+	IndexMessage{repo, idx}.encodeXDR(c.xw)
+	err := c.flush()
+	c.wmut.Unlock()
 
 	if err != nil {
-		c.close(err)
+		c.close <- err
 		return
 	}
 }
 
 // Request returns the bytes for the specified block after fetching them from the connected peer.
 func (c *rawConnection) Request(repo string, name string, offset int64, size int) ([]byte, error) {
-	c.Lock()
 	if c.isClosed() {
-		c.Unlock()
 		return nil, ErrClosed
 	}
+
+	c.imut.Lock()
+	id := c.nextID
+	c.nextID = (c.nextID + 1) & 0xfff
 	rc := make(chan asyncResult)
-	if _, ok := c.awaiting[c.nextID]; ok {
+	if _, ok := c.awaiting[id]; ok {
 		panic("id taken")
 	}
-	c.awaiting[c.nextID] = rc
-	header{0, c.nextID, messageTypeRequest}.encodeXDR(c.xw)
-	_, err := RequestMessage{repo, name, uint64(offset), uint32(size)}.encodeXDR(c.xw)
-	if err == nil {
-		err = c.flush()
-	}
+	c.awaiting[id] = rc
+	c.imut.Unlock()
+
+	c.wmut.Lock()
+	header{0, id, messageTypeRequest}.encodeXDR(c.xw)
+	RequestMessage{repo, name, uint64(offset), uint32(size)}.encodeXDR(c.xw)
+	err := c.flush()
+	c.wmut.Unlock()
+
 	if err != nil {
-		c.Unlock()
-		c.close(err)
+		c.close <- err
 		return nil, err
 	}
-	c.nextID = (c.nextID + 1) & 0xfff
-	c.Unlock()
 
 	res, ok := <-rc
 	if !ok {
@@ -208,46 +215,47 @@ func (c *rawConnection) Request(repo string, name string, offset int64, size int
 
 // ClusterConfig send the cluster configuration message to the peer and returns any error
 func (c *rawConnection) ClusterConfig(config ClusterConfigMessage) {
-	c.Lock()
-	defer c.Unlock()
-
 	if c.isClosed() {
 		return
 	}
 
-	header{0, c.nextID, messageTypeClusterConfig}.encodeXDR(c.xw)
+	c.imut.Lock()
+	id := c.nextID
 	c.nextID = (c.nextID + 1) & 0xfff
+	c.imut.Unlock()
+
+	c.wmut.Lock()
+	header{0, id, messageTypeClusterConfig}.encodeXDR(c.xw)
+	config.encodeXDR(c.xw)
+	err := c.flush()
+	c.wmut.Unlock()
 
-	_, err := config.encodeXDR(c.xw)
-	if err == nil {
-		err = c.flush()
-	}
 	if err != nil {
-		c.close(err)
+		c.close <- err
 	}
 }
 
 func (c *rawConnection) ping() bool {
-	c.Lock()
 	if c.isClosed() {
-		c.Unlock()
-		return false
-	}
-	rc := make(chan asyncResult, 1)
-	c.awaiting[c.nextID] = rc
-	header{0, c.nextID, messageTypePing}.encodeXDR(c.xw)
-	err := c.flush()
-	if err != nil {
-		c.Unlock()
-		c.close(err)
-		return false
-	} else if c.xw.Error() != nil {
-		c.Unlock()
-		c.close(c.xw.Error())
 		return false
 	}
+
+	c.imut.Lock()
+	id := c.nextID
 	c.nextID = (c.nextID + 1) & 0xfff
-	c.Unlock()
+	rc := make(chan asyncResult, 1)
+	c.awaiting[id] = rc
+	c.imut.Unlock()
+
+	c.wmut.Lock()
+	header{0, id, messageTypePing}.encodeXDR(c.xw)
+	err := c.flush()
+	c.wmut.Unlock()
+
+	if err != nil {
+		c.close <- err
+		return false
+	}
 
 	res, ok := <-rc
 	return ok && res.err == nil
@@ -258,21 +266,24 @@ type flusher interface {
 }
 
 func (c *rawConnection) flush() error {
-	c.wb.Flush()
+	if err := c.xw.Error(); err != nil {
+		return err
+	}
+
+	if err := c.wb.Flush(); err != nil {
+		return err
+	}
+
 	if f, ok := c.writer.(flusher); ok {
 		return f.Flush()
 	}
+
 	return nil
 }
 
-func (c *rawConnection) close(err error) {
-	c.Lock()
-	select {
-	case <-c.closed:
-		c.Unlock()
-		return
-	default:
-	}
+func (c *rawConnection) closer() {
+	err := <-c.close
+
 	close(c.closed)
 	for _, ch := range c.awaiting {
 		close(ch)
@@ -280,7 +291,6 @@ func (c *rawConnection) close(err error) {
 	c.awaiting = nil
 	c.writer.Close()
 	c.reader.Close()
-	c.Unlock()
 
 	c.receiver.Close(c.id, err)
 }
@@ -299,12 +309,12 @@ loop:
 	for !c.isClosed() {
 		var hdr header
 		hdr.decodeXDR(c.xr)
-		if c.xr.Error() != nil {
-			c.close(c.xr.Error())
+		if err := c.xr.Error(); err != nil {
+			c.close <- err
 			break loop
 		}
 		if hdr.version != 0 {
-			c.close(fmt.Errorf("protocol error: %s: unknown message version %#x", c.id, hdr.version))
+			c.close <- fmt.Errorf("protocol error: %s: unknown message version %#x", c.id, hdr.version)
 			break loop
 		}
 
@@ -312,8 +322,8 @@ loop:
 		case messageTypeIndex:
 			var im IndexMessage
 			im.decodeXDR(c.xr)
-			if c.xr.Error() != nil {
-				c.close(c.xr.Error())
+			if err := c.xr.Error(); err != nil {
+				c.close <- err
 				break loop
 			} else {
 
@@ -326,15 +336,12 @@ loop:
 
 				go c.receiver.Index(c.id, im.Repository, im.Files)
 			}
-			c.Lock()
-			c.hasRecvdIndex = true
-			c.Unlock()
 
 		case messageTypeIndexUpdate:
 			var im IndexMessage
 			im.decodeXDR(c.xr)
-			if c.xr.Error() != nil {
-				c.close(c.xr.Error())
+			if err := c.xr.Error(); err != nil {
+				c.close <- err
 				break loop
 			} else {
 				go c.receiver.IndexUpdate(c.id, im.Repository, im.Files)
@@ -343,8 +350,8 @@ loop:
 		case messageTypeRequest:
 			var req RequestMessage
 			req.decodeXDR(c.xr)
-			if c.xr.Error() != nil {
-				c.close(c.xr.Error())
+			if err := c.xr.Error(); err != nil {
+				c.close <- err
 				break loop
 			}
 			go c.processRequest(hdr.msgID, req)
@@ -352,16 +359,16 @@ loop:
 		case messageTypeResponse:
 			data := c.xr.ReadBytesMax(256 * 1024) // Sufficiently larger than max expected block size
 
-			if c.xr.Error() != nil {
-				c.close(c.xr.Error())
+			if err := c.xr.Error(); err != nil {
+				c.close <- err
 				break loop
 			}
 
 			go func(hdr header, err error) {
-				c.Lock()
+				c.imut.Lock()
 				rc, ok := c.awaiting[hdr.msgID]
 				delete(c.awaiting, hdr.msgID)
-				c.Unlock()
+				c.imut.Unlock()
 
 				if ok {
 					rc <- asyncResult{data, err}
@@ -370,44 +377,41 @@ loop:
 			}(hdr, c.xr.Error())
 
 		case messageTypePing:
-			c.Lock()
+			c.wmut.Lock()
 			header{0, hdr.msgID, messageTypePong}.encodeXDR(c.xw)
 			err := c.flush()
-			c.Unlock()
+			c.wmut.Unlock()
 			if err != nil {
-				c.close(err)
-				break loop
-			} else if c.xw.Error() != nil {
-				c.close(c.xw.Error())
+				c.close <- err
 				break loop
 			}
 
 		case messageTypePong:
-			c.RLock()
+			c.imut.Lock()
 			rc, ok := c.awaiting[hdr.msgID]
-			c.RUnlock()
 
 			if ok {
-				rc <- asyncResult{}
-				close(rc)
+				go func() {
+					rc <- asyncResult{}
+					close(rc)
+				}()
 
-				c.Lock()
 				delete(c.awaiting, hdr.msgID)
-				c.Unlock()
 			}
+			c.imut.Unlock()
 
 		case messageTypeClusterConfig:
 			var cm ClusterConfigMessage
 			cm.decodeXDR(c.xr)
-			if c.xr.Error() != nil {
-				c.close(c.xr.Error())
+			if err := c.xr.Error(); err != nil {
+				c.close <- err
 				break loop
 			} else {
 				go c.receiver.ClusterConfig(c.id, cm)
 			}
 
 		default:
-			c.close(fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType))
+			c.close <- fmt.Errorf("protocol error: %s: unknown message type %#x", c.id, hdr.msgType)
 			break loop
 		}
 	}
@@ -416,17 +420,16 @@ loop:
 func (c *rawConnection) processRequest(msgID int, req RequestMessage) {
 	data, _ := c.receiver.Request(c.id, req.Repository, req.Name, int64(req.Offset), int(req.Size))
 
-	c.Lock()
+	c.wmut.Lock()
 	header{0, msgID, messageTypeResponse}.encodeXDR(c.xw)
-	_, err := c.xw.WriteBytes(data)
-	if err == nil {
-		err = c.flush()
-	}
-	c.Unlock()
+	c.xw.WriteBytes(data)
+	err := c.flush()
+	c.wmut.Unlock()
 
 	buffers.Put(data)
+
 	if err != nil {
-		c.close(err)
+		c.close <- err
 	}
 }
 
@@ -436,22 +439,16 @@ func (c *rawConnection) pingerLoop() {
 	for {
 		select {
 		case <-ticker:
-			c.RLock()
-			ready := c.hasRecvdIndex && c.hasSentIndex
-			c.RUnlock()
-
-			if ready {
-				go func() {
-					rc <- c.ping()
-				}()
-				select {
-				case ok := <-rc:
-					if !ok {
-						c.close(fmt.Errorf("ping failure"))
-					}
-				case <-time.After(pingTimeout):
-					c.close(fmt.Errorf("ping timeout"))
+			go func() {
+				rc <- c.ping()
+			}()
+			select {
+			case ok := <-rc:
+				if !ok {
+					c.close <- fmt.Errorf("ping failure")
 				}
+			case <-time.After(pingTimeout):
+				c.close <- fmt.Errorf("ping timeout")
 			}
 		case <-c.closed:
 			return

protocol/protocol_test.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"testing"
 	"testing/quick"
+	"time"
 )
 
 func TestHeaderFunctions(t *testing.T) {
@@ -172,7 +173,13 @@ func TestClose(t *testing.T) {
 	c0 := NewConnection("c0", ar, bw, m0).(wireFormatConnection).next.(*rawConnection)
 	NewConnection("c1", br, aw, m1)
 
-	c0.close(nil)
+	c0.close <- nil
+
+	select {
+	case <-c0.closed:
+	case <-time.After(1 * time.Second):
+		t.Fatal("Did not close within a second")
+	}
 
 	if !c0.isClosed() {
 		t.Fatal("Connection should be closed")