Slightly more compact GUI resources

build.sh

@@ -3,22 +3,17 @@
 version=$(git describe --always)
 buildDir=dist
 
-if [[ $1 == "-f" ]] ; then
-	fast=yes
-	shift
-fi
-
 if [[ $fast != yes ]] ; then
 	go get -d
 	go test ./...
 fi
 
 if [[ -z $1 ]] ; then
-	go build -ldflags "-X main.Version $version" \
-	&& nrsc syncthing gui
+	go build -ldflags "-X main.Version $version"
+elif [[ $1 == "embed" ]] ; then
+	embedder main gui > gui.files.go
 elif [[ $1 == "tar" ]] ; then
 	go build -ldflags "-X main.Version $version" \
-	&& nrsc syncthing gui \
 	&& mkdir syncthing-dist \
 	&& cp syncthing README.md LICENSE syncthing-dist \
 	&& tar zcvf syncthing-dist.tar.gz syncthing-dist \
@@ -34,7 +29,6 @@ elif [[ $1 == "all" ]] ; then
 			export GOARCH="$goarch"
 			export name="syncthing-$goos-$goarch"
 			go build -ldflags "-X main.Version $version" \
-				&& nrsc syncthing gui \
 				&& mkdir -p "$name" \
 				&& cp syncthing "$buildDir/$name" \
 				&& cp README.md LICENSE "$name" \
@@ -53,7 +47,6 @@ elif [[ $1 == "all" ]] ; then
 				export GOARCH="$goarch"
 				export name="syncthing-$goos-${goarch}v$goarm"
 				go build -ldflags "-X main.Version $version" \
-					&& nrsc syncthing gui \
 					&& mkdir -p "$name" \
 					&& cp syncthing "$buildDir/$name" \
 					&& cp README.md LICENSE "$name" \
@@ -71,10 +64,10 @@ elif [[ $1 == "all" ]] ; then
 			export GOARCH="$goarch"
 			export name="syncthing-$goos-$goarch"
 			go build -ldflags "-X main.Version $version" \
-				&& nrsc syncthing.exe gui \
 				&& mkdir -p "$name" \
-				&& mv syncthing.exe "$buildDir/$name.exe" \
+				&& cp syncthing.exe "$buildDir/$name.exe" \
 				&& cp README.md LICENSE "$name" \
+				&& mv syncthing.exe "$name" \
 				&& zip -qr "$buildDir/$name.zip" "$name" \
 				&& rm -r  "$name"
 		done

gui.go

@@ -3,17 +3,17 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"io"
 	"log"
 	"mime"
 	"net/http"
 	"path/filepath"
 	"runtime"
 	"sync"
+	"time"
 
-	"bitbucket.org/tebeka/nrsc"
 	"github.com/calmh/syncthing/model"
 	"github.com/codegangsta/martini"
+	"github.com/cratonica/embed"
 )
 
 func startGUI(addr string, m *model.Model) {
@@ -26,9 +26,14 @@ func startGUI(addr string, m *model.Model) {
 	router.Get("/rest/need", restGetNeed)
 	router.Get("/rest/system", restGetSystem)
 
+	fs, err := embed.Unpack(Resources)
+	if err != nil {
+		panic(err)
+	}
+
 	go func() {
 		mr := martini.New()
-		mr.Use(nrscStatic("gui"))
+		mr.Use(embeddedStatic(fs))
 		mr.Use(martini.Recovery())
 		mr.Action(router.Handle)
 		mr.Map(m)
@@ -124,36 +129,28 @@ func restGetSystem(w http.ResponseWriter) {
 	json.NewEncoder(w).Encode(res)
 }
 
-func nrscStatic(path string) interface{} {
-	if err := nrsc.Initialize(); err != nil {
-		panic("Unable to initialize nrsc: " + err.Error())
-	}
+func embeddedStatic(fs map[string][]byte) interface{} {
+	var modt = time.Now().UTC().Format(http.TimeFormat)
+
 	return func(res http.ResponseWriter, req *http.Request, log *log.Logger) {
 		file := req.URL.Path
 
-		// nrsc expects there not to be a leading slash
 		if file[0] == '/' {
 			file = file[1:]
 		}
 
-		f := nrsc.Get(file)
-		if f == nil {
+		bs, ok := fs[file]
+		if !ok {
 			return
 		}
 
-		rdr, err := f.Open()
-		if err != nil {
-			http.Error(res, "Internal Server Error", http.StatusInternalServerError)
-		}
-		defer rdr.Close()
-
 		mtype := mime.TypeByExtension(filepath.Ext(req.URL.Path))
 		if len(mtype) != 0 {
 			res.Header().Set("Content-Type", mtype)
 		}
-		res.Header().Set("Content-Size", fmt.Sprintf("%d", f.Size()))
-		res.Header().Set("Last-Modified", f.ModTime().UTC().Format(http.TimeFormat))
+		res.Header().Set("Content-Size", fmt.Sprintf("%d", len(bs)))
+		res.Header().Set("Last-Modified", modt)
 
-		io.Copy(res, rdr)
+		res.Write(bs)
 	}
 }

gui/bootstrap/css/bootstrap-theme.css

@@ -1,397 +0,0 @@
-/*!
- * Bootstrap v3.0.3 (http://getbootstrap.com)
- * Copyright 2013 Twitter, Inc.
- * Licensed under http://www.apache.org/licenses/LICENSE-2.0
- */
-
-.btn-default,
-.btn-primary,
-.btn-success,
-.btn-info,
-.btn-warning,
-.btn-danger {
-  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.2);
-  -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.15), 0 1px 1px rgba(0, 0, 0, 0.075);
-          box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.15), 0 1px 1px rgba(0, 0, 0, 0.075);
-}
-
-.btn-default:active,
-.btn-primary:active,
-.btn-success:active,
-.btn-info:active,
-.btn-warning:active,
-.btn-danger:active,
-.btn-default.active,
-.btn-primary.active,
-.btn-success.active,
-.btn-info.active,
-.btn-warning.active,
-.btn-danger.active {
-  -webkit-box-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);
-          box-shadow: inset 0 3px 5px rgba(0, 0, 0, 0.125);
-}
-
-.btn:active,
-.btn.active {
-  background-image: none;
-}
-
-.btn-default {
-  text-shadow: 0 1px 0 #fff;
-  background-image: -webkit-linear-gradient(top, #ffffff 0%, #e0e0e0 100%);
-  background-image: linear-gradient(to bottom, #ffffff 0%, #e0e0e0 100%);
-  background-repeat: repeat-x;
-  border-color: #dbdbdb;
-  border-color: #ccc;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff', endColorstr='#ffe0e0e0', GradientType=0);
-  filter: progid:DXImageTransform.Microsoft.gradient(enabled=false);
-}
-
-.btn-default:hover,
-.btn-default:focus {
-  background-color: #e0e0e0;
-  background-position: 0 -15px;
-}
-
-.btn-default:active,
-.btn-default.active {
-  background-color: #e0e0e0;
-  border-color: #dbdbdb;
-}
-
-.btn-primary {
-  background-image: -webkit-linear-gradient(top, #428bca 0%, #2d6ca2 100%);
-  background-image: linear-gradient(to bottom, #428bca 0%, #2d6ca2 100%);
-  background-repeat: repeat-x;
-  border-color: #2b669a;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff428bca', endColorstr='#ff2d6ca2', GradientType=0);
-  filter: progid:DXImageTransform.Microsoft.gradient(enabled=false);
-}
-
-.btn-primary:hover,
-.btn-primary:focus {
-  background-color: #2d6ca2;
-  background-position: 0 -15px;
-}
-
-.btn-primary:active,
-.btn-primary.active {
-  background-color: #2d6ca2;
-  border-color: #2b669a;
-}
-
-.btn-success {
-  background-image: -webkit-linear-gradient(top, #5cb85c 0%, #419641 100%);
-  background-image: linear-gradient(to bottom, #5cb85c 0%, #419641 100%);
-  background-repeat: repeat-x;
-  border-color: #3e8f3e;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5cb85c', endColorstr='#ff419641', GradientType=0);
-  filter: progid:DXImageTransform.Microsoft.gradient(enabled=false);
-}
-
-.btn-success:hover,
-.btn-success:focus {
-  background-color: #419641;
-  background-position: 0 -15px;
-}
-
-.btn-success:active,
-.btn-success.active {
-  background-color: #419641;
-  border-color: #3e8f3e;
-}
-
-.btn-warning {
-  background-image: -webkit-linear-gradient(top, #f0ad4e 0%, #eb9316 100%);
-  background-image: linear-gradient(to bottom, #f0ad4e 0%, #eb9316 100%);
-  background-repeat: repeat-x;
-  border-color: #e38d13;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff0ad4e', endColorstr='#ffeb9316', GradientType=0);
-  filter: progid:DXImageTransform.Microsoft.gradient(enabled=false);
-}
-
-.btn-warning:hover,
-.btn-warning:focus {
-  background-color: #eb9316;
-  background-position: 0 -15px;
-}
-
-.btn-warning:active,
-.btn-warning.active {
-  background-color: #eb9316;
-  border-color: #e38d13;
-}
-
-.btn-danger {
-  background-image: -webkit-linear-gradient(top, #d9534f 0%, #c12e2a 100%);
-  background-image: linear-gradient(to bottom, #d9534f 0%, #c12e2a 100%);
-  background-repeat: repeat-x;
-  border-color: #b92c28;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffd9534f', endColorstr='#ffc12e2a', GradientType=0);
-  filter: progid:DXImageTransform.Microsoft.gradient(enabled=false);
-}
-
-.btn-danger:hover,
-.btn-danger:focus {
-  background-color: #c12e2a;
-  background-position: 0 -15px;
-}
-
-.btn-danger:active,
-.btn-danger.active {
-  background-color: #c12e2a;
-  border-color: #b92c28;
-}
-
-.btn-info {
-  background-image: -webkit-linear-gradient(top, #5bc0de 0%, #2aabd2 100%);
-  background-image: linear-gradient(to bottom, #5bc0de 0%, #2aabd2 100%);
-  background-repeat: repeat-x;
-  border-color: #28a4c9;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff2aabd2', GradientType=0);
-  filter: progid:DXImageTransform.Microsoft.gradient(enabled=false);
-}
-
-.btn-info:hover,
-.btn-info:focus {
-  background-color: #2aabd2;
-  background-position: 0 -15px;
-}
-
-.btn-info:active,
-.btn-info.active {
-  background-color: #2aabd2;
-  border-color: #28a4c9;
-}
-
-.thumbnail,
-.img-thumbnail {
-  -webkit-box-shadow: 0 1px 2px rgba(0, 0, 0, 0.075);
-          box-shadow: 0 1px 2px rgba(0, 0, 0, 0.075);
-}
-
-.dropdown-menu > li > a:hover,
-.dropdown-menu > li > a:focus {
-  background-color: #e8e8e8;
-  background-image: -webkit-linear-gradient(top, #f5f5f5 0%, #e8e8e8 100%);
-  background-image: linear-gradient(to bottom, #f5f5f5 0%, #e8e8e8 100%);
-  background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff5f5f5', endColorstr='#ffe8e8e8', GradientType=0);
-}
-
-.dropdown-menu > .active > a,
-.dropdown-menu > .active > a:hover,
-.dropdown-menu > .active > a:focus {
-  background-color: #357ebd;
-  background-image: -webkit-linear-gradient(top, #428bca 0%, #357ebd 100%);
-  background-image: linear-gradient(to bottom, #428bca 0%, #357ebd 100%);
-  background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff428bca', endColorstr='#ff357ebd', GradientType=0);
-}
-
-.navbar-default {
-  background-image: -webkit-linear-gradient(top, #ffffff 0%, #f8f8f8 100%);
-  background-image: linear-gradient(to bottom, #ffffff 0%, #f8f8f8 100%);
-  background-repeat: repeat-x;
-  border-radius: 4px;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff', endColorstr='#fff8f8f8', GradientType=0);
-  filter: progid:DXImageTransform.Microsoft.gradient(enabled=false);
-  -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.15), 0 1px 5px rgba(0, 0, 0, 0.075);
-          box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.15), 0 1px 5px rgba(0, 0, 0, 0.075);
-}
-
-.navbar-default .navbar-nav > .active > a {
-  background-image: -webkit-linear-gradient(top, #ebebeb 0%, #f3f3f3 100%);
-  background-image: linear-gradient(to bottom, #ebebeb 0%, #f3f3f3 100%);
-  background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffebebeb', endColorstr='#fff3f3f3', GradientType=0);
-  -webkit-box-shadow: inset 0 3px 9px rgba(0, 0, 0, 0.075);
-          box-shadow: inset 0 3px 9px rgba(0, 0, 0, 0.075);
-}
-
-.navbar-brand,
-.navbar-nav > li > a {
-  text-shadow: 0 1px 0 rgba(255, 255, 255, 0.25);
-}
-
-.navbar-inverse {
-  background-image: -webkit-linear-gradient(top, #3c3c3c 0%, #222222 100%);
-  background-image: linear-gradient(to bottom, #3c3c3c 0%, #222222 100%);
-  background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff3c3c3c', endColorstr='#ff222222', GradientType=0);
-  filter: progid:DXImageTransform.Microsoft.gradient(enabled=false);
-}
-
-.navbar-inverse .navbar-nav > .active > a {
-  background-image: -webkit-linear-gradient(top, #222222 0%, #282828 100%);
-  background-image: linear-gradient(to bottom, #222222 0%, #282828 100%);
-  background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff222222', endColorstr='#ff282828', GradientType=0);
-  -webkit-box-shadow: inset 0 3px 9px rgba(0, 0, 0, 0.25);
-          box-shadow: inset 0 3px 9px rgba(0, 0, 0, 0.25);
-}
-
-.navbar-inverse .navbar-brand,
-.navbar-inverse .navbar-nav > li > a {
-  text-shadow: 0 -1px 0 rgba(0, 0, 0, 0.25);
-}
-
-.navbar-static-top,
-.navbar-fixed-top,
-.navbar-fixed-bottom {
-  border-radius: 0;
-}
-
-.alert {
-  text-shadow: 0 1px 0 rgba(255, 255, 255, 0.2);
-  -webkit-box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.25), 0 1px 2px rgba(0, 0, 0, 0.05);
-          box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.25), 0 1px 2px rgba(0, 0, 0, 0.05);
-}
-
-.alert-success {
-  background-image: -webkit-linear-gradient(top, #dff0d8 0%, #c8e5bc 100%);
-  background-image: linear-gradient(to bottom, #dff0d8 0%, #c8e5bc 100%);
-  background-repeat: repeat-x;
-  border-color: #b2dba1;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffdff0d8', endColorstr='#ffc8e5bc', GradientType=0);
-}
-
-.alert-info {
-  background-image: -webkit-linear-gradient(top, #d9edf7 0%, #b9def0 100%);
-  background-image: linear-gradient(to bottom, #d9edf7 0%, #b9def0 100%);
-  background-repeat: repeat-x;
-  border-color: #9acfea;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffd9edf7', endColorstr='#ffb9def0', GradientType=0);
-}
-
-.alert-warning {
-  background-image: -webkit-linear-gradient(top, #fcf8e3 0%, #f8efc0 100%);
-  background-image: linear-gradient(to bottom, #fcf8e3 0%, #f8efc0 100%);
-  background-repeat: repeat-x;
-  border-color: #f5e79e;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fffcf8e3', endColorstr='#fff8efc0', GradientType=0);
-}
-
-.alert-danger {
-  background-image: -webkit-linear-gradient(top, #f2dede 0%, #e7c3c3 100%);
-  background-image: linear-gradient(to bottom, #f2dede 0%, #e7c3c3 100%);
-  background-repeat: repeat-x;
-  border-color: #dca7a7;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff2dede', endColorstr='#ffe7c3c3', GradientType=0);
-}
-
-.progress {
-  background-image: -webkit-linear-gradient(top, #ebebeb 0%, #f5f5f5 100%);
-  background-image: linear-gradient(to bottom, #ebebeb 0%, #f5f5f5 100%);
-  background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffebebeb', endColorstr='#fff5f5f5', GradientType=0);
-}
-
-.progress-bar {
-  background-image: -webkit-linear-gradient(top, #428bca 0%, #3071a9 100%);
-  background-image: linear-gradient(to bottom, #428bca 0%, #3071a9 100%);
-  background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff428bca', endColorstr='#ff3071a9', GradientType=0);
-}
-
-.progress-bar-success {
-  background-image: -webkit-linear-gradient(top, #5cb85c 0%, #449d44 100%);
-  background-image: linear-gradient(to bottom, #5cb85c 0%, #449d44 100%);
-  background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5cb85c', endColorstr='#ff449d44', GradientType=0);
-}
-
-.progress-bar-info {
-  background-image: -webkit-linear-gradient(top, #5bc0de 0%, #31b0d5 100%);
-  background-image: linear-gradient(to bottom, #5bc0de 0%, #31b0d5 100%);
-  background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff31b0d5', GradientType=0);
-}
-
-.progress-bar-warning {
-  background-image: -webkit-linear-gradient(top, #f0ad4e 0%, #ec971f 100%);
-  background-image: linear-gradient(to bottom, #f0ad4e 0%, #ec971f 100%);
-  background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff0ad4e', endColorstr='#ffec971f', GradientType=0);
-}
-
-.progress-bar-danger {
-  background-image: -webkit-linear-gradient(top, #d9534f 0%, #c9302c 100%);
-  background-image: linear-gradient(to bottom, #d9534f 0%, #c9302c 100%);
-  background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffd9534f', endColorstr='#ffc9302c', GradientType=0);
-}
-
-.list-group {
-  border-radius: 4px;
-  -webkit-box-shadow: 0 1px 2px rgba(0, 0, 0, 0.075);
-          box-shadow: 0 1px 2px rgba(0, 0, 0, 0.075);
-}
-
-.list-group-item.active,
-.list-group-item.active:hover,
-.list-group-item.active:focus {
-  text-shadow: 0 -1px 0 #3071a9;
-  background-image: -webkit-linear-gradient(top, #428bca 0%, #3278b3 100%);
-  background-image: linear-gradient(to bottom, #428bca 0%, #3278b3 100%);
-  background-repeat: repeat-x;
-  border-color: #3278b3;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff428bca', endColorstr='#ff3278b3', GradientType=0);
-}
-
-.panel {
-  -webkit-box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
-          box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05);
-}
-
-.panel-default > .panel-heading {
-  background-image: -webkit-linear-gradient(top, #f5f5f5 0%, #e8e8e8 100%);
-  background-image: linear-gradient(to bottom, #f5f5f5 0%, #e8e8e8 100%);
-  background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff5f5f5', endColorstr='#ffe8e8e8', GradientType=0);
-}
-
-.panel-primary > .panel-heading {
-  background-image: -webkit-linear-gradient(top, #428bca 0%, #357ebd 100%);
-  background-image: linear-gradient(to bottom, #428bca 0%, #357ebd 100%);
-  background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff428bca', endColorstr='#ff357ebd', GradientType=0);
-}
-
-.panel-success > .panel-heading {
-  background-image: -webkit-linear-gradient(top, #dff0d8 0%, #d0e9c6 100%);
-  background-image: linear-gradient(to bottom, #dff0d8 0%, #d0e9c6 100%);
-  background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffdff0d8', endColorstr='#ffd0e9c6', GradientType=0);
-}
-
-.panel-info > .panel-heading {
-  background-image: -webkit-linear-gradient(top, #d9edf7 0%, #c4e3f3 100%);
-  background-image: linear-gradient(to bottom, #d9edf7 0%, #c4e3f3 100%);
-  background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffd9edf7', endColorstr='#ffc4e3f3', GradientType=0);
-}
-
-.panel-warning > .panel-heading {
-  background-image: -webkit-linear-gradient(top, #fcf8e3 0%, #faf2cc 100%);
-  background-image: linear-gradient(to bottom, #fcf8e3 0%, #faf2cc 100%);
-  background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fffcf8e3', endColorstr='#fffaf2cc', GradientType=0);
-}
-
-.panel-danger > .panel-heading {
-  background-image: -webkit-linear-gradient(top, #f2dede 0%, #ebcccc 100%);
-  background-image: linear-gradient(to bottom, #f2dede 0%, #ebcccc 100%);
-  background-repeat: repeat-x;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff2dede', endColorstr='#ffebcccc', GradientType=0);
-}
-
-.well {
-  background-image: -webkit-linear-gradient(top, #e8e8e8 0%, #f5f5f5 100%);
-  background-image: linear-gradient(to bottom, #e8e8e8 0%, #f5f5f5 100%);
-  background-repeat: repeat-x;
-  border-color: #dcdcdc;
-  filter: progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffe8e8e8', endColorstr='#fff5f5f5', GradientType=0);
-  -webkit-box-shadow: inset 0 1px 3px rgba(0, 0, 0, 0.05), 0 1px 0 rgba(255, 255, 255, 0.1);
-          box-shadow: inset 0 1px 3px rgba(0, 0, 0, 0.05), 0 1px 0 rgba(255, 255, 255, 0.1);
-}

gui/index.html

@@ -9,7 +9,7 @@
 <link rel="shortcut icon" href="../../docs-assets/ico/favicon.png">
 
 <title>syncthing</title>
-<link href="bootstrap/css/bootstrap.css" rel="stylesheet">
+<link href="bootstrap/css/bootstrap.min.css" rel="stylesheet">
 <style type="text/css">
 html, body {
     height: 100%;

integration/.gitignore

@@ -0,0 +1,5 @@
+files-*
+conf-*
+md5-*
+genfiles
+md5r

integration/genfiles.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"crypto/rand"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	mr "math/rand"
+	"os"
+	"path"
+)
+
+func name() string {
+	var b [16]byte
+	rand.Reader.Read(b[:])
+	return fmt.Sprintf("%x", b[:])
+}
+
+func main() {
+	var files int
+	var maxexp int
+
+	flag.IntVar(&files, "files", 1000, "Number of files")
+	flag.IntVar(&maxexp, "maxexp", 20, "Maximum file size (max = 2^n + 128*1024 B)")
+	flag.Parse()
+
+	for i := 0; i < files; i++ {
+		n := name()
+		p0 := path.Join(string(n[0]), n[0:2])
+		os.MkdirAll(p0, 0755)
+		s := 1 << uint(mr.Intn(maxexp))
+		a := 128 * 1024
+		if a > s {
+			a = s
+		}
+		s += mr.Intn(a)
+		b := make([]byte, s)
+		rand.Reader.Read(b)
+		p1 := path.Join(p0, n)
+		ioutil.WriteFile(p1, b, 0644)
+	}
+}

integration/md5r.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"crypto/md5"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+)
+
+func main() {
+	flag.Parse()
+	args := flag.Args()
+
+	if len(args) == 0 {
+		args = []string{"."}
+	}
+
+	for _, path := range args {
+		err := filepath.Walk(path, walker)
+
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(1)
+		}
+	}
+}
+
+func walker(path string, info os.FileInfo, err error) error {
+	if err != nil {
+		return err
+	}
+
+	if !info.IsDir() {
+		sum, err := md5file(path)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("%s  %s\n", sum, path)
+	}
+
+	return nil
+}
+
+func md5file(fname string) (hash string, err error) {
+	f, err := os.Open(fname)
+	if err != nil {
+		return
+	}
+	defer f.Close()
+
+	h := md5.New()
+	io.Copy(h, f)
+	hb := h.Sum(nil)
+	hash = fmt.Sprintf("%x", hb)
+
+	return
+}

integration/test.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+rm -rf files-* conf-* md5-*
+
+extraopts=""
+p=$(pwd)
+
+go build genfiles.go
+go build md5r.go
+
+echo "Setting up (keys)..."
+i1=$(syncthing -c conf-1 2>&1 | awk '/My ID/ {print $6}')
+echo $i1
+i2=$(syncthing -c conf-2 2>&1 | awk '/My ID/ {print $6}')
+echo $i2
+i3=$(syncthing -c conf-3 2>&1 | awk '/My ID/ {print $6}')
+echo $i3
+
+echo "Setting up (files)..."
+for i in 1 2 3 ; do
+	cat >conf-$i/syncthing.ini <<EOT
+[repository]
+dir = $p/files-$i
+
+[nodes]
+$i1 = 127.0.0.1:22001
+$i2 = 127.0.0.1:22002
+$i3 = 127.0.0.1:22003
+EOT
+
+	mkdir files-$i
+	pushd files-$i >/dev/null
+	../genfiles -maxexp 21 -files 4000
+	../md5r > ../md5-$i
+	popd >/dev/null
+done
+
+echo "Starting..."
+for i in 1 2 3 ; do
+	sleep 1
+	syncthing -c conf-$i --no-gui -l :2200$i $extraopts &
+done
+
+cat md5-* | sort > md5-tot
+while true ; do
+	read
+	echo Verifying...
+
+	conv=0
+	for i in 1 2 3 ; do
+		pushd files-$i >/dev/null
+		../md5r | sort > ../md5-$i
+		popd >/dev/null
+		if ! cmp md5-$i md5-tot >/dev/null ; then
+			echo $i unconverged
+		else
+			conv=$((conv + 1))
+			echo $i converged
+		fi
+	done
+
+	if [[ $conv == 3 ]] ; then
+		kill %1
+		kill %2
+		kill %3
+		exit
+	fi
+done
+

main.go

@@ -119,6 +119,8 @@ func main() {
 
 	myID = string(certId(cert.Certificate[0]))
 	infoln("My ID:", myID)
+	log.SetPrefix("[" + myID[0:5] + "] ")
+	logger.SetPrefix("[" + myID[0:5] + "] ")
 
 	if opts.Debug.Profiler != "" {
 		go func() {
@@ -223,7 +225,9 @@ func main() {
 	go func() {
 		for {
 			time.Sleep(opts.Advanced.ScanInterval)
-			updateLocalModel(m)
+			if m.LocalAge() > opts.Advanced.ScanInterval.Seconds()/2 {
+				updateLocalModel(m)
+			}
 		}
 	}()
 
@@ -248,7 +252,7 @@ func printStatsLoop(m *model.Model) {
 			outbps := 8 * int(float64(stats.OutBytesTotal-lastStats[node].OutBytesTotal)/secs)
 
 			if inbps+outbps > 0 {
-				infof("%s: %sb/s in, %sb/s out", node, MetricPrefix(inbps), MetricPrefix(outbps))
+				infof("%s: %sb/s in, %sb/s out", node[0:5], MetricPrefix(inbps), MetricPrefix(outbps))
 			}
 
 			lastStats[node] = stats

model/filemonitor.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"log"
 	"os"
 	"path"
 	"sync"
@@ -24,6 +25,10 @@ type fileMonitor struct {
 }
 
 func (m *fileMonitor) FileBegins(cc <-chan content) error {
+	if m.model.trace["file"] {
+		log.Printf("FILE: FileBegins: " + m.name)
+	}
+
 	tmp := tempName(m.path, m.global.Modified)
 
 	dir := path.Dir(tmp)
@@ -104,6 +109,10 @@ func (m *fileMonitor) copyRemoteBlocks(cc <-chan content, outFile *os.File, writ
 }
 
 func (m *fileMonitor) FileDone() error {
+	if m.model.trace["file"] {
+		log.Printf("FILE: FileDone: " + m.name)
+	}
+
 	m.writeDone.Wait()
 
 	tmp := tempName(m.path, m.global.Modified)
@@ -118,7 +127,7 @@ func (m *fileMonitor) FileDone() error {
 
 	err := hashCheck(tmp, m.global.Blocks)
 	if err != nil {
-		return fmt.Errorf("%s: %s (tmp) (deleting)", path.Base(m.name), err.Error())
+		return err
 	}
 
 	err = os.Chtimes(tmp, time.Unix(m.global.Modified, 0), time.Unix(m.global.Modified, 0))
@@ -136,7 +145,7 @@ func (m *fileMonitor) FileDone() error {
 		return err
 	}
 
-	go m.model.updateLocalLocked(m.global)
+	m.model.updateLocal(m.global)
 	return nil
 }
 

model/filequeue.go

@@ -14,9 +14,10 @@ type Monitor interface {
 
 type FileQueue struct {
 	files        queuedFileList
-	lock         sync.Mutex
 	sorted       bool
+	fmut         sync.Mutex // protects files and sorted
 	availability map[string][]string
+	amut         sync.Mutex // protects availability
 }
 
 type queuedFile struct {
@@ -56,9 +57,21 @@ type queuedBlock struct {
 	index int
 }
 
+func NewFileQueue() *FileQueue {
+	return &FileQueue{
+		availability: make(map[string][]string),
+	}
+}
+
 func (q *FileQueue) Add(name string, blocks []Block, monitor Monitor) {
-	q.lock.Lock()
-	defer q.lock.Unlock()
+	q.fmut.Lock()
+	defer q.fmut.Unlock()
+
+	for _, f := range q.files {
+		if f.name == name {
+			panic("re-adding added file " + f.name)
+		}
+	}
 
 	q.files = append(q.files, queuedFile{
 		name:         name,
@@ -72,15 +85,15 @@ func (q *FileQueue) Add(name string, blocks []Block, monitor Monitor) {
 }
 
 func (q *FileQueue) Len() int {
-	q.lock.Lock()
-	defer q.lock.Unlock()
+	q.fmut.Lock()
+	defer q.fmut.Unlock()
 
 	return len(q.files)
 }
 
 func (q *FileQueue) Get(nodeID string) (queuedBlock, bool) {
-	q.lock.Lock()
-	defer q.lock.Unlock()
+	q.fmut.Lock()
+	defer q.fmut.Unlock()
 
 	if !q.sorted {
 		sort.Sort(q.files)
@@ -90,7 +103,11 @@ func (q *FileQueue) Get(nodeID string) (queuedBlock, bool) {
 	for i := range q.files {
 		qf := &q.files[i]
 
-		if len(q.availability[qf.name]) == 0 {
+		q.amut.Lock()
+		av := q.availability[qf.name]
+		q.amut.Unlock()
+
+		if len(av) == 0 {
 			// Noone has the file we want; abort.
 			if qf.remaining != len(qf.blocks) {
 				// We have already started on this file; close it down
@@ -103,7 +120,7 @@ func (q *FileQueue) Get(nodeID string) (queuedBlock, bool) {
 			return queuedBlock{}, false
 		}
 
-		for _, ni := range q.availability[qf.name] {
+		for _, ni := range av {
 			// Find and return the next block in the queue
 			if ni == nodeID {
 				for j, b := range qf.blocks {
@@ -127,8 +144,8 @@ func (q *FileQueue) Get(nodeID string) (queuedBlock, bool) {
 }
 
 func (q *FileQueue) Done(file string, offset int64, data []byte) {
-	q.lock.Lock()
-	defer q.lock.Unlock()
+	q.fmut.Lock()
+	defer q.fmut.Unlock()
 
 	c := content{
 		offset: offset,
@@ -167,8 +184,8 @@ func (q *FileQueue) Done(file string, offset int64, data []byte) {
 }
 
 func (q *FileQueue) Queued(file string) bool {
-	q.lock.Lock()
-	defer q.lock.Unlock()
+	q.fmut.Lock()
+	defer q.fmut.Unlock()
 
 	for _, qf := range q.files {
 		if qf.name == file {
@@ -179,8 +196,8 @@ func (q *FileQueue) Queued(file string) bool {
 }
 
 func (q *FileQueue) QueuedFiles() (files []string) {
-	q.lock.Lock()
-	defer q.lock.Unlock()
+	q.fmut.Lock()
+	defer q.fmut.Unlock()
 
 	for _, qf := range q.files {
 		files = append(files, qf.name)
@@ -201,27 +218,17 @@ func (q *FileQueue) deleteFile(n string) {
 	}
 }
 
-func (q *FileQueue) SetAvailable(file, node string) {
-	q.lock.Lock()
-	defer q.lock.Unlock()
-	if q.availability == nil {
-		q.availability = make(map[string][]string)
-	}
-	q.availability[file] = []string{node}
-}
+func (q *FileQueue) SetAvailable(file string, nodes []string) {
+	q.amut.Lock()
+	defer q.amut.Unlock()
 
-func (q *FileQueue) AddAvailable(file, node string) {
-	q.lock.Lock()
-	defer q.lock.Unlock()
-	if q.availability == nil {
-		q.availability = make(map[string][]string)
-	}
-	q.availability[file] = append(q.availability[file], node)
+	q.availability[file] = nodes
 }
 
 func (q *FileQueue) RemoveAvailable(toRemove string) {
-	q.lock.Lock()
-	defer q.lock.Unlock()
+	q.amut.Lock()
+	defer q.amut.Unlock()
+
 	for file, nodes := range q.availability {
 		for i, node := range nodes {
 			if node == toRemove {

model/filequeue_test.go

@@ -8,14 +8,14 @@ import (
 )
 
 func TestFileQueueAdd(t *testing.T) {
-	q := FileQueue{}
+	q := NewFileQueue()
 	q.Add("foo", nil, nil)
 }
 
 func TestFileQueueAddSorting(t *testing.T) {
-	q := FileQueue{}
-	q.SetAvailable("zzz", "nodeID")
-	q.SetAvailable("aaa", "nodeID")
+	q := NewFileQueue()
+	q.SetAvailable("zzz", []string{"nodeID"})
+	q.SetAvailable("aaa", []string{"nodeID"})
 
 	q.Add("zzz", []Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
 	q.Add("aaa", []Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
@@ -24,9 +24,9 @@ func TestFileQueueAddSorting(t *testing.T) {
 		t.Errorf("Incorrectly sorted get: %+v", b)
 	}
 
-	q = FileQueue{}
-	q.SetAvailable("zzz", "nodeID")
-	q.SetAvailable("aaa", "nodeID")
+	q = NewFileQueue()
+	q.SetAvailable("zzz", []string{"nodeID"})
+	q.SetAvailable("aaa", []string{"nodeID"})
 
 	q.Add("zzz", []Block{{Offset: 0, Size: 128}, {Offset: 128, Size: 128}}, nil)
 	b, _ = q.Get("nodeID") // Start on zzzz
@@ -42,7 +42,7 @@ func TestFileQueueAddSorting(t *testing.T) {
 }
 
 func TestFileQueueLen(t *testing.T) {
-	q := FileQueue{}
+	q := NewFileQueue()
 	q.Add("foo", nil, nil)
 	q.Add("bar", nil, nil)
 
@@ -52,9 +52,9 @@ func TestFileQueueLen(t *testing.T) {
 }
 
 func TestFileQueueGet(t *testing.T) {
-	q := FileQueue{}
-	q.SetAvailable("foo", "nodeID")
-	q.SetAvailable("bar", "nodeID")
+	q := NewFileQueue()
+	q.SetAvailable("foo", []string{"nodeID"})
+	q.SetAvailable("bar", []string{"nodeID"})
 
 	q.Add("foo", []Block{
 		{Offset: 0, Size: 128, Hash: []byte("some foo hash bytes")},
@@ -177,11 +177,9 @@ func TestFileQueueDone(t *testing.T) {
 */
 
 func TestFileQueueGetNodeIDs(t *testing.T) {
-	q := FileQueue{}
-	q.SetAvailable("a-foo", "nodeID")
-	q.AddAvailable("a-foo", "a")
-	q.SetAvailable("b-bar", "nodeID")
-	q.AddAvailable("b-bar", "b")
+	q := NewFileQueue()
+	q.SetAvailable("a-foo", []string{"nodeID", "a"})
+	q.SetAvailable("b-bar", []string{"nodeID", "b"})
 
 	q.Add("a-foo", []Block{
 		{Offset: 0, Size: 128, Hash: []byte("some foo hash bytes")},
@@ -254,9 +252,9 @@ func TestFileQueueThreadHandling(t *testing.T) {
 		total += i
 	}
 
-	q := FileQueue{}
+	q := NewFileQueue()
 	q.Add("foo", blocks, nil)
-	q.SetAvailable("foo", "nodeID")
+	q.SetAvailable("foo", []string{"nodeID"})
 
 	var start = make(chan bool)
 	var gotTot uint32

model/model.go

@@ -1,16 +1,5 @@
 package model
 
-/*
-
-Locking
-=======
-
-The model has read and write locks. These must be acquired as appropriate by
-public methods. To prevent deadlock situations, private methods should never
-acquire locks, but document what locks they require.
-
-*/
-
 import (
 	"crypto/sha1"
 	"errors"
@@ -28,33 +17,43 @@ import (
 )
 
 type Model struct {
-	sync.RWMutex
 	dir string
 
 	global    map[string]File // the latest version of each file as it exists in the cluster
+	gmut      sync.RWMutex    // protects global
 	local     map[string]File // the files we currently have locally on disk
+	lmut      sync.RWMutex    // protects local
 	remote    map[string]map[string]File
+	rmut      sync.RWMutex // protects remote
 	protoConn map[string]Connection
 	rawConn   map[string]io.Closer
-	fq        FileQueue // queue for files to fetch
-	dq        chan File // queue for files to delete
+	pmut      sync.RWMutex // protects protoConn and rawConn
 
-	updatedLocal int64 // timestamp of last update to local
-	updateGlobal int64 // timestamp of last update to remote
+	// Queue for files to fetch. fq can call back into the model, so we must ensure
+	// to hold no locks when calling methods on fq.
+	fq *FileQueue
+	dq chan File // queue for files to delete
 
+	updatedLocal        int64 // timestamp of last update to local
+	updateGlobal        int64 // timestamp of last update to remote
 	lastIdxBcast        time.Time
 	lastIdxBcastRequest time.Time
+	umut                sync.RWMutex // provides updated* and lastIdx*
 
 	rwRunning bool
 	delete    bool
+	initmut   sync.Mutex // protects rwRunning and delete
 
 	trace map[string]bool
 
 	fileLastChanged   map[string]time.Time
 	fileWasSuppressed map[string]int
+	fmut              sync.Mutex // protects fileLastChanged and fileWasSuppressed
 
 	parallellRequests int
 	limitRequestRate  chan struct{}
+
+	imut sync.Mutex // protects Index
 }
 
 type Connection interface {
@@ -92,6 +91,7 @@ func NewModel(dir string) *Model {
 		trace:             make(map[string]bool),
 		fileLastChanged:   make(map[string]time.Time),
 		fileWasSuppressed: make(map[string]int),
+		fq:                NewFileQueue(),
 		dq:                make(chan File),
 	}
 
@@ -116,8 +116,6 @@ func (m *Model) LimitRate(kbps int) {
 
 // Trace enables trace logging of the given facility. This is a debugging function; grep for m.trace.
 func (m *Model) Trace(t string) {
-	m.Lock()
-	defer m.Unlock()
 	m.trace[t] = true
 }
 
@@ -125,8 +123,8 @@ func (m *Model) Trace(t string) {
 // read/write mode the model will attempt to keep in sync with the cluster by
 // pulling needed files from peer nodes.
 func (m *Model) StartRW(del bool, threads int) {
-	m.Lock()
-	defer m.Unlock()
+	m.initmut.Lock()
+	defer m.initmut.Unlock()
 
 	if m.rwRunning {
 		panic("starting started model")
@@ -138,19 +136,26 @@ func (m *Model) StartRW(del bool, threads int) {
 
 	go m.cleanTempFiles()
 	if del {
-		go m.deleteFiles()
+		go m.deleteLoop()
 	}
 }
 
 // Generation returns an opaque integer that is guaranteed to increment on
 // every change to the local repository or global model.
 func (m *Model) Generation() int64 {
-	m.RLock()
-	defer m.RUnlock()
+	m.umut.RLock()
+	defer m.umut.RUnlock()
 
 	return m.updatedLocal + m.updateGlobal
 }
 
+func (m *Model) LocalAge() float64 {
+	m.umut.RLock()
+	defer m.umut.RUnlock()
+
+	return time.Since(time.Unix(m.updatedLocal, 0)).Seconds()
+}
+
 type ConnectionInfo struct {
 	protocol.Statistics
 	Address string
@@ -162,8 +167,7 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
 		RemoteAddr() net.Addr
 	}
 
-	m.RLock()
-	defer m.RUnlock()
+	m.pmut.RLock()
 
 	var res = make(map[string]ConnectionInfo)
 	for node, conn := range m.protoConn {
@@ -175,14 +179,15 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
 		}
 		res[node] = ci
 	}
+
+	m.pmut.RUnlock()
 	return res
 }
 
 // LocalSize returns the number of files, deleted files and total bytes for all
 // files in the global model.
 func (m *Model) GlobalSize() (files, deleted, bytes int) {
-	m.RLock()
-	defer m.RUnlock()
+	m.gmut.RLock()
 
 	for _, f := range m.global {
 		if f.Flags&protocol.FlagDeleted == 0 {
@@ -192,14 +197,15 @@ func (m *Model) GlobalSize() (files, deleted, bytes int) {
 			deleted++
 		}
 	}
+
+	m.gmut.RUnlock()
 	return
 }
 
 // LocalSize returns the number of files, deleted files and total bytes for all
 // files in the local repository.
 func (m *Model) LocalSize() (files, deleted, bytes int) {
-	m.RLock()
-	defer m.RUnlock()
+	m.lmut.RLock()
 
 	for _, f := range m.local {
 		if f.Flags&protocol.FlagDeleted == 0 {
@@ -209,14 +215,16 @@ func (m *Model) LocalSize() (files, deleted, bytes int) {
 			deleted++
 		}
 	}
+
+	m.lmut.RUnlock()
 	return
 }
 
 // InSyncSize returns the number and total byte size of the local files that
 // are in sync with the global model.
 func (m *Model) InSyncSize() (files, bytes int) {
-	m.RLock()
-	defer m.RUnlock()
+	m.gmut.RLock()
+	m.lmut.RLock()
 
 	for n, f := range m.local {
 		if gf, ok := m.global[n]; ok && f.Equals(gf) {
@@ -224,27 +232,33 @@ func (m *Model) InSyncSize() (files, bytes int) {
 			bytes += f.Size()
 		}
 	}
+
+	m.lmut.RUnlock()
+	m.gmut.RUnlock()
 	return
 }
 
 // NeedFiles returns the list of currently needed files and the total size.
 func (m *Model) NeedFiles() (files []File, bytes int) {
-	m.RLock()
-	defer m.RUnlock()
+	qf := m.fq.QueuedFiles()
 
-	for _, n := range m.fq.QueuedFiles() {
+	m.gmut.RLock()
+
+	for _, n := range qf {
 		f := m.global[n]
 		files = append(files, f)
 		bytes += f.Size()
 	}
+
+	m.gmut.RUnlock()
 	return
 }
 
 // Index is called when a new node is connected and we receive their full index.
 // Implements the protocol.Model interface.
 func (m *Model) Index(nodeID string, fs []protocol.FileInfo) {
-	m.Lock()
-	defer m.Unlock()
+	m.imut.Lock()
+	defer m.imut.Unlock()
 
 	if m.trace["net"] {
 		log.Printf("NET IDX(in): %s: %d files", nodeID, len(fs))
@@ -254,7 +268,10 @@ func (m *Model) Index(nodeID string, fs []protocol.FileInfo) {
 	for _, f := range fs {
 		m.indexUpdate(repo, f)
 	}
+
+	m.rmut.Lock()
 	m.remote[nodeID] = repo
+	m.rmut.Unlock()
 
 	m.recomputeGlobal()
 	m.recomputeNeed()
@@ -263,22 +280,25 @@ func (m *Model) Index(nodeID string, fs []protocol.FileInfo) {
 // IndexUpdate is called for incremental updates to connected nodes' indexes.
 // Implements the protocol.Model interface.
 func (m *Model) IndexUpdate(nodeID string, fs []protocol.FileInfo) {
-	m.Lock()
-	defer m.Unlock()
+	m.imut.Lock()
+	defer m.imut.Unlock()
 
 	if m.trace["net"] {
 		log.Printf("NET IDXUP(in): %s: %d files", nodeID, len(fs))
 	}
 
+	m.rmut.Lock()
 	repo, ok := m.remote[nodeID]
 	if !ok {
 		log.Printf("WARNING: Index update from node %s that does not have an index", nodeID)
+		m.rmut.Unlock()
 		return
 	}
 
 	for _, f := range fs {
 		m.indexUpdate(repo, f)
 	}
+	m.rmut.Unlock()
 
 	m.recomputeGlobal()
 	m.recomputeNeed()
@@ -301,11 +321,13 @@ func (m *Model) indexUpdate(repo map[string]File, f protocol.FileInfo) {
 	repo[f.Name] = fileFromFileInfo(f)
 }
 
-// Close removes the peer from the model and closes the underlyign connection if possible.
+// Close removes the peer from the model and closes the underlying connection if possible.
 // Implements the protocol.Model interface.
 func (m *Model) Close(node string, err error) {
-	m.Lock()
-	defer m.Unlock()
+	m.fq.RemoveAvailable(node)
+
+	m.pmut.Lock()
+	m.rmut.Lock()
 
 	conn, ok := m.rawConn[node]
 	if ok {
@@ -315,7 +337,9 @@ func (m *Model) Close(node string, err error) {
 	delete(m.remote, node)
 	delete(m.protoConn, node)
 	delete(m.rawConn, node)
-	m.fq.RemoveAvailable(node)
+
+	m.rmut.Unlock()
+	m.pmut.Unlock()
 
 	m.recomputeGlobal()
 	m.recomputeNeed()
@@ -325,10 +349,14 @@ func (m *Model) Close(node string, err error) {
 // Implements the protocol.Model interface.
 func (m *Model) Request(nodeID, name string, offset int64, size uint32, hash []byte) ([]byte, error) {
 	// Verify that the requested file exists in the local and global model.
-	m.RLock()
+	m.lmut.RLock()
 	lf, localOk := m.local[name]
+	m.lmut.RUnlock()
+
+	m.gmut.RLock()
 	_, globalOk := m.global[name]
-	m.RUnlock()
+	m.gmut.RUnlock()
+
 	if !localOk || !globalOk {
 		log.Printf("SECURITY (nonexistent file) REQ(in): %s: %q o=%d s=%d h=%x", nodeID, name, offset, size, hash)
 		return nil, ErrNoSuchFile
@@ -365,33 +393,40 @@ func (m *Model) Request(nodeID, name string, offset int64, size uint32, hash []b
 // ReplaceLocal replaces the local repository index with the given list of files.
 // Change suppression is applied to files changing too often.
 func (m *Model) ReplaceLocal(fs []File) {
-	m.Lock()
-	defer m.Unlock()
-
 	var updated bool
 	var newLocal = make(map[string]File)
 
+	m.lmut.RLock()
 	for _, f := range fs {
 		newLocal[f.Name] = f
 		if ef := m.local[f.Name]; !ef.Equals(f) {
 			updated = true
 		}
 	}
+	m.lmut.RUnlock()
 
 	if m.markDeletedLocals(newLocal) {
 		updated = true
 	}
 
+	m.lmut.RLock()
 	if len(newLocal) != len(m.local) {
 		updated = true
 	}
+	m.lmut.RUnlock()
 
 	if updated {
+		m.lmut.Lock()
 		m.local = newLocal
+		m.lmut.Unlock()
+
 		m.recomputeGlobal()
 		m.recomputeNeed()
+
+		m.umut.Lock()
 		m.updatedLocal = time.Now().Unix()
 		m.lastIdxBcastRequest = time.Now()
+		m.umut.Unlock()
 	}
 }
 
@@ -399,13 +434,12 @@ func (m *Model) ReplaceLocal(fs []File) {
 // in protocol data types. Does not track deletes, should only be used to seed
 // the local index from a cache file at startup.
 func (m *Model) SeedLocal(fs []protocol.FileInfo) {
-	m.Lock()
-	defer m.Unlock()
-
+	m.lmut.Lock()
 	m.local = make(map[string]File)
 	for _, f := range fs {
 		m.local[f.Name] = fileFromFileInfo(f)
 	}
+	m.lmut.Unlock()
 
 	m.recomputeGlobal()
 	m.recomputeNeed()
@@ -413,19 +447,12 @@ func (m *Model) SeedLocal(fs []protocol.FileInfo) {
 
 // ConnectedTo returns true if we are connected to the named node.
 func (m *Model) ConnectedTo(nodeID string) bool {
-	m.RLock()
-	defer m.RUnlock()
+	m.pmut.RLock()
 	_, ok := m.protoConn[nodeID]
+	m.pmut.RUnlock()
 	return ok
 }
 
-// ProtocolIndex returns the current local index in protocol data types.
-func (m *Model) ProtocolIndex() []protocol.FileInfo {
-	m.RLock()
-	defer m.RUnlock()
-	return m.protocolIndex()
-}
-
 // RepoID returns a unique ID representing the current repository location.
 func (m *Model) RepoID() string {
 	return fmt.Sprintf("%x", sha1.Sum([]byte(m.dir)))
@@ -436,54 +463,57 @@ func (m *Model) RepoID() string {
 // repository changes.
 func (m *Model) AddConnection(rawConn io.Closer, protoConn Connection) {
 	nodeID := protoConn.ID()
-	m.Lock()
+	m.pmut.Lock()
 	m.protoConn[nodeID] = protoConn
 	m.rawConn[nodeID] = rawConn
-	m.Unlock()
-
-	m.RLock()
-	idx := m.protocolIndex()
-	m.RUnlock()
+	m.pmut.Unlock()
 
 	go func() {
+		idx := m.ProtocolIndex()
 		protoConn.Index(idx)
 	}()
 
-	if m.rwRunning {
-		for i := 0; i < m.parallellRequests; i++ {
-			i := i
-			go func() {
-				if m.trace["pull"] {
-					log.Println("PULL: Starting", nodeID, i)
-				}
-				for {
-					m.RLock()
-					if _, ok := m.protoConn[nodeID]; !ok {
-						if m.trace["pull"] {
-							log.Println("PULL: Exiting", nodeID, i)
-						}
-						m.RUnlock()
-						return
-					}
-					m.RUnlock()
+	m.initmut.Lock()
+	rw := m.rwRunning
+	m.initmut.Unlock()
+	if !rw {
+		return
+	}
 
-					qb, ok := m.fq.Get(nodeID)
-					if ok {
-						if m.trace["pull"] {
-							log.Println("PULL: Request", nodeID, i, qb.name, qb.block.Offset)
-						}
-						data, _ := protoConn.Request(qb.name, qb.block.Offset, qb.block.Size, qb.block.Hash)
-						m.fq.Done(qb.name, qb.block.Offset, data)
-					} else {
-						time.Sleep(1 * time.Second)
+	for i := 0; i < m.parallellRequests; i++ {
+		i := i
+		go func() {
+			if m.trace["pull"] {
+				log.Println("PULL: Starting", nodeID, i)
+			}
+			for {
+				m.pmut.RLock()
+				if _, ok := m.protoConn[nodeID]; !ok {
+					if m.trace["pull"] {
+						log.Println("PULL: Exiting", nodeID, i)
 					}
+					m.pmut.RUnlock()
+					return
 				}
-			}()
-		}
+				m.pmut.RUnlock()
+
+				qb, ok := m.fq.Get(nodeID)
+				if ok {
+					if m.trace["pull"] {
+						log.Println("PULL: Request", nodeID, i, qb.name, qb.block.Offset)
+					}
+					data, _ := protoConn.Request(qb.name, qb.block.Offset, qb.block.Size, qb.block.Hash)
+					m.fq.Done(qb.name, qb.block.Offset, data)
+				} else {
+					time.Sleep(1 * time.Second)
+				}
+			}
+		}()
 	}
 }
 
 func (m *Model) shouldSuppressChange(name string) bool {
+	m.fmut.Lock()
 	sup := shouldSuppressChange(m.fileLastChanged[name], m.fileWasSuppressed[name])
 	if sup {
 		m.fileWasSuppressed[name]++
@@ -491,6 +521,7 @@ func (m *Model) shouldSuppressChange(name string) bool {
 		m.fileWasSuppressed[name] = 0
 		m.fileLastChanged[name] = time.Now()
 	}
+	m.fmut.Unlock()
 	return sup
 }
 
@@ -505,10 +536,13 @@ func shouldSuppressChange(lastChange time.Time, numChanges int) bool {
 	return false
 }
 
-// protocolIndex returns the current local index in protocol data types.
+// ProtocolIndex returns the current local index in protocol data types.
 // Must be called with the read lock held.
-func (m *Model) protocolIndex() []protocol.FileInfo {
+func (m *Model) ProtocolIndex() []protocol.FileInfo {
 	var index []protocol.FileInfo
+
+	m.lmut.RLock()
+
 	for _, f := range m.local {
 		mf := fileInfoFromFile(f)
 		if m.trace["idx"] {
@@ -520,13 +554,16 @@ func (m *Model) protocolIndex() []protocol.FileInfo {
 		}
 		index = append(index, mf)
 	}
+
+	m.lmut.RUnlock()
 	return index
 }
 
 func (m *Model) requestGlobal(nodeID, name string, offset int64, size uint32, hash []byte) ([]byte, error) {
-	m.RLock()
+	m.pmut.RLock()
 	nc, ok := m.protoConn[nodeID]
-	m.RUnlock()
+	m.pmut.RUnlock()
+
 	if !ok {
 		return nil, fmt.Errorf("requestGlobal: no such node: %s", nodeID)
 	}
@@ -540,18 +577,23 @@ func (m *Model) requestGlobal(nodeID, name string, offset int64, size uint32, ha
 
 func (m *Model) broadcastIndexLoop() {
 	for {
-		m.RLock()
+		m.umut.RLock()
 		bcastRequested := m.lastIdxBcastRequest.After(m.lastIdxBcast)
 		holdtimeExceeded := time.Since(m.lastIdxBcastRequest) > idxBcastHoldtime
-		m.RUnlock()
+		m.umut.RUnlock()
 
 		maxDelayExceeded := time.Since(m.lastIdxBcast) > idxBcastMaxDelay
 		if bcastRequested && (holdtimeExceeded || maxDelayExceeded) {
-			m.Lock()
+			idx := m.ProtocolIndex()
+
 			var indexWg sync.WaitGroup
 			indexWg.Add(len(m.protoConn))
-			idx := m.protocolIndex()
+
+			m.umut.Lock()
 			m.lastIdxBcast = time.Now()
+			m.umut.Unlock()
+
+			m.pmut.RLock()
 			for _, node := range m.protoConn {
 				node := node
 				if m.trace["net"] {
@@ -562,7 +604,8 @@ func (m *Model) broadcastIndexLoop() {
 					indexWg.Done()
 				}()
 			}
-			m.Unlock()
+			m.pmut.RUnlock()
+
 			indexWg.Wait()
 		}
 		time.Sleep(idxBcastHoldtime)
@@ -570,13 +613,16 @@ func (m *Model) broadcastIndexLoop() {
 }
 
 // markDeletedLocals sets the deleted flag on files that have gone missing locally.
-// Must be called with the write lock held.
 func (m *Model) markDeletedLocals(newLocal map[string]File) bool {
 	// For every file in the existing local table, check if they are also
 	// present in the new local table. If they are not, check that we already
 	// had the newest version available according to the global table and if so
 	// note the file as having been deleted.
 	var updated bool
+
+	m.gmut.RLock()
+	m.lmut.RLock()
+
 	for n, f := range m.local {
 		if _, ok := newLocal[n]; !ok {
 			if gf := m.global[n]; !gf.NewerThan(f) {
@@ -590,50 +636,72 @@ func (m *Model) markDeletedLocals(newLocal map[string]File) bool {
 			}
 		}
 	}
+
+	m.lmut.RUnlock()
+	m.gmut.RUnlock()
+
 	return updated
 }
 
-func (m *Model) updateLocalLocked(f File) {
-	m.Lock()
-	m.updateLocal(f)
-	m.Unlock()
-}
-
 func (m *Model) updateLocal(f File) {
+	var updated bool
+
+	m.lmut.Lock()
 	if ef, ok := m.local[f.Name]; !ok || !ef.Equals(f) {
 		m.local[f.Name] = f
+		updated = true
+	}
+	m.lmut.Unlock()
+
+	if updated {
 		m.recomputeGlobal()
-		m.recomputeNeed()
+		// We don't recomputeNeed here for two reasons:
+		// - a need shouldn't have arisen due to having a newer local file
+		// - recomputeNeed might call into fq.Add but we might have been called by
+		//   fq which would be a deadlock on fq
+
+		m.umut.Lock()
 		m.updatedLocal = time.Now().Unix()
 		m.lastIdxBcastRequest = time.Now()
+		m.umut.Unlock()
 	}
 }
 
-// Must be called with the write lock held.
 func (m *Model) recomputeGlobal() {
 	var newGlobal = make(map[string]File)
 
+	m.lmut.RLock()
 	for n, f := range m.local {
 		newGlobal[n] = f
 	}
+	m.lmut.RUnlock()
 
+	var available = make(map[string][]string)
+
+	m.rmut.RLock()
 	var highestMod int64
 	for nodeID, fs := range m.remote {
 		for n, nf := range fs {
 			if lf, ok := newGlobal[n]; !ok || nf.NewerThan(lf) {
 				newGlobal[n] = nf
-				m.fq.SetAvailable(n, nodeID)
+				available[n] = []string{nodeID}
 				if nf.Modified > highestMod {
 					highestMod = nf.Modified
 				}
 			} else if lf.Equals(nf) {
-				m.fq.AddAvailable(n, nodeID)
+				available[n] = append(available[n], nodeID)
 			}
 		}
 	}
+	m.rmut.RUnlock()
+
+	for f, ns := range available {
+		m.fq.SetAvailable(f, ns)
+	}
 
 	// Figure out if anything actually changed
 
+	m.gmut.RLock()
 	var updated bool
 	if highestMod > m.updateGlobal || len(newGlobal) != len(m.global) {
 		updated = true
@@ -645,20 +713,35 @@ func (m *Model) recomputeGlobal() {
 			}
 		}
 	}
+	m.gmut.RUnlock()
 
 	if updated {
-		m.updateGlobal = time.Now().Unix()
+		m.gmut.Lock()
+		m.umut.Lock()
 		m.global = newGlobal
+		m.updateGlobal = time.Now().Unix()
+		m.umut.Unlock()
+		m.gmut.Unlock()
 	}
 }
 
-// Must be called with the write lock held.
 func (m *Model) recomputeNeed() {
+	type addOrder struct {
+		n      string
+		remote []Block
+		fm     *fileMonitor
+	}
+
+	var toDelete []File
+	var toAdd []addOrder
+
+	m.gmut.RLock()
+
 	for n, gf := range m.global {
-		if m.fq.Queued(n) {
-			continue
-		}
+		m.lmut.RLock()
 		lf, ok := m.local[n]
+		m.lmut.RUnlock()
+
 		if !ok || gf.NewerThan(lf) {
 			if gf.Flags&protocol.FlagInvalid != 0 {
 				// Never attempt to sync invalid files
@@ -677,7 +760,7 @@ func (m *Model) recomputeNeed() {
 			}
 
 			if gf.Flags&protocol.FlagDeleted != 0 {
-				m.dq <- gf
+				toDelete = append(toDelete, gf)
 			} else {
 				local, remote := BlockDiff(lf.Blocks, gf.Blocks)
 				fm := fileMonitor{
@@ -687,22 +770,29 @@ func (m *Model) recomputeNeed() {
 					model:       m,
 					localBlocks: local,
 				}
-				m.fq.Add(n, remote, &fm)
+				toAdd = append(toAdd, addOrder{n, remote, &fm})
 			}
 		}
 	}
+
+	m.gmut.RUnlock()
+
+	for _, ao := range toAdd {
+		if !m.fq.Queued(ao.n) {
+			m.fq.Add(ao.n, ao.remote, ao.fm)
+		}
+	}
+	for _, gf := range toDelete {
+		m.dq <- gf
+	}
 }
 
 func (m *Model) WhoHas(name string) []string {
-	m.RLock()
-	defer m.RUnlock()
-	return m.whoHas(name)
-}
-
-// Must be called with the read lock held.
-func (m *Model) whoHas(name string) []string {
 	var remote []string
 
+	m.gmut.RLock()
+	m.rmut.RLock()
+
 	gf := m.global[name]
 	for node, files := range m.remote {
 		if file, ok := files[name]; ok && file.Equals(gf) {
@@ -710,10 +800,12 @@ func (m *Model) whoHas(name string) []string {
 		}
 	}
 
+	m.rmut.RUnlock()
+	m.gmut.RUnlock()
 	return remote
 }
 
-func (m *Model) deleteFiles() {
+func (m *Model) deleteLoop() {
 	for file := range m.dq {
 		if m.trace["file"] {
 			log.Println("FILE: Delete", file.Name)
@@ -723,7 +815,8 @@ func (m *Model) deleteFiles() {
 		if err != nil {
 			log.Printf("WARNING: %s: %v", file.Name, err)
 		}
-		m.updateLocalLocked(file)
+
+		m.updateLocal(file)
 	}
 }
 

model/walk.go

@@ -85,6 +85,9 @@ func (m *Model) loadIgnoreFiles(ign map[string][]string) filepath.WalkFunc {
 func (m *Model) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.WalkFunc {
 	return func(p string, info os.FileInfo, err error) error {
 		if err != nil {
+			if m.trace["file"] {
+				log.Printf("FILE: %q: %v", p, err)
+			}
 			return nil
 		}
 
@@ -110,38 +113,31 @@ func (m *Model) walkAndHashFiles(res *[]File, ign map[string][]string) filepath.
 		}
 
 		if info.Mode()&os.ModeType == 0 {
-			fi, err := os.Stat(p)
-			if err != nil {
-				return nil
-			}
-			modified := fi.ModTime().Unix()
+			modified := info.ModTime().Unix()
 
-			m.RLock()
-			hf, ok := m.local[rn]
-			m.RUnlock()
+			m.lmut.RLock()
+			lf, ok := m.local[rn]
+			m.lmut.RUnlock()
 
-			if ok && hf.Modified == modified {
-				if nf := uint32(info.Mode()); nf != hf.Flags {
-					hf.Flags = nf
-					hf.Version++
+			if ok && lf.Modified == modified {
+				if nf := uint32(info.Mode()); nf != lf.Flags {
+					lf.Flags = nf
+					lf.Version++
 				}
-				*res = append(*res, hf)
+				*res = append(*res, lf)
 			} else {
-				m.Lock()
 				if m.shouldSuppressChange(rn) {
 					if m.trace["file"] {
 						log.Println("FILE: SUPPRESS:", rn, m.fileWasSuppressed[rn], time.Since(m.fileLastChanged[rn]))
 					}
 
 					if ok {
-						hf.Flags = protocol.FlagInvalid
-						hf.Version++
-						*res = append(*res, hf)
+						lf.Flags = protocol.FlagInvalid
+						lf.Version++
+						*res = append(*res, lf)
 					}
-					m.Unlock()
 					return nil
 				}
-				m.Unlock()
 
 				if m.trace["file"] {
 					log.Printf("FILE: Hash %q", p)
@@ -198,9 +194,9 @@ func (m *Model) Walk(followSymlinks bool) (files []File, ignore map[string][]str
 			return
 		}
 
-		for _, fi := range fis {
-			if fi.Mode()&os.ModeSymlink != 0 {
-				dir := path.Join(m.dir, fi.Name()) + "/"
+		for _, info := range fis {
+			if info.Mode()&os.ModeSymlink != 0 {
+				dir := path.Join(m.dir, info.Name()) + "/"
 				filepath.Walk(dir, m.loadIgnoreFiles(ignore))
 				filepath.Walk(dir, hashFiles)
 			}

protocol/protocol.go

@@ -341,14 +341,15 @@ func (c *Connection) processRequest(msgID int, req request) {
 	c.Lock()
 	c.mwriter.writeUint32(encodeHeader(header{0, msgID, messageTypeResponse}))
 	c.mwriter.writeResponse(data)
-	err := c.flush()
+	err := c.mwriter.err
+	if err == nil {
+		err = c.flush()
+	}
 	c.Unlock()
 
 	buffers.Put(data)
 	if err != nil {
 		c.close(err)
-	} else if c.mwriter.err != nil {
-		c.close(c.mwriter.err)
 	}
 }