Translation update

UPnP API Additions to address outstanding parts of #432

CONTRIBUTORS

@@ -5,6 +5,11 @@ Audrius Butkevicius <audrius.butkevicius@gmail.com>
 Arthur Axel fREW Schmidt <frew@afoolishmanifesto.com> <frioux@gmail.com>
 Ben Sidhom <bsidhom@gmail.com>
 Brandon Philips <brandon@ifup.org>
+Caleb Callaway <enlightened.despot@gmail.com>
+Chris Joel <chris@scriptolo.gy>
+Daniel Martí <mvdan@mvdan.cc>
+Felix Ableitner <me@nutomic.com>
+Felix Unterpaintner <bigbear2nd@gmail.com>
 Gilli Sigurdsson <gilli@vx.is>
 James Patterson <jamespatterson@operamail.com> <jpjp@users.noreply.github.com>
 Jens Diemer <github.com@jensdiemer.de> <git@jensdiemer.de>
@@ -13,6 +18,7 @@ Lode Hoste <zillode@zillode.be>
 Marcin Dziadus <dziadus.marcin@gmail.com>
 Michael Tilli <pyfisch@gmail.com>
 Philippe Schommers <philippe@schommers.be>
+Phill Luby <phill.luby@newredo.com>
 Ryan Sullivan <kayoticsully@gmail.com>
 Tully Robinson <tully@tojr.org>
 Veeti Paananen <veeti.paananen@rojekti.fi>

Godeps/Godeps.json

@@ -30,6 +30,10 @@
 			"Comment": "null-15",
 			"Rev": "12e4b4183793ac4b061921e7980845e750679fd0"
 		},
+		{
+			"ImportPath": "github.com/AudriusButkevicius/lfu-go",
+			"Rev": "164bcecceb92fd6037f4d18a8d97b495ec6ef669"
+		},
 		{
 			"ImportPath": "github.com/bkaradzic/go-lz4",
 			"Rev": "93a831dcee242be64a9cc9803dda84af25932de7"

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2012 Dave Grijalva
+
+Permission is hereby granted, free of charge, to any person obtaining a 
+copy of this software and associated documentation files (the "Software"), 
+to deal in the Software without restriction, including without limitation 
+the rights to use, copy, modify, merge, publish, distribute, sublicense, 
+and/or sell copies of the Software, and to permit persons to whom the 
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included 
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 
+THE SOFTWARE.

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/README.md

@@ -0,0 +1,19 @@
+A simple LFU cache for golang.  Based on the paper [An O(1) algorithm for implementing the LFU cache eviction scheme](http://dhruvbird.com/lfu.pdf).
+
+Usage:
+
+```go
+import "github.com/dgrijalva/lfu-go"
+
+// Make a new thing
+c := lfu.New()
+
+// Set some values
+c.Set("myKey", myValue)
+
+// Retrieve some values
+myValue = c.Get("myKey")
+
+// Evict some values
+c.Evict(1)
+```

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/lfu.go

@@ -0,0 +1,156 @@
+package lfu
+
+import (
+	"container/list"
+	"sync"
+)
+
+type Eviction struct {
+	Key   string
+	Value interface{}
+}
+
+type Cache struct {
+	// If len > UpperBound, cache will automatically evict
+	// down to LowerBound.  If either value is 0, this behavior
+	// is disabled.
+	UpperBound      int
+	LowerBound      int
+	values          map[string]*cacheEntry
+	freqs           *list.List
+	len             int
+	lock            *sync.Mutex
+	EvictionChannel chan<- Eviction
+}
+
+type cacheEntry struct {
+	key      string
+	value    interface{}
+	freqNode *list.Element
+}
+
+type listEntry struct {
+	entries map[*cacheEntry]byte
+	freq    int
+}
+
+func New() *Cache {
+	c := new(Cache)
+	c.values = make(map[string]*cacheEntry)
+	c.freqs = list.New()
+	c.lock = new(sync.Mutex)
+	return c
+}
+
+func (c *Cache) Get(key string) interface{} {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	if e, ok := c.values[key]; ok {
+		c.increment(e)
+		return e.value
+	}
+	return nil
+}
+
+func (c *Cache) Set(key string, value interface{}) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	if e, ok := c.values[key]; ok {
+		// value already exists for key.  overwrite
+		e.value = value
+		c.increment(e)
+	} else {
+		// value doesn't exist.  insert
+		e := new(cacheEntry)
+		e.key = key
+		e.value = value
+		c.values[key] = e
+		c.increment(e)
+		c.len++
+		// bounds mgmt
+		if c.UpperBound > 0 && c.LowerBound > 0 {
+			if c.len > c.UpperBound {
+				c.evict(c.len - c.LowerBound)
+			}
+		}
+	}
+}
+
+func (c *Cache) Len() int {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return c.len
+}
+
+func (c *Cache) Evict(count int) int {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return c.evict(count)
+}
+
+func (c *Cache) evict(count int) int {
+	// No lock here so it can be called
+	// from within the lock (during Set)
+	var evicted int
+	for i := 0; i < count; {
+		if place := c.freqs.Front(); place != nil {
+			for entry, _ := range place.Value.(*listEntry).entries {
+				if i < count {
+					if c.EvictionChannel != nil {
+						c.EvictionChannel <- Eviction{
+							Key:   entry.key,
+							Value: entry.value,
+						}
+					}
+					delete(c.values, entry.key)
+					c.remEntry(place, entry)
+					evicted++
+					c.len--
+					i++
+				}
+			}
+		}
+	}
+	return evicted
+}
+
+func (c *Cache) increment(e *cacheEntry) {
+	currentPlace := e.freqNode
+	var nextFreq int
+	var nextPlace *list.Element
+	if currentPlace == nil {
+		// new entry
+		nextFreq = 1
+		nextPlace = c.freqs.Front()
+	} else {
+		// move up
+		nextFreq = currentPlace.Value.(*listEntry).freq + 1
+		nextPlace = currentPlace.Next()
+	}
+
+	if nextPlace == nil || nextPlace.Value.(*listEntry).freq != nextFreq {
+		// create a new list entry
+		li := new(listEntry)
+		li.freq = nextFreq
+		li.entries = make(map[*cacheEntry]byte)
+		if currentPlace != nil {
+			nextPlace = c.freqs.InsertAfter(li, currentPlace)
+		} else {
+			nextPlace = c.freqs.PushFront(li)
+		}
+	}
+	e.freqNode = nextPlace
+	nextPlace.Value.(*listEntry).entries[e] = 1
+	if currentPlace != nil {
+		// remove from current position
+		c.remEntry(currentPlace, e)
+	}
+}
+
+func (c *Cache) remEntry(place *list.Element, entry *cacheEntry) {
+	entries := place.Value.(*listEntry).entries
+	delete(entries, entry)
+	if len(entries) == 0 {
+		c.freqs.Remove(place)
+	}
+}

Godeps/_workspace/src/github.com/AudriusButkevicius/lfu-go/lfu_test.go

@@ -0,0 +1,68 @@
+package lfu
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestLFU(t *testing.T) {
+	c := New()
+	c.Set("a", "a")
+	if v := c.Get("a"); v != "a" {
+		t.Errorf("Value was not saved: %v != 'a'", v)
+	}
+	if l := c.Len(); l != 1 {
+		t.Errorf("Length was not updated: %v != 1", l)
+	}
+
+	c.Set("b", "b")
+	if v := c.Get("b"); v != "b" {
+		t.Errorf("Value was not saved: %v != 'b'", v)
+	}
+	if l := c.Len(); l != 2 {
+		t.Errorf("Length was not updated: %v != 2", l)
+	}
+
+	c.Get("a")
+	evicted := c.Evict(1)
+	if v := c.Get("a"); v != "a" {
+		t.Errorf("Value was improperly evicted: %v != 'a'", v)
+	}
+	if v := c.Get("b"); v != nil {
+		t.Errorf("Value was not evicted: %v", v)
+	}
+	if l := c.Len(); l != 1 {
+		t.Errorf("Length was not updated: %v != 1", l)
+	}
+	if evicted != 1 {
+		t.Errorf("Number of evicted items is wrong: %v != 1", evicted)
+	}
+}
+
+func TestBoundsMgmt(t *testing.T) {
+	c := New()
+	c.UpperBound = 10
+	c.LowerBound = 5
+
+	for i := 0; i < 100; i++ {
+		c.Set(fmt.Sprintf("%v", i), i)
+	}
+	if c.Len() > 10 {
+		t.Errorf("Bounds management failed to evict properly: %v", c.Len())
+	}
+}
+
+func TestEviction(t *testing.T) {
+	ch := make(chan Eviction, 1)
+
+	c := New()
+	c.EvictionChannel = ch
+	c.Set("a", "b")
+	c.Evict(1)
+
+	ev := <-ch
+
+	if ev.Key != "a" || ev.Value.(string) != "b" {
+		t.Error("Incorrect item")
+	}
+}

build.go

@@ -35,6 +35,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"time"
 )
 
 var (
@@ -42,6 +43,7 @@ var (
 	goarch    string
 	goos      string
 	noupgrade bool
+	version   string
 )
 
 const minGoVersion = 1.3
@@ -64,6 +66,7 @@ func main() {
 	flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
 	flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
 	flag.BoolVar(&noupgrade, "no-upgrade", false, "Disable upgrade functionality")
+	flag.StringVar(&version, "version", getVersion(), "Set compiled in version string")
 	flag.Parse()
 
 	switch goarch {
@@ -280,7 +283,7 @@ func clean() {
 func ldflags() string {
 	var b bytes.Buffer
 	b.WriteString("-w")
-	b.WriteString(fmt.Sprintf(" -X main.Version %s", version()))
+	b.WriteString(fmt.Sprintf(" -X main.Version %s", version))
 	b.WriteString(fmt.Sprintf(" -X main.BuildStamp %d", buildStamp()))
 	b.WriteString(fmt.Sprintf(" -X main.BuildUser %s", buildUser()))
 	b.WriteString(fmt.Sprintf(" -X main.BuildHost %s", buildHost()))
@@ -298,8 +301,11 @@ func rmr(paths ...string) {
 	}
 }
 
-func version() string {
-	v := run("git", "describe", "--always", "--dirty")
+func getVersion() string {
+	v, err := runError("git", "describe", "--always", "--dirty")
+	if err != nil {
+		return "unknown-dev"
+	}
 	v = versionRe.ReplaceAllFunc(v, func(s []byte) []byte {
 		s[0] = '+'
 		return s
@@ -308,7 +314,10 @@ func version() string {
 }
 
 func buildStamp() int64 {
-	bs := run("git", "show", "-s", "--format=%ct")
+	bs, err := runError("git", "show", "-s", "--format=%ct")
+	if err != nil {
+		return time.Now().Unix()
+	}
 	s, _ := strconv.ParseInt(string(bs), 10, 64)
 	return s
 }
@@ -345,12 +354,11 @@ func buildArch() string {
 }
 
 func archiveName() string {
-	return fmt.Sprintf("syncthing-%s-%s", buildArch(), version())
+	return fmt.Sprintf("syncthing-%s-%s", buildArch(), version)
 }
 
 func run(cmd string, args ...string) []byte {
-	ecmd := exec.Command(cmd, args...)
-	bs, err := ecmd.CombinedOutput()
+	bs, err := runError(cmd, args...)
 	if err != nil {
 		log.Println(cmd, strings.Join(args, " "))
 		log.Println(string(bs))
@@ -359,6 +367,15 @@ func run(cmd string, args ...string) []byte {
 	return bytes.TrimSpace(bs)
 }
 
+func runError(cmd string, args ...string) ([]byte, error) {
+	ecmd := exec.Command(cmd, args...)
+	bs, err := ecmd.CombinedOutput()
+	if err != nil {
+		return nil, err
+	}
+	return bytes.TrimSpace(bs), nil
+}
+
 func runPrint(cmd string, args ...string) {
 	log.Println(cmd, strings.Join(args, " "))
 	ecmd := exec.Command(cmd, args...)

cmd/syncthing/gui.go

@@ -34,6 +34,7 @@ import (
 	"code.google.com/p/go.crypto/bcrypt"
 	"github.com/syncthing/syncthing/internal/auto"
 	"github.com/syncthing/syncthing/internal/config"
+	"github.com/syncthing/syncthing/internal/discover"
 	"github.com/syncthing/syncthing/internal/events"
 	"github.com/syncthing/syncthing/internal/logger"
 	"github.com/syncthing/syncthing/internal/model"
@@ -154,8 +155,13 @@ func startGUI(cfg config.GUIConfiguration, assetDir string, m *model.Model) erro
 		handler = redirectToHTTPSMiddleware(handler)
 	}
 
+	srv := http.Server{
+		Handler:     handler,
+		ReadTimeout: 2 * time.Second,
+	}
+
 	go func() {
-		err := http.Serve(listener, handler)
+		err := srv.Serve(listener)
 		if err != nil {
 			panic(err)
 		}
@@ -446,7 +452,17 @@ func restPostDiscoveryHint(w http.ResponseWriter, r *http.Request) {
 }
 
 func restGetDiscovery(w http.ResponseWriter, r *http.Request) {
-	json.NewEncoder(w).Encode(discoverer.All())
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	registry := discoverer.All()
+
+	// Device ids can't be marshalled as keys so we need to manually
+	// rebuild this map using strings.
+	devices := make(map[string][]discover.CacheEntry, len(registry))
+	for device, _ := range registry {
+		devices[device.String()] = registry[device]
+	}
+
+	json.NewEncoder(w).Encode(devices)
 }
 
 func restGetReport(m *model.Model, w http.ResponseWriter, r *http.Request) {

cmd/syncthing/main.go

@@ -103,6 +103,7 @@ var (
 	stop           = make(chan int)
 	discoverer     *discover.Discoverer
 	externalPort   int
+	igd            *upnp.IGD
 	cert           tls.Certificate
 )
 
@@ -193,7 +194,7 @@ func main() {
 	if err != nil {
 		l.Fatalln("home:", err)
 	}
-	flag.StringVar(&generateDir, "generate", "", "Generate key in specified dir, then exit")
+	flag.StringVar(&generateDir, "generate", "", "Generate key and config in specified dir, then exit")
 	flag.StringVar(&guiAddress, "gui-address", guiAddress, "Override GUI address")
 	flag.StringVar(&guiAuthentication, "gui-authentication", guiAuthentication, "Override GUI authentication; username:password")
 	flag.StringVar(&guiAPIKey, "gui-apikey", guiAPIKey, "Override GUI API key")
@@ -239,17 +240,31 @@ func main() {
 		if err == nil {
 			l.Warnln("Key exists; will not overwrite.")
 			l.Infoln("Device ID:", protocol.NewDeviceID(cert.Certificate[0]))
-			return
+		} else {
+			newCertificate(dir, "")
+			cert, err = loadCert(dir, "")
+			myID = protocol.NewDeviceID(cert.Certificate[0])
+			if err != nil {
+				l.Fatalln("load cert:", err)
+			}
+			if err == nil {
+				l.Infoln("Device ID:", protocol.NewDeviceID(cert.Certificate[0]))
+			}
 		}
 
-		newCertificate(dir, "")
-		cert, err = loadCert(dir, "")
+		cfgFile := filepath.Join(dir, "config.xml")
+		if _, err := os.Stat(cfgFile); err == nil {
+			l.Warnln("Config exists; will not overwrite.")
+			return
+		}
+		var myName, _ = os.Hostname()
+		var newCfg = defaultConfig(myName)
+		var cfg = config.Wrap(cfgFile, newCfg)
+		err = cfg.Save()
 		if err != nil {
-			l.Fatalln("load cert:", err)
-		}
-		if err == nil {
-			l.Infoln("Device ID:", protocol.NewDeviceID(cert.Certificate[0]))
+			l.Warnln("Failed to save config", err)
 		}
+
 		return
 	}
 
@@ -346,13 +361,20 @@ func syncthingMain() {
 	// Load the configuration file, if it exists.
 	// If it does not, create a template.
 
-	cfg, err = config.Load(cfgFile, myID)
-	if err == nil {
-		myCfg := cfg.Devices()[myID]
-		if myCfg.Name == "" {
-			myName, _ = os.Hostname()
+	if info, err := os.Stat(cfgFile); err == nil {
+		if !info.Mode().IsRegular() {
+			l.Fatalln("Config file is not a file?")
+		}
+		cfg, err = config.Load(cfgFile, myID)
+		if err == nil {
+			myCfg := cfg.Devices()[myID]
+			if myCfg.Name == "" {
+				myName, _ = os.Hostname()
+			} else {
+				myName = myCfg.Name
+			}
 		} else {
-			myName = myCfg.Name
+			l.Fatalln("Configuration:", err)
 		}
 	} else {
 		l.Infoln("No config file; starting with empty defaults")
@@ -453,6 +475,7 @@ func syncthingMain() {
 	externalPort = addr.Port
 
 	// UPnP
+	igd = nil
 
 	if opts.UPnPEnabled {
 		setupUPnP()
@@ -568,17 +591,11 @@ func setupGUI(cfg *config.ConfigWrapper, m *model.Model) {
 }
 
 func sanityCheckFolders(cfg *config.ConfigWrapper, m *model.Model) {
-	var err error
-
 nextFolder:
 	for id, folder := range cfg.Folders() {
 		if folder.Invalid != "" {
 			continue
 		}
-		folder.Path, err = osutil.ExpandTilde(folder.Path)
-		if err != nil {
-			l.Fatalln("home:", err)
-		}
 		m.AddFolder(folder)
 
 		fi, err := os.Stat(folder.Path)
@@ -591,11 +608,25 @@ nextFolder:
 				l.Warnf("Stopping folder %q - path does not exist, but has files in index", folder.ID)
 				cfg.InvalidateFolder(id, "folder path missing")
 				continue nextFolder
+			} else if !folder.HasMarker() {
+				l.Warnf("Stopping folder %q - path exists, but folder marker missing, check for mount issues", folder.ID)
+				cfg.InvalidateFolder(id, "folder marker missing")
+				continue nextFolder
 			}
 		} else if os.IsNotExist(err) {
 			// If we don't have any files in the index, and the directory
 			// doesn't exist, try creating it.
 			err = os.MkdirAll(folder.Path, 0700)
+			if err != nil {
+				l.Warnf("Stopping folder %q - %v", err)
+				cfg.InvalidateFolder(id, err.Error())
+				continue nextFolder
+			}
+			err = folder.CreateMarker()
+		} else if !folder.HasMarker() {
+			// If we don't have any files in the index, and the path does exist
+			// but the marker is not there, create it.
+			err = folder.CreateMarker()
 		}
 
 		if err != nil {
@@ -660,20 +691,20 @@ func setupUPnP() {
 		} else {
 			// Set up incoming port forwarding, if necessary and possible
 			port, _ := strconv.Atoi(portStr)
-			igd, err := upnp.Discover()
-			if err == nil {
+			igds := upnp.Discover()
+			if len(igds) > 0 {
+				// Configure the first discovered IGD only. This is a work-around until we have a better mechanism
+				// for handling multiple IGDs, which will require changes to the global discovery service
+				igd = igds[0]
+
 				externalPort = setupExternalPort(igd, port)
 				if externalPort == 0 {
 					l.Warnln("Failed to create UPnP port mapping")
 				} else {
-					l.Infoln("Created UPnP port mapping - external port", externalPort)
-				}
-			} else {
-				l.Infof("No UPnP gateway detected")
-				if debugNet {
-					l.Debugf("UPnP: %v", err)
+					l.Infof("Created UPnP port mapping for external port %d on UPnP device %s.", externalPort, igd.FriendlyIdentifier())
 				}
 			}
+
 			if opts.UPnPRenewal > 0 {
 				go renewUPnP(port)
 			}
@@ -684,7 +715,11 @@ func setupUPnP() {
 }
 
 func setupExternalPort(igd *upnp.IGD, port int) int {
-	// We seed the random number generator with the device ID to get a
+	if igd == nil {
+		return 0
+	}
+
+	// We seed the random number generator with the node ID to get a
 	// repeatable sequence of random external ports.
 	rnd := rand.NewSource(certSeed(cert.Certificate[0]))
 	for i := 0; i < 10; i++ {
@@ -702,32 +737,46 @@ func renewUPnP(port int) {
 		opts := cfg.Options()
 		time.Sleep(time.Duration(opts.UPnPRenewal) * time.Minute)
 
-		igd, err := upnp.Discover()
-		if err != nil {
-			continue
+		// Make sure our IGD reference isn't nil
+		if igd == nil {
+			l.Infoln("Undefined IGD during UPnP port renewal. Re-discovering...")
+			igds := upnp.Discover()
+			if len(igds) > 0 {
+				// Configure the first discovered IGD only. This is a work-around until we have a better mechanism
+				// for handling multiple IGDs, which will require changes to the global discovery service
+				igd = igds[0]
+			} else {
+				l.Infof("Failed to re-discover IGD during UPnP port mapping renewal.")
+				continue
+			}
 		}
 
 		// Just renew the same port that we already have
 		if externalPort != 0 {
-			err = igd.AddPortMapping(upnp.TCP, externalPort, port, "syncthing", opts.UPnPLease*60)
+			err := igd.AddPortMapping(upnp.TCP, externalPort, port, "syncthing", opts.UPnPLease*60)
 			if err == nil {
-				l.Infoln("Renewed UPnP port mapping - external port", externalPort)
-				continue
+				l.Infof("Renewed UPnP port mapping for external port %d on device %s.", externalPort, igd.FriendlyIdentifier())
+			} else {
+				l.Warnf("Error renewing UPnP port mapping for external port %d on device %s: %s", externalPort, igd.FriendlyIdentifier(), err.Error())
 			}
+
+			continue
 		}
 
 		// Something strange has happened. We didn't have an external port before?
 		// Or perhaps the gateway has changed?
 		// Retry the same port sequence from the beginning.
+		l.Infoln("No UPnP port mapping defined, updating...")
+
 		r := setupExternalPort(igd, port)
 		if r != 0 {
 			externalPort = r
-			l.Infoln("Updated UPnP port mapping - external port", externalPort)
+			l.Infof("Updated UPnP port mapping for external port %d on device %s.", externalPort, igd.FriendlyIdentifier())
 			discoverer.StopGlobal()
 			discoverer.StartGlobal(opts.GlobalAnnServer, uint16(r))
-			continue
+		} else {
+			l.Warnf("Failed to update UPnP port mapping for external port on device " + igd.FriendlyIdentifier() + ".")
 		}
-		l.Warnln("Failed to update UPnP port mapping - external port", externalPort)
 	}
 }
 

cmd/syncthing/main_test.go

@@ -13,6 +13,107 @@
 // You should have received a copy of the GNU General Public License along
 // with this program. If not, see <http://www.gnu.org/licenses/>.
 
-package main_test
+package main
 
-// Empty test file to generate 0% coverage rather than no coverage
+import (
+	"os"
+	"testing"
+
+	"github.com/syncthing/syncthing/internal/config"
+	"github.com/syncthing/syncthing/internal/files"
+	"github.com/syncthing/syncthing/internal/model"
+	"github.com/syncthing/syncthing/internal/protocol"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+func TestSanityCheck(t *testing.T) {
+	fcfg := config.FolderConfiguration{
+		ID:   "folder",
+		Path: "testdata/testfolder",
+	}
+	cfg := config.Wrap("/tmp/test", config.Configuration{
+		Folders: []config.FolderConfiguration{fcfg},
+	})
+
+	for _, file := range []string{".stfolder", "testfolder", "testfolder/.stfolder"} {
+		_, err := os.Stat("testdata/" + file)
+		if err == nil {
+			t.Error("Found unexpected file")
+		}
+	}
+
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+
+	// Case 1 - new folder, directory and marker created
+
+	m := model.NewModel(cfg, "device", "syncthing", "dev", db)
+	sanityCheckFolders(cfg, m)
+
+	if cfg.Folders()["folder"].Invalid != "" {
+		t.Error("Unexpected error", cfg.Folders()["folder"].Invalid)
+	}
+
+	s, err := os.Stat("testdata/testfolder")
+	if err != nil || !s.IsDir() {
+		t.Error(err)
+	}
+
+	_, err = os.Stat("testdata/testfolder/.stfolder")
+	if err != nil {
+		t.Error(err)
+	}
+
+	os.Remove("testdata/testfolder/.stfolder")
+	os.Remove("testdata/testfolder/")
+
+	// Case 2 - new folder, marker created
+
+	fcfg.Path = "testdata/"
+	cfg = config.Wrap("/tmp/test", config.Configuration{
+		Folders: []config.FolderConfiguration{fcfg},
+	})
+
+	m = model.NewModel(cfg, "device", "syncthing", "dev", db)
+	sanityCheckFolders(cfg, m)
+
+	if cfg.Folders()["folder"].Invalid != "" {
+		t.Error("Unexpected error", cfg.Folders()["folder"].Invalid)
+	}
+
+	_, err = os.Stat("testdata/.stfolder")
+	if err != nil {
+		t.Error(err)
+	}
+
+	os.Remove("testdata/.stfolder")
+
+	// Case 3 - marker missing
+
+	set := files.NewSet("folder", db)
+	set.Update(protocol.LocalDeviceID, []protocol.FileInfo{
+		{Name: "dummyfile"},
+	})
+
+	m = model.NewModel(cfg, "device", "syncthing", "dev", db)
+	sanityCheckFolders(cfg, m)
+
+	if cfg.Folders()["folder"].Invalid != "folder marker missing" {
+		t.Error("Incorrect error")
+	}
+
+	// Case 4 - path missing
+
+	fcfg.Path = "testdata/testfolder"
+	cfg = config.Wrap("/tmp/test", config.Configuration{
+		Folders: []config.FolderConfiguration{fcfg},
+	})
+
+	m = model.NewModel(cfg, "device", "syncthing", "dev", db)
+	sanityCheckFolders(cfg, m)
+
+	if cfg.Folders()["folder"].Invalid != "folder path missing" {
+		t.Error("Incorrect error")
+	}
+}

gui/app.js

@@ -666,15 +666,21 @@ syncthing.controller('SyncthingCtrl', function ($scope, $http, $translate, $loca
     };
 
     $scope.addDevice = function () {
-        $scope.currentDevice = {
-            AddressesStr: 'dynamic',
-            Compression: true,
-            Introducer: false
-        };
-        $scope.editingExisting = false;
-        $scope.editingSelf = false;
-        $scope.deviceEditor.$setPristine();
-        $('#editDevice').modal();
+        $http.get(urlbase + '/discovery')
+		.success(function (registry) {
+			$scope.discovery = registry;
+		})
+		.then(function () {
+			$scope.currentDevice = {
+			    AddressesStr: 'dynamic',
+			    Compression: true,
+			    Introducer: false
+			};
+			$scope.editingExisting = false;
+			$scope.editingSelf = false;
+			$scope.deviceEditor.$setPristine();
+			$('#editDevice').modal();
+		});
     };
 
     $scope.deleteDevice = function () {
@@ -1225,3 +1231,68 @@ syncthing.directive('modal', function () {
         },
     };
 });
+
+syncthing.directive('identicon', ['$window', function ($window) {
+  var svgNS = 'http://www.w3.org/2000/svg';
+
+  function Identicon (value, size) {
+    var svg = document.createElementNS(svgNS, 'svg');
+    var shouldFillRectAt = function (row, col) {
+      return !($window.parseInt(value.charCodeAt(row + col * size), 10) % 2);
+    };
+    var shouldMirrorRectAt = function (row, col) {
+      return !(size % 2 && col === middleCol)
+    };
+    var mirrorColFor = function (col) {
+      return size - col - 1;
+    };
+    var fillRectAt = function (row, col) {
+      var rect = document.createElementNS(svgNS, 'rect');
+
+      rect.setAttribute('x', (col * rectSize) + '%');
+      rect.setAttribute('y', (row * rectSize) + '%');
+      rect.setAttribute('width', rectSize + '%');
+      rect.setAttribute('height', rectSize + '%');
+
+      svg.appendChild(rect);
+    };
+    var rect;
+    var row;
+    var col;
+    var middleCol;
+    var rectSize;
+
+    svg.setAttribute('class', 'identicon');
+    size = size || 5;
+    rectSize = 100 / size;
+    middleCol = Math.ceil(size / 2) - 1;
+
+    if (value) {
+      value = value.toString().replace(/[\W_]/i, '');
+
+      for (row = 0; row < size; ++row) {
+        for (col = middleCol; col > -1; --col) {
+          if (shouldFillRectAt(row, col)) {
+            fillRectAt(row, col);
+
+            if (shouldMirrorRectAt(row, col)) {
+              fillRectAt(row, mirrorColFor(col));
+            }
+          }
+        }
+      }
+    }
+
+    return svg;
+  }
+
+  return {
+    restrict: 'E',
+    scope: {
+      value: '='
+    },
+    link: function (scope, element, attributes) {
+      element.append(new Identicon(scope.value));
+    }
+  }
+}]);

gui/index.html

@@ -190,7 +190,7 @@
         <div class="panel panel-default" ng-repeat="deviceCfg in [thisDevice()]">
           <div class="panel-heading" data-toggle="collapse" href="#device-this" style="cursor: pointer">
             <h3 class="panel-title">
-              <span class="glyphicon glyphicon-home"></span>&emsp;{{deviceName(deviceCfg)}}
+              <identicon data-value="deviceCfg.DeviceID"></identicon>&emsp;{{deviceName(deviceCfg)}}
             </h3>
           </div>
           <div id="device-this" class="panel-collapse collapse in">
@@ -236,7 +236,7 @@
           <div class="panel panel-{{deviceClass(deviceCfg)}}" ng-repeat="deviceCfg in otherDevices()">
             <div class="panel-heading" data-toggle="collapse" data-parent="#devices" href="#device-{{$index}}" style="cursor: pointer">
               <h3 class="panel-title">
-                <span class="glyphicon glyphicon-retweet"></span>&emsp;{{deviceName(deviceCfg)}}
+                <identicon data-value="deviceCfg.DeviceID"></identicon>&emsp;{{deviceName(deviceCfg)}}
                 <span class="pull-right hidden-xs">
                   <span ng-if="connections[deviceCfg.DeviceID] && completion[deviceCfg.DeviceID]._total == 100">
                     <span translate>Up to Date</span> (100%)
@@ -383,7 +383,10 @@
           <form role="form" name="deviceEditor">
             <div class="form-group" ng-class="{'has-error': deviceEditor.deviceID.$invalid && deviceEditor.deviceID.$dirty}">
               <label translate for="deviceID">Device ID</label>
-              <input ng-if="!editingExisting" name="deviceID" id="deviceID" class="form-control text-monospace" type="text" ng-model="currentDevice.DeviceID" required valid-deviceid></input>
+              <input ng-if="!editingExisting" name="deviceID" id="deviceID" class="form-control text-monospace" type="text" ng-model="currentDevice.DeviceID" required valid-deviceid list="discovery-list" />
+              <datalist id="discovery-list" ng-if="!editingExisting">
+                <option ng-repeat="(id,address) in discovery" value="{{ id }}" />
+              </datalist>
               <div ng-if="editingExisting" class="well well-sm text-monospace">{{currentDevice.DeviceID}}</div>
               <p class="help-block">
                 <span translate ng-if="deviceEditor.deviceID.$valid || deviceEditor.deviceID.$pristine">The device ID to enter here can be found in the "Edit > Show ID" dialog on the other device. Spaces and dashes are optional (ignored).</span>
@@ -445,7 +448,7 @@
               <div class="col-md-12">
                 <div class="form-group" ng-class="{'has-error': folderEditor.folderID.$invalid && folderEditor.folderID.$dirty}">
                   <label for="folderID"><span translate>Folder ID</span></label>
-                  <input name="folderID" ng-disabled="editingExisting" id="folderID" class="form-control" type="text" ng-model="currentFolder.ID" required unique-folder ng-pattern="/^[a-zA-Z0-9-_.]{1,64}$/"></input>
+                  <input name="folderID" ng-readonly="editingExisting" id="folderID" class="form-control" type="text" ng-model="currentFolder.ID" required unique-folder ng-pattern="/^[a-zA-Z0-9-_.]{1,64}$/"></input>
                   <p class="help-block">
                     <span translate ng-if="folderEditor.folderID.$valid || folderEditor.folderID.$pristine">Short identifier for the folder. Must be the same on all cluster devices.</span>
                     <span translate ng-if="folderEditor.folderID.$error.uniqueFolder">The folder ID must be unique.</span>
@@ -455,7 +458,7 @@
                 </div>
                 <div class="form-group" ng-class="{'has-error': folderEditor.folderPath.$invalid && folderEditor.folderPath.$dirty}">
                   <label translate for="folderPath">Folder Path</label>
-                  <input name="folderPath" ng-disabled="editingExisting" id="folderPath" class="form-control" type="text" ng-model="currentFolder.Path" required></input>
+                  <input name="folderPath" ng-readonly="editingExisting" id="folderPath" class="form-control" type="text" ng-model="currentFolder.Path" required></input>
                   <p class="help-block">
                     <span translate ng-if="folderEditor.folderPath.$valid || folderEditor.folderPath.$pristine">Path to the folder on the local computer. Will be created if it does not exist. The tilde character (~) can be used as a shortcut for</span> <code>{{system.tilde}}</code>.
                     <span translate ng-if="folderEditor.folderPath.$error.required && folderEditor.folderPath.$dirty">The folder path cannot be blank.</span>
@@ -781,18 +784,24 @@
           <li>Audrius Butkevicius</li>
           <li>Ben Sidhom</li>
           <li>Brandon Philips</li>
-          <li>Gilli Sigurdsson</li>
-          <li>James Patterson</li>
+          <li>Caleb Callaway</li>
+          <li>Chris Joel</li>
+          <li>Daniel Martí</li>
+          <li>Felix Ableitner</li>
+          <li>Felix Unterpaintner</li>
         </ul>
       </div>
       <div class="col-md-6">
         <ul>
+          <li>Gilli Sigurdsson</li>
+          <li>James Patterson</li>
           <li>Jens Diemer</li>
           <li>Jochen Voss</li>
           <li>Lode Hoste</li>
           <li>Marcin Dziadus</li>
           <li>Michael Tilli</li>
           <li>Philippe Schommers</li>
+          <li>Phill Luby</li>
           <li>Ryan Sullivan</li>
           <li>Tully Robinson</li>
           <li>Veeti Paananen</li>
@@ -815,7 +824,6 @@
     </ul>
   </modal>
 
-
   <script src="angular/angular.min.js"></script>
   <script src="angular/angular-translate.min.js"></script>
   <script src="angular/angular-translate-loader.min.js"></script>

gui/lang/lang-de.json

@@ -38,7 +38,7 @@
     "Folder Master": "Keine Veränderungen zulassen",
     "Folder Path": "Verzeichnis Pfad",
     "GUI Authentication Password": "Passwort für Zugang zur Benutzeroberfläche",
-    "GUI Authentication User": "Nutzername für Zugang zur Benutzeroberfläche.",
+    "GUI Authentication User": "Nutzername für Zugang zur Benutzeroberfläche",
     "GUI Listen Addresses": "Adresse(n) für die Benutzeroberfläche",
     "Generate": "Generieren",
     "Global Discovery": "Globale Auffindung",

gui/lang/lang-hu.json

@@ -0,0 +1,140 @@
+{
+    "API Key": "API kulcs",
+    "About": "Névjegy",
+    "Add Device": "Eszköz hozzáadása",
+    "Add Folder": "Mappa hozzáadása",
+    "Address": "Cím",
+    "Addresses": "Címek",
+    "Allow Anonymous Usage Reporting?": "Engedélyezed a névtelen felhasználási adatok küldését?",
+    "Anonymous Usage Reporting": "Névtelen felhasználási adatok küldése",
+    "Any devices configured on an introducer device will be added to this device as well.": "Any devices configured on an introducer device will be added to this device as well.",
+    "Bugs": "Hibák",
+    "CPU Utilization": "Processzor használat",
+    "Close": "Bezárás",
+    "Comment, when used at the start of a line": "Megjegyzés, a sor elején használva",
+    "Compression is recommended in most setups.": "A tömörítés a a legtöbb esetben ajánlott",
+    "Connection Error": "Kapcsolódási hiba",
+    "Copyright © 2014 Jakob Borg and the following Contributors:": "Copyright © 2014 Jakob Borg és az alábbi Közreműködők",
+    "Delete": "Törlés",
+    "Device ID": "Eszköz azonosító",
+    "Device Identification": "Eszköz azonosító",
+    "Device Name": "Eszköz neve",
+    "Disconnected": "Kapcsolat bontva",
+    "Documentation": "Dokumentáció",
+    "Download Rate": "Letöltési sebesség",
+    "Edit": "Szerkesztés",
+    "Edit Device": "Eszköz szerkesztése",
+    "Edit Folder": "Mappa szerkesztése",
+    "Editing": "Szerkesztés",
+    "Enable UPnP": "UPnP engedélyezése",
+    "Enter comma separated \"ip:port\" addresses or \"dynamic\" to perform automatic discovery of the address.": "Add meg kettősponttal elválasztva (ip:port) a címet vagy add meg a \"dynamic\" szót az a cím automatikus észleléséhez.",
+    "Enter ignore patterns, one per line.": "Figyelmen kívül hagyáshoz ide írhatod a mintákat, soronként egyet",
+    "Error": "Hiba",
+    "File Versioning": "Fájl verziózás",
+    "File permission bits are ignored when looking for changes. Use on FAT filesystems.": "A fájl jogosultságok figyelmen kívül hagyása. FAT fájlrendszernél használatos.",
+    "Files are moved to date stamped versions in a .stversions folder when replaced or deleted by syncthing.": "A fájlok időpecsételt verziói a .stversions mappában kerülnek áthelyezésre, amikor felülírásra vagy törlésre kerülnek a Sincthing által.",
+    "Files are protected from changes made on other devices, but changes made on this device will be sent to the rest of the cluster.": "A fájlok védve vannak a más eszközökön történt változásokkal szemben, de az ezen az eszközön történt változások érvényesek lesznek a többire.",
+    "Folder ID": "Mappa azonosító",
+    "Folder Master": "Központi mappa",
+    "Folder Path": "Mappa elérési útja",
+    "GUI Authentication Password": "Grafikus felület jelszava",
+    "GUI Authentication User": "Grafikus felület felhasználó neve ",
+    "GUI Listen Addresses": "Grafikus felület címe",
+    "Generate": "Generálás",
+    "Global Discovery": "Globális felfedezés",
+    "Global Discovery Server": "Globális felfedező szerver",
+    "Global State": "Globális állapot",
+    "Idle": "Tétlen",
+    "Ignore Patterns": "Figyelmen kívül hagyás",
+    "Ignore Permissions": "Jogosultságok figyelmen kívül hagyása",
+    "Incoming Rate Limit (KiB/s)": "Bejövő sebesség korlát (KIB/mp)",
+    "Introducer": "Introducer",
+    "Inversion of the given condition (i.e. do not exclude)": "A feltétel ellentéte (pl. ki nem hagyás)",
+    "Keep Versions": "Megtartott verziók",
+    "Last seen": "Utoljára látva",
+    "Latest Release": "Utolsó kiadás",
+    "Local Discovery": "Helyi felfedezés",
+    "Local State": "Helyi állapot",
+    "Maximum Age": "Maximális kor",
+    "Multi level wildcard (matches multiple directory levels)": "Több szintű helyettesítő karakter (több könyvtár szintre érvényesül)",
+    "Never": "Soha",
+    "No": "Nem",
+    "No File Versioning": "Nincs fájl verziózás",
+    "Notice": "Megjegyzés",
+    "OK": "Rendben",
+    "Offline": "Nincs kapcsolat",
+    "Online": "Kapcsolódva",
+    "Out Of Sync": "Nincs szinkronban",
+    "Outgoing Rate Limit (KiB/s)": "Kimenő sávszélesség (KiB/mp)",
+    "Override Changes": "Változtatások felülbírálása",
+    "Path to the folder on the local computer. Will be created if it does not exist. The tilde character (~) can be used as a shortcut for": "A mappa elérési útja az eszközön. Amennyiben nem létezik, a program automatikusan létrehozza. A hullámvonal (~)  a következő helyettesítésre használható: ",
+    "Path where versions should be stored (leave empty for the default .stversions folder in the folder).": "Elérési út ahol a verziók tárolásra kerülnek (szabadon hagyva az alapértelmezett .stversions mappa lesz használva)",
+    "Please wait": "Kérlek várj",
+    "Preview": "Előnézet",
+    "Preview Usage Report": "Felhasználási adatok átnézése",
+    "Quick guide to supported patterns": "Rövid útmutató a használható mintákról",
+    "RAM Utilization": "Memória használat",
+    "Rescan": "Átnézés",
+    "Rescan Interval": "Átnézési intervallum",
+    "Restart": "Újraindítás",
+    "Restart Needed": "Újraindítás szükséges",
+    "Restarting": "Újraindulás",
+    "Save": "Mentés",
+    "Scanning": "Átnézés",
+    "Select the devices to share this folder with.": "Válaszd ki az eszközöket amelyekkel meg szeretnéd osztani a mappát",
+    "Settings": "Beállítások",
+    "Share With Devices": "Megosztás eszközökkel",
+    "Shared With": "Megosztva velük",
+    "Short identifier for the folder. Must be the same on all cluster devices.": "Rövid azonosító. Minden megosztott eszközön azonos kell legyen.",
+    "Show ID": "Azonosító mutatása",
+    "Shown instead of Device ID in the cluster status. Will be advertised to other devices as an optional default name.": "Az eszköz azonosító helyett jelenik meg. A többi eszközön alapértelmezett névként használható. ",
+    "Shown instead of Device ID in the cluster status. Will be updated to the name the device advertises if left empty.": "Az eszköz azonosító helyett jelenik meg. Üresen hagyva az eszköz saját neve lesz használva ",
+    "Shutdown": "Leállítás",
+    "Simple File Versioning": "Egyszerű fájl verziózás",
+    "Single level wildcard (matches within a directory only)": "Egyszintű helyettesítő karakter (csak egy mappára érvényes)",
+    "Source Code": "Forráskód",
+    "Staggered File Versioning": "Többszintű fájl verziózás",
+    "Start Browser": "Böngésző indítása",
+    "Stopped": "Leállítva",
+    "Support / Forum": "Támogatás / Fórum",
+    "Sync Protocol Listen Addresses": "Szinkronizációs protokoll címe",
+    "Synchronization": "Szinkronizálás",
+    "Syncing": "Szinkronizálás",
+    "Syncthing has been shut down.": "Syncthing leállítva",
+    "Syncthing includes the following software or portions thereof:": "Syncthing a következő programokat, vagy komponenseket tartalmazza.",
+    "Syncthing is restarting.": "Syncthing újraindul",
+    "Syncthing is upgrading.": "Syncthing frissül",
+    "Syncthing seems to be down, or there is a problem with your Internet connection. Retrying…": "A Syncthing úgy tűnik, hogy nem működik, vagy valami probléma van az hálózati kapcsolattal. Újra próbálom...",
+    "The aggregated statistics are publicly available at {%url%}.": "Az összevont statisztikák nyilvánosan elérhetők a {{url}} címen.",
+    "The configuration has been saved but not activated. Syncthing must restart to activate the new configuration.": "A beállítások elmentésre kerültek, de nem lettek aktiválva. Indíts újra a Syncthing-et, hogy aktiváld őket.",
+    "The device ID cannot be blank.": "Az eszköz azonosító nem lehet üres.",
+    "The device ID to enter here can be found in the \"Edit > Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).": "Az eszköz azonosító az \"Azonosító mutatása\" ablakban található az eszközökön. A szóközök és kötőjelek opcionálisak (nem lesznek figyelembe véve)",
+    "The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.": "A titkosított felhasználási adatok naponta kerülnek küldésre. Arra használjuk őket hogy kövessük a különböző platformokat, mappa méreteket és program verziókat. Amennyiben az elküldésre kerülő adat megváltozik, ez az ablak újra megjelenik.",
+    "The entered device ID does not look valid. It should be a 52 or 56 character string consisting of letters and numbers, with spaces and dashes being optional.": "A beírt eszköz azonosító nem valódi. Az azonosító 52 vagy 56 karakter hosszú, számokból és betűkből áll, opcionálisan szóközöket és kötőjeleket tartalmaz.",
+    "The folder ID cannot be blank.": "A mappa azonosító nem lehet üres.",
+    "The folder ID must be a short identifier (64 characters or less) consisting of letters, numbers and the dot (.), dash (-) and underscode (_) characters only.": "A mappa azonosító rövid (maximum 64 karakter), betűkből, számokból és a pont (.), kötőjel (-) és alulvonás (_) karakterekből állhat.",
+    "The folder ID must be unique.": "A mappa azonosító egyedi kell legyen",
+    "The folder path cannot be blank.": "Az elérési út nem lehet üres",
+    "The following intervals are used: for the first hour a version is kept every 30 seconds, for the first day a version is kept every hour, for the first 30 days a version is kept every day, until the maximum age a version is kept every week.": "A következő intervallumokat használjuk: egy régi verziót őrzünk meg az első órában minden 30 másodpercben, az első nap minden órában, az első 30 napban minden nap, egészen addig amíg el nem érjük a maximálisan megtartható verziók számát minden héten.",
+    "The maximum age must be a number and cannot be blank.": "A maximális kornak számnak kell lenni és nem lehet üres",
+    "The maximum time to keep a version (in days, set to 0 to keep versions forever).": "A verziók megtartásának maximális ideje (napokban, ha 0-t adsz meg örökre megmaradnak).",
+    "The number of old versions to keep, per file.": "A megtartott régi verziók száma, fájlonként.",
+    "The number of versions must be a number and cannot be blank.": "A megtartott verziók száma nem lehet üres",
+    "The rescan interval must be at least 5 seconds.": "Az átnézési intervallumnak legalább 5 másodpercnek kell lennie.",
+    "Unknown": "Ismeretlen",
+    "Up to Date": "Friss",
+    "Upgrade To {%version%}": "Frissítés a {{version}} verzióra",
+    "Upgrading": "Frissítés",
+    "Upload Rate": "Feltöltési sebesség",
+    "Use Compression": "Tömörítés használata",
+    "Use HTTPS for GUI": "HTTPS használata a GUI-hoz",
+    "Version": "Verzió",
+    "Versions Path": "Verziók útvonala",
+    "Versions are automatically deleted if they are older than the maximum age or exceed the number of files allowed in an interval.": "A régi verziók automatikusan törlődnek, amennyiben öregebbek mint a maximum kor, vagy már több van belőlük mint az adott intervallumban megtartható maximum.",
+    "When adding a new device, keep in mind that this device must be added on the other side too.": "Amikor új eszközt adsz hozzá, tartsd észben hogy, ezt az eszközt is hozzá kell adni a másik oldalon.",
+    "When adding a new folder, keep in mind that the Folder ID is used to tie folders together between devices. They are case sensitive and must match exactly between all devices.": "Amikor új mappát adsz hozzá, tartsd észben hogy a mappa azonosító arra való hogy összekösd a mappákat az eszközeiden. Az azonosító kisbetű-nagybetű érzékeny és pontosan egyeznie kell az eszközökön.",
+    "Yes": "Igen",
+    "You must keep at least one version.": "Legalább egy verziót meg kell tartanod",
+    "full documentation": "teljes dokumentáció",
+    "items": "tételek"
+}

gui/lang/lang-it.json

@@ -81,9 +81,9 @@
     "Restarting": "Riavvio",
     "Save": "Salva",
     "Scanning": "Scansione in corso",
-    "Select the devices to share this folder with.": "Select the devices to share this folder with.",
+    "Select the devices to share this folder with.": "Seleziona i dispositivi con i quali condividere questa cartella.",
     "Settings": "Impostazioni",
-    "Share With Devices": "Share With Devices",
+    "Share With Devices": "Condividi Con i Dispositivi",
     "Shared With": "Condiviso Con",
     "Short identifier for the folder. Must be the same on all cluster devices.": "Short identifier for the folder. Must be the same on all cluster devices.",
     "Show ID": "Mostra ID",

gui/lang/lang-pl.json

@@ -0,0 +1,140 @@
+{
+    "API Key": "Klucz API",
+    "About": "O Syncthing",
+    "Add Device": "Dodaj urządzenie",
+    "Add Folder": "Dodaj folder",
+    "Address": "Adres",
+    "Addresses": "Adresy",
+    "Allow Anonymous Usage Reporting?": "Zezwalaj na anonimowe statystyki użycia",
+    "Anonymous Usage Reporting": "Anonimowe statystyki użycia",
+    "Any devices configured on an introducer device will be added to this device as well.": "Wszystkie urządzenia skonfigurowane na urządzeniu wprowadzającym zostaną dodane także do tego urządzenia.",
+    "Bugs": "Błędy",
+    "CPU Utilization": "Użycie CPU",
+    "Close": "Zamknij",
+    "Comment, when used at the start of a line": "Komentarz, jeżeli użyty na początku linii",
+    "Compression is recommended in most setups.": "Kompresja jest zalecana w większości przypadków",
+    "Connection Error": "Błąd połączenia",
+    "Copyright © 2014 Jakob Borg and the following Contributors:": "Copyright © 2014 Jakob Borg i następujący współautorzy:",
+    "Delete": "Usuń",
+    "Device ID": "ID urządzenia",
+    "Device Identification": "Identyfikator urządzenia",
+    "Device Name": "Nazwa urządzenia",
+    "Disconnected": "Rozłączony",
+    "Documentation": "Dokumentacja",
+    "Download Rate": "Prędkość pobierania",
+    "Edit": "Edytuj",
+    "Edit Device": "Edytuj urządzenie",
+    "Edit Folder": "Edytuj folder",
+    "Editing": "Edytowanie",
+    "Enable UPnP": "Włącz UPnP",
+    "Enter comma separated \"ip:port\" addresses or \"dynamic\" to perform automatic discovery of the address.": "Wprowadź adresy oddzielone przecinkiem \"ip:port\", lub wpisz \"dynamic\" w celu wykrycia adresu. ",
+    "Enter ignore patterns, one per line.": "Wprowadz wzorce ignorowania, jeden w każdej linii.",
+    "Error": "Błąd",
+    "File Versioning": "Wersjonowanie plików",
+    "File permission bits are ignored when looking for changes. Use on FAT filesystems.": "Uprawnienia plików są ignorowane przy poszukiwaniu zmian. Używaj w systemie plików FAT",
+    "Files are moved to date stamped versions in a .stversions folder when replaced or deleted by syncthing.": "W momencie zmiany lub usuwania pliki są przenoszone do oznaczonych datą wersji wewnątrz folderu .stversions",
+    "Files are protected from changes made on other devices, but changes made on this device will be sent to the rest of the cluster.": "Pliki są zabezpieczone przed zmianami na innym urządzeniu, jednak zmiany w tym urządzeniu będą wysłane do reszty.",
+    "Folder ID": "ID folderu",
+    "Folder Master": "Główny folder",
+    "Folder Path": "Ścieżka folderu",
+    "GUI Authentication Password": "Hasło",
+    "GUI Authentication User": "Użytkownik",
+    "GUI Listen Addresses": "Adres nasłuchu",
+    "Generate": "Generuj",
+    "Global Discovery": "Globalne odnajdywanie",
+    "Global Discovery Server": "Globalny serwer rozgłoszeniowy",
+    "Global State": "Status globalny",
+    "Idle": "Bezczynny",
+    "Ignore Patterns": "Wzorce ignorowania",
+    "Ignore Permissions": "Ignoruj uprawnienia",
+    "Incoming Rate Limit (KiB/s)": "Ograniczenie prędkości odbierania (KiB/s)",
+    "Introducer": "Wprowadzający",
+    "Inversion of the given condition (i.e. do not exclude)": "Odwrócenie podanego wzorca (np. nie wykluczaj)",
+    "Keep Versions": "Zachowuj wersje",
+    "Last seen": "Ostatnio widziany",
+    "Latest Release": "Najnowsza wersja",
+    "Local Discovery": "Lokalne odnajdywanie",
+    "Local State": "Status lokalny",
+    "Maximum Age": "Maksymalny wiek",
+    "Multi level wildcard (matches multiple directory levels)": "Wieloznaczność na poziomie katalogów i plików (uwzględnia nazwy folderów i plików)",
+    "Never": "Nigdy",
+    "No": "Nie",
+    "No File Versioning": "Bez wersjonowania pliku",
+    "Notice": "Wskazówka",
+    "OK": "OK",
+    "Offline": "Rozłączony",
+    "Online": "Połączony",
+    "Out Of Sync": "Niezsynchronizowane",
+    "Outgoing Rate Limit (KiB/s)": "Ograniczenie prędkości wysyłania (KiB/s)",
+    "Override Changes": "Nadpisz zmiany",
+    "Path to the folder on the local computer. Will be created if it does not exist. The tilde character (~) can be used as a shortcut for": "Ścieżka do lokalnego folderu. Zostanie utworzona jeżeli nie istnieje.\nZnak tyldy (~) może zostać użyty jako skrót do",
+    "Path where versions should be stored (leave empty for the default .stversions folder in the folder).": "Ścieżka gdzie będą przechowywane wersje (pozostaw puste dla domyślnego folderu .stversions)",
+    "Please wait": "Proszę czekać",
+    "Preview": "Podgląd",
+    "Preview Usage Report": "Podgląd raportu użycia.",
+    "Quick guide to supported patterns": "Krótki przewodnik po obsługiwanych wzorcach",
+    "RAM Utilization": "Użycie pamięci RAM",
+    "Rescan": "Skanuj ponownie",
+    "Rescan Interval": "Interwał skanowania",
+    "Restart": "Uruchom ponownie",
+    "Restart Needed": "Wymagane ponowne uruchomienie",
+    "Restarting": "Uruchamianie ponowne",
+    "Save": "Zapisz",
+    "Scanning": "Skanowanie",
+    "Select the devices to share this folder with.": "Wybierz urządzenie, któremu udostępnić folder.",
+    "Settings": "Ustrawienia",
+    "Share With Devices": "Udostępnij dla urządzenia",
+    "Shared With": "Współdzielony z",
+    "Short identifier for the folder. Must be the same on all cluster devices.": "Krótki identyfikator folderu. Musi być taki sam na wszystkich urządzeniach.",
+    "Show ID": "Pokaż ID",
+    "Shown instead of Device ID in the cluster status. Will be advertised to other devices as an optional default name.": "Pokazane w statusie zamiast ID urządzenia.Zostanie wysłane do innych urządzeń jako opcjonalna domyślna nazwa.",
+    "Shown instead of Device ID in the cluster status. Will be updated to the name the device advertises if left empty.": "Pokazane w statusie zamiast ID urządzenia. Zostanie zaktualizowane do nazwy urządzenia jeżeli pozostanie puste.",
+    "Shutdown": "Wyłącz",
+    "Simple File Versioning": "Proste wersjonowanie pliku",
+    "Single level wildcard (matches within a directory only)": "Wieloznaczność na poziomie plików (uwzględnia nazwy plików)",
+    "Source Code": "Kod źródłowy",
+    "Staggered File Versioning": "Rozbudowane wersjonowanie pliku",
+    "Start Browser": "Uruchom przeglądarkę",
+    "Stopped": "Zatrzymany",
+    "Support / Forum": "Wsparcie / Forum",
+    "Sync Protocol Listen Addresses": "Adres nasłuchu protokołu synchronizacji",
+    "Synchronization": "Synchronizacja",
+    "Syncing": "Synchronizowanie",
+    "Syncthing has been shut down.": "Syncthing został wyłączony",
+    "Syncthing includes the following software or portions thereof:": "Syncthing zawiera następujące oprogramowanie lub ich częśći:",
+    "Syncthing is restarting.": "Restart Syncthing",
+    "Syncthing is upgrading.": "Aktualizowanie Syncthing",
+    "Syncthing seems to be down, or there is a problem with your Internet connection. Retrying…": "Syncthing wydaje się być wyłączony lub jest problem z twoim połączeniem internetowym. Próbuje ponownie...",
+    "The aggregated statistics are publicly available at {%url%}.": "Zebrane statystyki są publicznie dostępna pod adresem {{url}}.",
+    "The configuration has been saved but not activated. Syncthing must restart to activate the new configuration.": "Konfiguracja została zapisana lecz nie jest aktywna. Syncthing musi zostać zrestartowany aby aktywować nową konfiguracje.",
+    "The device ID cannot be blank.": "ID urządzenia nie może być puste.",
+    "The device ID to enter here can be found in the \"Edit > Show ID\" dialog on the other device. Spaces and dashes are optional (ignored).": "ID urządzenia można znaleźć w \"Edytuj -> Pokaż ID\" na zdalnym urządzeniu.\nOdstępy i myślniki są opcjonalne (ignorowane)",
+    "The encrypted usage report is sent daily. It is used to track common platforms, folder sizes and app versions. If the reported data set is changed you will be prompted with this dialog again.": "Zaszyfrowane raporty użycia są wysyłane codziennie. Są one używane w celach statystycznych platform, rozmiarów katalogów i wersji programu. Jeżeli zgłaszane dane ulegną zmianie, ponownie wyświetli się ta informacja.",
+    "The entered device ID does not look valid. It should be a 52 or 56 character string consisting of letters and numbers, with spaces and dashes being optional.": "Wprowadzone ID urządzenia wygląda na niepoprawne. Musi zawierać 52 lub 56 znaków składających się z liter i cyfr. Odstępy i myślniki są opcjonalne.",
+    "The folder ID cannot be blank.": "ID folderu nie może być puste.",
+    "The folder ID must be a short identifier (64 characters or less) consisting of letters, numbers and the dot (.), dash (-) and underscode (_) characters only.": "ID folderu musi być krótkim identyfikatorem (64 lub mniej znaków) zawierać litery, cyfry, znaki kropki (.), myślnika (-) i podkreślenia (_).",
+    "The folder ID must be unique.": "ID folderu musi być unikalne.",
+    "The folder path cannot be blank.": "Ścieżka folderu nie może być pusta.",
+    "The following intervals are used: for the first hour a version is kept every 30 seconds, for the first day a version is kept every hour, for the first 30 days a version is kept every day, until the maximum age a version is kept every week.": "Następujący interwał jest używany: dla pierwszej godziny wersja jest zachowywana co 30 sekund, dla pierwszego dnia wersja jest zachowywana co godzinę, dla pierwszego miesiąca wersja jest zachowywana codziennie, aż do maksymalnego odstępu zapisywania wersji co tydzień.",
+    "The maximum age must be a number and cannot be blank.": "Maksymalny wiek musi być liczbą i nie może być pusty.",
+    "The maximum time to keep a version (in days, set to 0 to keep versions forever).": "Maksymalny czas zachowania wersji (w dniach, ustaw 0 aby zachować na zawsze)",
+    "The number of old versions to keep, per file.": "Liczba wersji pliku do zachowania.",
+    "The number of versions must be a number and cannot be blank.": "Liczba wersji musi być liczbą i nie może być pusta.",
+    "The rescan interval must be at least 5 seconds.": "Interwał skanowania musi wynosić co najmniej 5 sekund.",
+    "Unknown": "Nieznany",
+    "Up to Date": "Aktualny",
+    "Upgrade To {%version%}": "Aktualizuj do {{version}}",
+    "Upgrading": "Aktualizowanie",
+    "Upload Rate": "Prędkość wysyłania",
+    "Use Compression": "Używaj kompresji",
+    "Use HTTPS for GUI": "Używaj HTTPS",
+    "Version": "Wersja",
+    "Versions Path": "Ścieżka wersji",
+    "Versions are automatically deleted if they are older than the maximum age or exceed the number of files allowed in an interval.": "Wersje zostają automatycznie usunięte jeżeli są starsze niż maksymalny wiek lub przekraczają liczbę dopuszczalnych wersji.",
+    "When adding a new device, keep in mind that this device must be added on the other side too.": "Gdy dodajesz nowe urządzenie, pamiętaj że urządzenie musi zostać dodane także po drugiej stronie.",
+    "When adding a new folder, keep in mind that the Folder ID is used to tie folders together between devices. They are case sensitive and must match exactly between all devices.": "Przy dodawaniu nowego folderu, pamiętaj, że ID użyte jest do łączenia folderów pomiędzy urządzeniami. Wielkość liter ciągu ma znaczenie musi zgadzać się na wszystkich urządzeniach.",
+    "Yes": "Tak",
+    "You must keep at least one version.": "Musisz posiadać przynajmniej jedną wersję",
+    "full documentation": "pełna dokumentacja",
+    "items": "pozycji."
+}

gui/lang/valid-langs.js

@@ -1 +1 @@
-var validLangs = ["bg","de","en","fr","it","lt","pt-PT","sv","zh-CN","zh-TW"]
+var validLangs = ["bg","de","en","fr","hu","it","lt","pl","pt-PT","sv","zh-CN","zh-TW"]

gui/overrides.css

@@ -28,6 +28,43 @@ ul+h5 {
     margin-top: 1.5em;
 }
 
+identicon {
+  display: inline-block;
+  position: relative;
+  width: 1em;
+  height: 1em;
+  line-height: 1em;
+  overflow: visible;
+}
+
+.identicon {
+    width: 30px;
+    height: 30px;
+}
+
+.identicon rect {
+    opacity: 0.85;
+    fill: #888;
+}
+
+.panel-heading .identicon {
+    display: block;
+    position: absolute;
+    top: -4px;
+    left: -9px;
+}
+
+.panel-heading {
+    position: relative;
+    overflow: hidden;
+}
+
+[class*="-info"] .identicon rect,
+[class*="-success"] .identicon rect,
+[class*="-primary"] .identicon rect {
+    fill: #fff;
+}
+
 .text-monospace {
     font-family: Menlo, Monaco, Consolas, "Courier New", monospace;
 }

internal/config/config.go

@@ -21,18 +21,20 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"reflect"
 	"sort"
 	"strconv"
 
 	"code.google.com/p/go.crypto/bcrypt"
 	"github.com/syncthing/syncthing/internal/logger"
+	"github.com/syncthing/syncthing/internal/osutil"
 	"github.com/syncthing/syncthing/internal/protocol"
 )
 
 var l = logger.DefaultLogger
 
-const CurrentVersion = 5
+const CurrentVersion = 6
 
 type Configuration struct {
 	Version int                   `xml:"version,attr"`
@@ -55,6 +57,7 @@ type FolderConfiguration struct {
 	RescanIntervalS int                         `xml:"rescanIntervalS,attr" default:"60"`
 	IgnorePerms     bool                        `xml:"ignorePerms,attr"`
 	Versioning      VersioningConfiguration     `xml:"versioning"`
+	LenientMtimes   bool                        `xml:"lenientMtimes"`
 
 	Invalid string `xml:"-"` // Set at runtime when there is an error, not saved
 
@@ -64,6 +67,28 @@ type FolderConfiguration struct {
 	Deprecated_Nodes     []FolderDeviceConfiguration `xml:"node" json:"-"`
 }
 
+func (f *FolderConfiguration) CreateMarker() error {
+	if !f.HasMarker() {
+		marker := filepath.Join(f.Path, ".stfolder")
+		fd, err := os.Create(marker)
+		if err != nil {
+			return err
+		}
+		fd.Close()
+		osutil.HideFile(marker)
+	}
+
+	return nil
+}
+
+func (f *FolderConfiguration) HasMarker() bool {
+	_, err := os.Stat(filepath.Join(f.Path, ".stfolder"))
+	if err != nil {
+		return false
+	}
+	return true
+}
+
 func (r *FolderConfiguration) DeviceIDs() []protocol.DeviceID {
 	if r.deviceIDs == nil {
 		for _, n := range r.Devices {
@@ -148,6 +173,7 @@ type OptionsConfiguration struct {
 	RestartOnWakeup      bool     `xml:"restartOnWakeup" default:"true"`
 	AutoUpgradeIntervalH int      `xml:"autoUpgradeIntervalH" default:"12"` // 0 for off
 	KeepTemporariesH     int      `xml:"keepTemporariesH" default:"24"`     // 0 for off
+	CacheIgnoredFiles    bool     `xml:"cacheIgnoredFiles" default:"true"`
 
 	Deprecated_RescanIntervalS int    `xml:"rescanIntervalS,omitempty" json:"-"`
 	Deprecated_UREnabled       bool   `xml:"urEnabled,omitempty" json:"-"`
@@ -272,6 +298,11 @@ func (cfg *Configuration) prepare(myID protocol.DeviceID) {
 		convertV4V5(cfg)
 	}
 
+	// Upgrade to v6 configuration if appropriate
+	if cfg.Version == 5 {
+		convertV5V6(cfg)
+	}
+
 	// Hash old cleartext passwords
 	if len(cfg.GUI.Password) > 0 && cfg.GUI.Password[0] != '$' {
 		hash, err := bcrypt.GenerateFromPassword([]byte(cfg.GUI.Password), 0)
@@ -344,6 +375,19 @@ func ChangeRequiresRestart(from, to Configuration) bool {
 	return false
 }
 
+func convertV5V6(cfg *Configuration) {
+	// Added ".stfolder" file at folder roots to identify mount issues
+	// Doesn't affect the config itself, but uses config migrations to identify
+	// the migration point.
+	for _, folder := range Wrap("", *cfg).Folders() {
+		// Best attempt, if it fails, it fails, the user will have to fix
+		// it up manually, as the repo will not get started.
+		folder.CreateMarker()
+	}
+
+	cfg.Version = 6
+}
+
 func convertV4V5(cfg *Configuration) {
 	// Renamed a bunch of fields in the structs.
 	if cfg.Deprecated_Nodes == nil {

internal/config/config_test.go

@@ -16,6 +16,7 @@
 package config
 
 import (
+	"fmt"
 	"os"
 	"reflect"
 	"testing"
@@ -50,6 +51,7 @@ func TestDefaultValues(t *testing.T) {
 		RestartOnWakeup:      true,
 		AutoUpgradeIntervalH: 12,
 		KeepTemporariesH:     24,
+		CacheIgnoredFiles:    true,
 	}
 
 	cfg := New(device1)
@@ -60,17 +62,26 @@ func TestDefaultValues(t *testing.T) {
 }
 
 func TestDeviceConfig(t *testing.T) {
-	for i, ver := range []string{"v1", "v2", "v3", "v4", "v5"} {
-		wr, err := Load("testdata/"+ver+".xml", device1)
+	for i := 1; i <= CurrentVersion; i++ {
+		os.Remove("testdata/.stfolder")
+		wr, err := Load(fmt.Sprintf("testdata/v%d.xml", i), device1)
 		if err != nil {
 			t.Fatal(err)
 		}
+
+		_, err = os.Stat("testdata/.stfolder")
+		if i < 6 && err != nil {
+			t.Fatal(err)
+		} else if i >= 6 && err == nil {
+			t.Fatal("Unexpected file")
+		}
+
 		cfg := wr.cfg
 
 		expectedFolders := []FolderConfiguration{
 			{
 				ID:              "test",
-				Path:            "~/Sync",
+				Path:            "testdata/",
 				Devices:         []FolderDeviceConfiguration{{DeviceID: device1}, {DeviceID: device4}},
 				ReadOnly:        true,
 				RescanIntervalS: 600,
@@ -92,8 +103,8 @@ func TestDeviceConfig(t *testing.T) {
 		}
 		expectedDeviceIDs := []protocol.DeviceID{device1, device4}
 
-		if cfg.Version != 5 {
-			t.Errorf("%d: Incorrect version %d != 5", i, cfg.Version)
+		if cfg.Version != CurrentVersion {
+			t.Errorf("%d: Incorrect version %d != %d", i, cfg.Version, CurrentVersion)
 		}
 		if !reflect.DeepEqual(cfg.Folders, expectedFolders) {
 			t.Errorf("%d: Incorrect Folders\n  A: %#v\n  E: %#v", i, cfg.Folders, expectedFolders)
@@ -138,6 +149,7 @@ func TestOverriddenValues(t *testing.T) {
 		RestartOnWakeup:      false,
 		AutoUpgradeIntervalH: 24,
 		KeepTemporariesH:     48,
+		CacheIgnoredFiles:    false,
 	}
 
 	cfg, err := Load("testdata/overridenvalues.xml", device1)
@@ -296,7 +308,7 @@ func TestPrepare(t *testing.T) {
 }
 
 func TestRequiresRestart(t *testing.T) {
-	wr, err := Load("testdata/v5.xml", device1)
+	wr, err := Load("testdata/v6.xml", device1)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/config/testdata/overridenvalues.xml

@@ -18,5 +18,6 @@
         <restartOnWakeup>false</restartOnWakeup>
         <autoUpgradeIntervalH>24</autoUpgradeIntervalH>
         <keepTemporariesH>48</keepTemporariesH>
+        <cacheIgnoredFiles>false</cacheIgnoredFiles>
     </options>
 </configuration>

internal/config/testdata/v1.xml

@@ -1,5 +1,5 @@
 <configuration version="1">
-    <repository id="test" directory="~/Sync">
+    <repository id="test" directory="testdata/">
         <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ" name="node one">
             <address>a</address>
         </node>

internal/config/testdata/v2.xml

@@ -1,5 +1,5 @@
 <configuration version="2">
-    <repository id="test" directory="~/Sync" ro="true">
+    <repository id="test" directory="testdata/" ro="true">
         <node id="P56IOI7MZJNU2IQGDREYDM2MGTMGL3BXNPQ6W5BTBBZ4TJXZWICQ"/>
         <node id="AIR6LPZ7K4PTTUXQSMUUCPQ5YWOEDFIIQJUG7772YQXXR5YD6AWQ"/>
         <node id="C4YBIESWDUAIGU62GOSRXCRAAJDWVE3TKCPMURZE2LH5QHAF576A"/>

internal/config/testdata/v3.xml

@@ -1,5 +1,5 @@
 <configuration version="3">
-    <repository id="test" directory="~/Sync" ro="true" ignorePerms="false">
+    <repository id="test" directory="testdata/" ro="true" ignorePerms="false">
         <node id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" compression="false"></node>
         <node id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" compression="false"></node>
     </repository>

internal/config/testdata/v4.xml

@@ -1,5 +1,5 @@
 <configuration version="4">
-    <repository id="test" directory="~/Sync" ro="true" ignorePerms="false" rescanIntervalS="600">
+    <repository id="test" directory="testdata/" ro="true" ignorePerms="false" rescanIntervalS="600">
         <node id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></node>
         <node id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></node>
     </repository>

internal/config/testdata/v5.xml

@@ -1,5 +1,5 @@
 <configuration version="5">
-    <folder id="test" path="~/Sync" ro="true" ignorePerms="false" rescanIntervalS="600">
+    <folder id="test" path="testdata/" ro="true" ignorePerms="false" rescanIntervalS="600">
         <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
         <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
     </folder>

internal/config/testdata/v6.xml

@@ -0,0 +1,12 @@
+<configuration version="6">
+    <folder id="test" path="testdata/" ro="true" ignorePerms="false" rescanIntervalS="600">
+        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
+        <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
+    </folder>
+    <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="true">
+        <address>a</address>
+    </device>
+    <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="true">
+        <address>b</address>
+    </device>
+</configuration>

internal/config/testdata/versioningconfig.xml

@@ -1,5 +1,5 @@
 <configuration version="2">
-    <repository id="test" directory="~/Sync" ro="true">
+    <repository id="test" directory="testdata/" ro="true">
         <versioning type="simple">
             <param key="foo" val="bar"/>
             <param key="baz" val="quux"/>

internal/config/wrapper.go

@@ -167,6 +167,12 @@ func (w *ConfigWrapper) Folders() map[string]FolderConfiguration {
 	if w.folderMap == nil {
 		w.folderMap = make(map[string]FolderConfiguration, len(w.cfg.Folders))
 		for _, fld := range w.cfg.Folders {
+			path, err := osutil.ExpandTilde(fld.Path)
+			if err != nil {
+				l.Warnln("home:", err)
+				continue
+			}
+			fld.Path = path
 			w.folderMap[fld.ID] = fld
 		}
 	}

internal/discover/discover.go

@@ -39,7 +39,7 @@ type Discoverer struct {
 	cacheLifetime    time.Duration
 	broadcastBeacon  beacon.Interface
 	multicastBeacon  beacon.Interface
-	registry         map[protocol.DeviceID][]cacheEntry
+	registry         map[protocol.DeviceID][]CacheEntry
 	registryLock     sync.RWMutex
 	extServer        string
 	extPort          uint16
@@ -51,9 +51,9 @@ type Discoverer struct {
 	extAnnounceOKmut sync.Mutex
 }
 
-type cacheEntry struct {
-	addr string
-	seen time.Time
+type CacheEntry struct {
+	Address string
+	Seen    time.Time
 }
 
 var (
@@ -68,7 +68,7 @@ func NewDiscoverer(id protocol.DeviceID, addresses []string) *Discoverer {
 		globalBcastIntv: 1800 * time.Second,
 		errorRetryIntv:  60 * time.Second,
 		cacheLifetime:   5 * time.Minute,
-		registry:        make(map[protocol.DeviceID][]cacheEntry),
+		registry:        make(map[protocol.DeviceID][]CacheEntry),
 	}
 }
 
@@ -139,16 +139,16 @@ func (d *Discoverer) Lookup(device protocol.DeviceID) []string {
 	if len(cached) > 0 {
 		addrs := make([]string, len(cached))
 		for i := range cached {
-			addrs[i] = cached[i].addr
+			addrs[i] = cached[i].Address
 		}
 		return addrs
 	} else if len(d.extServer) != 0 {
 		addrs := d.externalLookup(device)
-		cached = make([]cacheEntry, len(addrs))
+		cached = make([]CacheEntry, len(addrs))
 		for i := range addrs {
-			cached[i] = cacheEntry{
-				addr: addrs[i],
-				seen: time.Now(),
+			cached[i] = CacheEntry{
+				Address: addrs[i],
+				Seen:    time.Now(),
 			}
 		}
 
@@ -169,11 +169,11 @@ func (d *Discoverer) Hint(device string, addrs []string) {
 	})
 }
 
-func (d *Discoverer) All() map[protocol.DeviceID][]cacheEntry {
+func (d *Discoverer) All() map[protocol.DeviceID][]CacheEntry {
 	d.registryLock.RLock()
-	devices := make(map[protocol.DeviceID][]cacheEntry, len(d.registry))
+	devices := make(map[protocol.DeviceID][]CacheEntry, len(d.registry))
 	for device, addrs := range d.registry {
-		addrsCopy := make([]cacheEntry, len(addrs))
+		addrsCopy := make([]CacheEntry, len(addrs))
 		copy(addrsCopy, addrs)
 		devices[device] = addrsCopy
 	}
@@ -365,14 +365,14 @@ func (d *Discoverer) registerDevice(addr net.Addr, device Device) bool {
 			deviceAddr = ua.String()
 		}
 		for i := range current {
-			if current[i].addr == deviceAddr {
-				current[i].seen = time.Now()
+			if current[i].Address == deviceAddr {
+				current[i].Seen = time.Now()
 				goto done
 			}
 		}
-		current = append(current, cacheEntry{
-			addr: deviceAddr,
-			seen: time.Now(),
+		current = append(current, CacheEntry{
+			Address: deviceAddr,
+			Seen:    time.Now(),
 		})
 	done:
 	}
@@ -388,7 +388,7 @@ func (d *Discoverer) registerDevice(addr net.Addr, device Device) bool {
 	if len(current) > len(orig) {
 		addrs := make([]string, len(current))
 		for i := range current {
-			addrs[i] = current[i].addr
+			addrs[i] = current[i].Address
 		}
 		events.Default.Log(events.DeviceDiscovered, map[string]interface{}{
 			"device": id.String(),
@@ -468,11 +468,11 @@ func (d *Discoverer) externalLookup(device protocol.DeviceID) []string {
 	return addrs
 }
 
-func (d *Discoverer) filterCached(c []cacheEntry) []cacheEntry {
+func (d *Discoverer) filterCached(c []CacheEntry) []CacheEntry {
 	for i := 0; i < len(c); {
-		if ago := time.Since(c[i].seen); ago > d.cacheLifetime {
+		if ago := time.Since(c[i].Seen); ago > d.cacheLifetime {
 			if debug {
-				l.Debugf("removing cached address %s: seen %v ago", c[i].addr, ago)
+				l.Debugf("removing cached address %s: seen %v ago", c[i].Address, ago)
 			}
 			c[i] = c[len(c)-1]
 			c = c[:len(c)-1]

internal/files/blockmap.go

@@ -0,0 +1,201 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
+
+// Package files provides a set type to track local/remote files with newness
+// checks. We must do a certain amount of normalization in here. We will get
+// fed paths with either native or wire-format separators and encodings
+// depending on who calls us. We transform paths to wire-format (NFC and
+// slashes) on the way to the database, and transform to native format
+// (varying separator and encoding) on the way back out.
+
+package files
+
+import (
+	"bytes"
+	"encoding/binary"
+	"sort"
+	"sync"
+
+	"github.com/syncthing/syncthing/internal/config"
+	"github.com/syncthing/syncthing/internal/protocol"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+var blockFinder *BlockFinder
+
+type BlockMap struct {
+	db     *leveldb.DB
+	folder string
+}
+
+func NewBlockMap(db *leveldb.DB, folder string) *BlockMap {
+	return &BlockMap{
+		db:     db,
+		folder: folder,
+	}
+}
+
+// Add files to the block map, ignoring any deleted or invalid files.
+func (m *BlockMap) Add(files []protocol.FileInfo) error {
+	batch := new(leveldb.Batch)
+	buf := make([]byte, 4)
+	for _, file := range files {
+		if file.IsDirectory() || file.IsDeleted() || file.IsInvalid() {
+			continue
+		}
+
+		for i, block := range file.Blocks {
+			binary.BigEndian.PutUint32(buf, uint32(i))
+			batch.Put(m.blockKey(block.Hash, file.Name), buf)
+		}
+	}
+	return m.db.Write(batch, nil)
+}
+
+// Update block map state, removing any deleted or invalid files.
+func (m *BlockMap) Update(files []protocol.FileInfo) error {
+	batch := new(leveldb.Batch)
+	buf := make([]byte, 4)
+	for _, file := range files {
+		if file.IsDirectory() {
+			continue
+		}
+
+		if file.IsDeleted() || file.IsInvalid() {
+			for _, block := range file.Blocks {
+				batch.Delete(m.blockKey(block.Hash, file.Name))
+			}
+			continue
+		}
+
+		for i, block := range file.Blocks {
+			binary.BigEndian.PutUint32(buf, uint32(i))
+			batch.Put(m.blockKey(block.Hash, file.Name), buf)
+		}
+	}
+	return m.db.Write(batch, nil)
+}
+
+// Drop block map, removing all entries related to this block map from the db.
+func (m *BlockMap) Drop() error {
+	batch := new(leveldb.Batch)
+	iter := m.db.NewIterator(util.BytesPrefix(m.blockKey(nil, "")[:1+64]), nil)
+	defer iter.Release()
+	for iter.Next() {
+		batch.Delete(iter.Key())
+	}
+	if iter.Error() != nil {
+		return iter.Error()
+	}
+	return m.db.Write(batch, nil)
+}
+
+func (m *BlockMap) blockKey(hash []byte, file string) []byte {
+	return toBlockKey(hash, m.folder, file)
+}
+
+type BlockFinder struct {
+	db      *leveldb.DB
+	folders []string
+	mut     sync.RWMutex
+}
+
+func NewBlockFinder(db *leveldb.DB, cfg *config.ConfigWrapper) *BlockFinder {
+	if blockFinder != nil {
+		return blockFinder
+	}
+
+	f := &BlockFinder{
+		db: db,
+	}
+	f.Changed(cfg.Raw())
+	cfg.Subscribe(f)
+	return f
+}
+
+// Implements config.Handler interface
+func (f *BlockFinder) Changed(cfg config.Configuration) error {
+	folders := make([]string, len(cfg.Folders))
+	for i, folder := range cfg.Folders {
+		folders[i] = folder.ID
+	}
+
+	sort.Strings(folders)
+
+	f.mut.Lock()
+	f.folders = folders
+	f.mut.Unlock()
+
+	return nil
+}
+
+// An iterator function which iterates over all matching blocks for the given
+// hash. The iterator function has to return either true (if they are happy with
+// the block) or false to continue iterating for whatever reason.
+// The iterator finally returns the result, whether or not a satisfying block
+// was eventually found.
+func (f *BlockFinder) Iterate(hash []byte, iterFn func(string, string, uint32) bool) bool {
+	f.mut.RLock()
+	folders := f.folders
+	f.mut.RUnlock()
+	for _, folder := range folders {
+		key := toBlockKey(hash, folder, "")
+		iter := f.db.NewIterator(util.BytesPrefix(key), nil)
+		defer iter.Release()
+
+		for iter.Next() && iter.Error() == nil {
+			folder, file := fromBlockKey(iter.Key())
+			index := binary.BigEndian.Uint32(iter.Value())
+			if iterFn(folder, nativeFilename(file), index) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// m.blockKey returns a byte slice encoding the following information:
+//	   keyTypeBlock (1 byte)
+//	   folder (64 bytes)
+//	   block hash (32 bytes)
+//	   file name (variable size)
+func toBlockKey(hash []byte, folder, file string) []byte {
+	o := make([]byte, 1+64+32+len(file))
+	o[0] = keyTypeBlock
+	copy(o[1:], []byte(folder))
+	copy(o[1+64:], []byte(hash))
+	copy(o[1+64+32:], []byte(file))
+	return o
+}
+
+func fromBlockKey(data []byte) (string, string) {
+	if len(data) < 1+64+32+1 {
+		panic("Incorrect key length")
+	}
+	if data[0] != keyTypeBlock {
+		panic("Incorrect key type")
+	}
+
+	file := string(data[1+64+32:])
+
+	slice := data[1 : 1+64]
+	izero := bytes.IndexByte(slice, 0)
+	if izero > -1 {
+		return string(slice[:izero]), file
+	}
+	return string(slice), file
+}

internal/files/blockmap_test.go

@@ -0,0 +1,235 @@
+// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package files
+
+import (
+	"testing"
+
+	"github.com/syncthing/syncthing/internal/config"
+	"github.com/syncthing/syncthing/internal/protocol"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/storage"
+)
+
+func genBlocks(n int) []protocol.BlockInfo {
+	b := make([]protocol.BlockInfo, n)
+	for i := range b {
+		h := make([]byte, 32)
+		for j := range h {
+			h[j] = byte(i + j)
+		}
+		b[i].Size = uint32(i)
+		b[i].Hash = h
+	}
+	return b
+}
+
+var f1, f2, f3 protocol.FileInfo
+
+func init() {
+	blocks := genBlocks(30)
+
+	f1 = protocol.FileInfo{
+		Name:   "f1",
+		Blocks: blocks[:10],
+	}
+
+	f2 = protocol.FileInfo{
+		Name:   "f2",
+		Blocks: blocks[10:20],
+	}
+
+	f3 = protocol.FileInfo{
+		Name:   "f3",
+		Blocks: blocks[20:],
+	}
+}
+
+func setup() (*leveldb.DB, *BlockFinder) {
+	// Setup
+
+	db, err := leveldb.Open(storage.NewMemStorage(), nil)
+	if err != nil {
+		panic(err)
+	}
+
+	wrapper := config.Wrap("", config.Configuration{})
+	wrapper.SetFolder(config.FolderConfiguration{
+		ID: "folder1",
+	})
+	wrapper.SetFolder(config.FolderConfiguration{
+		ID: "folder2",
+	})
+
+	return db, NewBlockFinder(db, wrapper)
+}
+
+func dbEmpty(db *leveldb.DB) bool {
+	iter := db.NewIterator(nil, nil)
+	defer iter.Release()
+	if iter.Next() {
+		return false
+	}
+	return true
+}
+
+func TestBlockMapAddUpdateWipe(t *testing.T) {
+	db, f := setup()
+
+	if !dbEmpty(db) {
+		t.Fatal("db not empty")
+	}
+
+	m := NewBlockMap(db, "folder1")
+
+	f3.Flags |= protocol.FlagDirectory
+
+	err := m.Add([]protocol.FileInfo{f1, f2, f3})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index uint32) bool {
+		if folder != "folder1" || file != "f1" || index != 0 {
+			t.Fatal("Mismatch")
+		}
+		return true
+	})
+
+	f.Iterate(f2.Blocks[0].Hash, func(folder, file string, index uint32) bool {
+		if folder != "folder1" || file != "f2" || index != 0 {
+			t.Fatal("Mismatch")
+		}
+		return true
+	})
+
+	f.Iterate(f3.Blocks[0].Hash, func(folder, file string, index uint32) bool {
+		t.Fatal("Unexpected block")
+		return true
+	})
+
+	f3.Flags = f1.Flags
+	f1.Flags |= protocol.FlagDeleted
+	f2.Flags |= protocol.FlagInvalid
+
+	// Should remove
+	err = m.Update([]protocol.FileInfo{f1, f2, f3})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index uint32) bool {
+		t.Fatal("Unexpected block")
+		return false
+	})
+
+	f.Iterate(f2.Blocks[0].Hash, func(folder, file string, index uint32) bool {
+		t.Fatal("Unexpected block")
+		return false
+	})
+
+	f.Iterate(f3.Blocks[0].Hash, func(folder, file string, index uint32) bool {
+		if folder != "folder1" || file != "f3" || index != 0 {
+			t.Fatal("Mismatch")
+		}
+		return true
+	})
+
+	err = m.Drop()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !dbEmpty(db) {
+		t.Fatal("db not empty")
+	}
+
+	// Should not add
+	err = m.Add([]protocol.FileInfo{f1, f2})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !dbEmpty(db) {
+		t.Fatal("db not empty")
+	}
+
+	f1.Flags = 0
+	f2.Flags = 0
+	f3.Flags = 0
+}
+
+func TestBlockMapFinderLookup(t *testing.T) {
+	db, f := setup()
+
+	m1 := NewBlockMap(db, "folder1")
+	m2 := NewBlockMap(db, "folder2")
+
+	err := m1.Add([]protocol.FileInfo{f1})
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = m2.Add([]protocol.FileInfo{f1})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	counter := 0
+	f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index uint32) bool {
+		counter++
+		switch counter {
+		case 1:
+			if folder != "folder1" || file != "f1" || index != 0 {
+				t.Fatal("Mismatch")
+			}
+		case 2:
+			if folder != "folder2" || file != "f1" || index != 0 {
+				t.Fatal("Mismatch")
+			}
+		default:
+			t.Fatal("Unexpected block")
+		}
+		return false
+	})
+	if counter != 2 {
+		t.Fatal("Incorrect count", counter)
+	}
+
+	f1.Flags |= protocol.FlagDeleted
+
+	err = m1.Update([]protocol.FileInfo{f1})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	counter = 0
+	f.Iterate(f1.Blocks[0].Hash, func(folder, file string, index uint32) bool {
+		counter++
+		switch counter {
+		case 1:
+			if folder != "folder2" || file != "f1" || index != 0 {
+				t.Fatal("Mismatch")
+			}
+		default:
+			t.Fatal("Unexpected block")
+		}
+		return false
+	})
+	if counter != 1 {
+		t.Fatal("Incorrect count")
+	}
+}

internal/files/leveldb.go

@@ -49,6 +49,7 @@ func clock(v uint64) uint64 {
 const (
 	keyTypeDevice = iota
 	keyTypeGlobal
+	keyTypeBlock
 )
 
 type fileVersion struct {

internal/files/set.go

@@ -42,6 +42,7 @@ type Set struct {
 	mutex        sync.Mutex
 	folder       string
 	db           *leveldb.DB
+	blockmap     *BlockMap
 }
 
 func NewSet(folder string, db *leveldb.DB) *Set {
@@ -49,6 +50,7 @@ func NewSet(folder string, db *leveldb.DB) *Set {
 		localVersion: make(map[protocol.DeviceID]uint64),
 		folder:       folder,
 		db:           db,
+		blockmap:     NewBlockMap(db, folder),
 	}
 
 	var deviceID protocol.DeviceID
@@ -80,6 +82,10 @@ func (s *Set) Replace(device protocol.DeviceID, fs []protocol.FileInfo) {
 		// Reset the local version if all files were removed.
 		s.localVersion[device] = 0
 	}
+	if device == protocol.LocalDeviceID {
+		s.blockmap.Drop()
+		s.blockmap.Add(fs)
+	}
 }
 
 func (s *Set) ReplaceWithDelete(device protocol.DeviceID, fs []protocol.FileInfo) {
@@ -92,6 +98,10 @@ func (s *Set) ReplaceWithDelete(device protocol.DeviceID, fs []protocol.FileInfo
 	if lv := ldbReplaceWithDelete(s.db, []byte(s.folder), device[:], fs); lv > s.localVersion[device] {
 		s.localVersion[device] = lv
 	}
+	if device == protocol.LocalDeviceID {
+		s.blockmap.Drop()
+		s.blockmap.Add(fs)
+	}
 }
 
 func (s *Set) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
@@ -104,6 +114,9 @@ func (s *Set) Update(device protocol.DeviceID, fs []protocol.FileInfo) {
 	if lv := ldbUpdate(s.db, []byte(s.folder), device[:], fs); lv > s.localVersion[device] {
 		s.localVersion[device] = lv
 	}
+	if device == protocol.LocalDeviceID {
+		s.blockmap.Update(fs)
+	}
 }
 
 func (s *Set) WithNeed(device protocol.DeviceID, fn fileIterator) {
@@ -179,6 +192,11 @@ func ListFolders(db *leveldb.DB) []string {
 // database.
 func DropFolder(db *leveldb.DB, folder string) {
 	ldbDropFolder(db, []byte(folder))
+	bm := &BlockMap{
+		db:     db,
+		folder: folder,
+	}
+	bm.Drop()
 }
 
 func normalizeFilenames(fs []protocol.FileInfo) {

internal/ignore/ignore.go

@@ -23,31 +23,94 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
+	"sync"
 
 	"github.com/syncthing/syncthing/internal/fnmatch"
 )
 
+var caches = make(map[string]MatcherCache)
+
 type Pattern struct {
 	match   *regexp.Regexp
 	include bool
 }
 
-type Patterns []Pattern
+type Matcher struct {
+	patterns   []Pattern
+	oldMatches map[string]bool
 
-func Load(file string) (Patterns, error) {
-	seen := make(map[string]bool)
-	return loadIgnoreFile(file, seen)
+	newMatches map[string]bool
+	mut        sync.Mutex
 }
 
-func Parse(r io.Reader, file string) (Patterns, error) {
+type MatcherCache struct {
+	patterns []Pattern
+	matches  *map[string]bool
+}
+
+func Load(file string, cache bool) (*Matcher, error) {
+	seen := make(map[string]bool)
+	matcher, err := loadIgnoreFile(file, seen)
+	if !cache || err != nil {
+		return matcher, err
+	}
+
+	// Get the current cache object for the given file
+	cached, ok := caches[file]
+	if !ok || !patternsEqual(cached.patterns, matcher.patterns) {
+		// Nothing in cache or a cache mismatch, create a new cache which will
+		// store matches for the given set of patterns.
+		// Initialize oldMatches to indicate that we are interested in
+		// caching.
+		matcher.oldMatches = make(map[string]bool)
+		matcher.newMatches = make(map[string]bool)
+		caches[file] = MatcherCache{
+			patterns: matcher.patterns,
+			matches:  &matcher.newMatches,
+		}
+		return matcher, nil
+	}
+
+	// Patterns haven't changed, so we can reuse the old matches, create a new
+	// matches map and update the pointer. (This prevents matches map from
+	// growing indefinately, as we only cache whatever we've matched in the last
+	// iteration, rather than through runtime history)
+	matcher.oldMatches = *cached.matches
+	matcher.newMatches = make(map[string]bool)
+	cached.matches = &matcher.newMatches
+	caches[file] = cached
+	return matcher, nil
+}
+
+func Parse(r io.Reader, file string) (*Matcher, error) {
 	seen := map[string]bool{
 		file: true,
 	}
 	return parseIgnoreFile(r, file, seen)
 }
 
-func (l Patterns) Match(file string) bool {
-	for _, pattern := range l {
+func (m *Matcher) Match(file string) (result bool) {
+	if len(m.patterns) == 0 {
+		return false
+	}
+
+	// We have old matches map set, means we should do caching
+	if m.oldMatches != nil {
+		// Capture the result to the new matches regardless of who returns it
+		defer func() {
+			m.mut.Lock()
+			m.newMatches[file] = result
+			m.mut.Unlock()
+		}()
+		// Check perhaps we've seen this file before, and we already know
+		// what the outcome is going to be.
+		result, ok := m.oldMatches[file]
+		if ok {
+			return result
+		}
+	}
+
+	for _, pattern := range m.patterns {
 		if pattern.match.MatchString(file) {
 			return pattern.include
 		}
@@ -55,7 +118,7 @@ func (l Patterns) Match(file string) bool {
 	return false
 }
 
-func loadIgnoreFile(file string, seen map[string]bool) (Patterns, error) {
+func loadIgnoreFile(file string, seen map[string]bool) (*Matcher, error) {
 	if seen[file] {
 		return nil, fmt.Errorf("Multiple include of ignore file %q", file)
 	}
@@ -70,8 +133,8 @@ func loadIgnoreFile(file string, seen map[string]bool) (Patterns, error) {
 	return parseIgnoreFile(fd, file, seen)
 }
 
-func parseIgnoreFile(fd io.Reader, currentFile string, seen map[string]bool) (Patterns, error) {
-	var exps Patterns
+func parseIgnoreFile(fd io.Reader, currentFile string, seen map[string]bool) (*Matcher, error) {
+	var exps Matcher
 
 	addPattern := func(line string) error {
 		include := true
@@ -86,27 +149,27 @@ func parseIgnoreFile(fd io.Reader, currentFile string, seen map[string]bool) (Pa
 			if err != nil {
 				return fmt.Errorf("Invalid pattern %q in ignore file", line)
 			}
-			exps = append(exps, Pattern{exp, include})
+			exps.patterns = append(exps.patterns, Pattern{exp, include})
 		} else if strings.HasPrefix(line, "**/") {
 			// Add the pattern as is, and without **/ so it matches in current dir
 			exp, err := fnmatch.Convert(line, fnmatch.FNM_PATHNAME)
 			if err != nil {
 				return fmt.Errorf("Invalid pattern %q in ignore file", line)
 			}
-			exps = append(exps, Pattern{exp, include})
+			exps.patterns = append(exps.patterns, Pattern{exp, include})
 
 			exp, err = fnmatch.Convert(line[3:], fnmatch.FNM_PATHNAME)
 			if err != nil {
 				return fmt.Errorf("Invalid pattern %q in ignore file", line)
 			}
-			exps = append(exps, Pattern{exp, include})
+			exps.patterns = append(exps.patterns, Pattern{exp, include})
 		} else if strings.HasPrefix(line, "#include ") {
 			includeFile := filepath.Join(filepath.Dir(currentFile), line[len("#include "):])
 			includes, err := loadIgnoreFile(includeFile, seen)
 			if err != nil {
 				return err
 			} else {
-				exps = append(exps, includes...)
+				exps.patterns = append(exps.patterns, includes.patterns...)
 			}
 		} else {
 			// Path name or pattern, add it so it matches files both in
@@ -115,13 +178,13 @@ func parseIgnoreFile(fd io.Reader, currentFile string, seen map[string]bool) (Pa
 			if err != nil {
 				return fmt.Errorf("Invalid pattern %q in ignore file", line)
 			}
-			exps = append(exps, Pattern{exp, include})
+			exps.patterns = append(exps.patterns, Pattern{exp, include})
 
 			exp, err = fnmatch.Convert("**/"+line, fnmatch.FNM_PATHNAME)
 			if err != nil {
 				return fmt.Errorf("Invalid pattern %q in ignore file", line)
 			}
-			exps = append(exps, Pattern{exp, include})
+			exps.patterns = append(exps.patterns, Pattern{exp, include})
 		}
 		return nil
 	}
@@ -155,5 +218,17 @@ func parseIgnoreFile(fd io.Reader, currentFile string, seen map[string]bool) (Pa
 		}
 	}
 
-	return exps, nil
+	return &exps, nil
+}
+
+func patternsEqual(a, b []Pattern) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if a[i].include != b[i].include || a[i].match.String() != b[i].match.String() {
+			return false
+		}
+	}
+	return true
 }

internal/ignore/ignore_test.go

@@ -13,19 +13,19 @@
 // You should have received a copy of the GNU General Public License along
 // with this program. If not, see <http://www.gnu.org/licenses/>.
 
-package ignore_test
+package ignore
 
 import (
 	"bytes"
+	"io/ioutil"
+	"os"
 	"path/filepath"
 	"runtime"
 	"testing"
-
-	"github.com/syncthing/syncthing/internal/ignore"
 )
 
 func TestIgnore(t *testing.T) {
-	pats, err := ignore.Load("testdata/.stignore")
+	pats, err := Load("testdata/.stignore", true)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -72,7 +72,7 @@ func TestExcludes(t *testing.T) {
 	i*2
 	!ign2
 	`
-	pats, err := ignore.Parse(bytes.NewBufferString(stignore), ".stignore")
+	pats, err := Parse(bytes.NewBufferString(stignore), ".stignore")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -112,7 +112,7 @@ func TestBadPatterns(t *testing.T) {
 	}
 
 	for _, pat := range badPatterns {
-		parsed, err := ignore.Parse(bytes.NewBufferString(pat), ".stignore")
+		parsed, err := Parse(bytes.NewBufferString(pat), ".stignore")
 		if err == nil {
 			t.Errorf("No error for pattern %q: %v", pat, parsed)
 		}
@@ -120,7 +120,7 @@ func TestBadPatterns(t *testing.T) {
 }
 
 func TestCaseSensitivity(t *testing.T) {
-	ign, _ := ignore.Parse(bytes.NewBufferString("test"), ".stignore")
+	ign, _ := Parse(bytes.NewBufferString("test"), ".stignore")
 
 	match := []string{"test"}
 	dontMatch := []string{"foo"}
@@ -145,6 +145,144 @@ func TestCaseSensitivity(t *testing.T) {
 	}
 }
 
+func TestCaching(t *testing.T) {
+	fd1, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fd2, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer fd1.Close()
+	defer fd2.Close()
+	defer os.Remove(fd1.Name())
+	defer os.Remove(fd2.Name())
+
+	_, err = fd1.WriteString("/x/\n#include " + filepath.Base(fd2.Name()) + "\n")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fd2.WriteString("/y/\n")
+
+	pats, err := Load(fd1.Name(), true)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if pats.oldMatches == nil || len(pats.oldMatches) != 0 {
+		t.Fatal("Expected empty map")
+	}
+
+	if pats.newMatches == nil || len(pats.newMatches) != 0 {
+		t.Fatal("Expected empty map")
+	}
+
+	if len(pats.patterns) != 4 {
+		t.Fatal("Incorrect number of patterns loaded", len(pats.patterns), "!=", 4)
+	}
+
+	// Cache some outcomes
+
+	for _, letter := range []string{"a", "b", "x", "y"} {
+		pats.Match(letter)
+	}
+
+	if len(pats.newMatches) != 4 {
+		t.Fatal("Expected 4 cached results")
+	}
+
+	// Reload file, expect old outcomes to be provided
+
+	pats, err = Load(fd1.Name(), true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(pats.oldMatches) != 4 {
+		t.Fatal("Expected 4 cached results")
+	}
+
+	// Match less this time
+
+	for _, letter := range []string{"b", "x", "y"} {
+		pats.Match(letter)
+	}
+
+	if len(pats.newMatches) != 3 {
+		t.Fatal("Expected 3 cached results")
+	}
+
+	// Reload file, expect the new outcomes to be provided
+
+	pats, err = Load(fd1.Name(), true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(pats.oldMatches) != 3 {
+		t.Fatal("Expected 3 cached results", len(pats.oldMatches))
+	}
+
+	// Modify the include file, expect empty cache
+
+	fd2.WriteString("/z/\n")
+
+	pats, err = Load(fd1.Name(), true)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(pats.oldMatches) != 0 {
+		t.Fatal("Expected 0 cached results")
+	}
+
+	// Cache some outcomes again
+
+	for _, letter := range []string{"b", "x", "y"} {
+		pats.Match(letter)
+	}
+
+	// Verify that outcomes provided on next laod
+
+	pats, err = Load(fd1.Name(), true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(pats.oldMatches) != 3 {
+		t.Fatal("Expected 3 cached results")
+	}
+
+	// Modify the root file, expect cache to be invalidated
+
+	fd1.WriteString("/a/\n")
+
+	pats, err = Load(fd1.Name(), true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(pats.oldMatches) != 0 {
+		t.Fatal("Expected cache invalidation")
+	}
+
+	// Cache some outcomes again
+
+	for _, letter := range []string{"b", "x", "y"} {
+		pats.Match(letter)
+	}
+
+	// Verify that outcomes provided on next laod
+
+	pats, err = Load(fd1.Name(), true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(pats.oldMatches) != 3 {
+		t.Fatal("Expected 3 cached results")
+	}
+}
+
 func TestCommentsAndBlankLines(t *testing.T) {
 	stignore := `
 	// foo
@@ -157,8 +295,83 @@ func TestCommentsAndBlankLines(t *testing.T) {
 
 
 	`
-	pats, _ := ignore.Parse(bytes.NewBufferString(stignore), ".stignore")
-	if len(pats) > 0 {
+	pats, _ := Parse(bytes.NewBufferString(stignore), ".stignore")
+	if len(pats.patterns) > 0 {
 		t.Errorf("Expected no patterns")
 	}
 }
+
+var result bool
+
+func BenchmarkMatch(b *testing.B) {
+	stignore := `
+.frog
+.frog*
+.frogfox
+.whale
+.whale/*
+.dolphin
+.dolphin/*
+~ferret~.*
+.ferret.*
+flamingo.*
+flamingo
+*.crow
+*.crow
+	`
+	pats, _ := Parse(bytes.NewBufferString(stignore), ".stignore")
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		result = pats.Match("filename")
+	}
+}
+
+func BenchmarkMatchCached(b *testing.B) {
+	stignore := `
+.frog
+.frog*
+.frogfox
+.whale
+.whale/*
+.dolphin
+.dolphin/*
+~ferret~.*
+.ferret.*
+flamingo.*
+flamingo
+*.crow
+*.crow
+	`
+	// Caches per file, hence write the patterns to a file.
+	fd, err := ioutil.TempFile("", "")
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	_, err = fd.WriteString(stignore)
+	defer fd.Close()
+	defer os.Remove(fd.Name())
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// Load the patterns
+	pats, err := Load(fd.Name(), true)
+	if err != nil {
+		b.Fatal(err)
+	}
+	// Cache the outcome for "filename"
+	pats.Match("filename")
+
+	// This load should now load the cached outcomes as the set of patterns
+	// has not changed.
+	pats, err = Load(fd.Name(), true)
+	if err != nil {
+		b.Fatal(err)
+	}
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		result = pats.Match("filename")
+	}
+}

internal/model/model.go

@@ -81,8 +81,9 @@ type service interface {
 }
 
 type Model struct {
-	cfg *config.ConfigWrapper
-	db  *leveldb.DB
+	cfg    *config.ConfigWrapper
+	db     *leveldb.DB
+	finder *files.BlockFinder
 
 	deviceName    string
 	clientName    string
@@ -93,7 +94,7 @@ type Model struct {
 	folderDevices  map[string][]protocol.DeviceID                         // folder -> deviceIDs
 	deviceFolders  map[protocol.DeviceID][]string                         // deviceID -> folders
 	deviceStatRefs map[protocol.DeviceID]*stats.DeviceStatisticsReference // deviceID -> statsRef
-	folderIgnores  map[string]ignore.Patterns                             // folder -> list of ignore patterns
+	folderIgnores  map[string]*ignore.Matcher                             // folder -> matcher object
 	folderRunners  map[string]service                                     // folder -> puller or scanner
 	fmut           sync.RWMutex                                           // protects the above
 
@@ -130,13 +131,14 @@ func NewModel(cfg *config.ConfigWrapper, deviceName, clientName, clientVersion s
 		folderDevices:      make(map[string][]protocol.DeviceID),
 		deviceFolders:      make(map[protocol.DeviceID][]string),
 		deviceStatRefs:     make(map[protocol.DeviceID]*stats.DeviceStatisticsReference),
-		folderIgnores:      make(map[string]ignore.Patterns),
+		folderIgnores:      make(map[string]*ignore.Matcher),
 		folderRunners:      make(map[string]service),
 		folderState:        make(map[string]folderState),
 		folderStateChanged: make(map[string]time.Time),
 		protoConn:          make(map[protocol.DeviceID]protocol.Connection),
 		rawConn:            make(map[protocol.DeviceID]io.Closer),
 		deviceVer:          make(map[protocol.DeviceID]string),
+		finder:             files.NewBlockFinder(db, cfg),
 	}
 
 	var timeout = 20 * 60 // seconds
@@ -167,11 +169,12 @@ func (m *Model) StartFolderRW(folder string) {
 		panic("cannot start already running folder " + folder)
 	}
 	p := &Puller{
-		folder:      folder,
-		dir:         cfg.Path,
-		scanIntv:    time.Duration(cfg.RescanIntervalS) * time.Second,
-		model:       m,
-		ignorePerms: cfg.IgnorePerms,
+		folder:        folder,
+		dir:           cfg.Path,
+		scanIntv:      time.Duration(cfg.RescanIntervalS) * time.Second,
+		model:         m,
+		ignorePerms:   cfg.IgnorePerms,
+		lenientMtimes: cfg.LenientMtimes,
 	}
 	m.folderRunners[folder] = p
 	m.fmut.Unlock()
@@ -184,6 +187,10 @@ func (m *Model) StartFolderRW(folder string) {
 		p.versioner = factory(folder, cfg.Path, cfg.Versioning.Params)
 	}
 
+	if cfg.LenientMtimes {
+		l.Infof("Folder %q is running with LenientMtimes workaround. Syncing may not work properly.", folder)
+	}
+
 	go p.Serve()
 }
 
@@ -266,6 +273,8 @@ func (m *Model) DeviceStatistics() map[string]stats.DeviceStatistics {
 
 // Returns the completion status, in percent, for the given device and folder.
 func (m *Model) Completion(device protocol.DeviceID, folder string) float64 {
+	defer m.leveldbPanicWorkaround()
+
 	var tot int64
 
 	m.fmut.RLock()
@@ -325,6 +334,8 @@ func sizeOfFile(f protocol.FileIntf) (files, deleted int, bytes int64) {
 // GlobalSize returns the number of files, deleted files and total bytes for all
 // files in the global model.
 func (m *Model) GlobalSize(folder string) (files, deleted int, bytes int64) {
+	defer m.leveldbPanicWorkaround()
+
 	m.fmut.RLock()
 	defer m.fmut.RUnlock()
 	if rf, ok := m.folderFiles[folder]; ok {
@@ -342,6 +353,8 @@ func (m *Model) GlobalSize(folder string) (files, deleted int, bytes int64) {
 // LocalSize returns the number of files, deleted files and total bytes for all
 // files in the local folder.
 func (m *Model) LocalSize(folder string) (files, deleted int, bytes int64) {
+	defer m.leveldbPanicWorkaround()
+
 	m.fmut.RLock()
 	defer m.fmut.RUnlock()
 	if rf, ok := m.folderFiles[folder]; ok {
@@ -361,6 +374,8 @@ func (m *Model) LocalSize(folder string) (files, deleted int, bytes int64) {
 
 // NeedSize returns the number and total size of currently needed files.
 func (m *Model) NeedSize(folder string) (files int, bytes int64) {
+	defer m.leveldbPanicWorkaround()
+
 	m.fmut.RLock()
 	defer m.fmut.RUnlock()
 	if rf, ok := m.folderFiles[folder]; ok {
@@ -380,6 +395,8 @@ func (m *Model) NeedSize(folder string) (files int, bytes int64) {
 // NeedFiles returns the list of currently needed files, stopping at maxFiles
 // files or maxBlocks blocks. Limits <= 0 are ignored.
 func (m *Model) NeedFolderFilesLimited(folder string, maxFiles, maxBlocks int) []protocol.FileInfo {
+	defer m.leveldbPanicWorkaround()
+
 	m.fmut.RLock()
 	defer m.fmut.RUnlock()
 	nblocks := 0
@@ -423,7 +440,10 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F
 
 	for i := 0; i < len(fs); {
 		lamport.Default.Tick(fs[i].Version)
-		if ignores.Match(fs[i].Name) {
+		if ignores != nil && ignores.Match(fs[i].Name) {
+			if debug {
+				l.Debugln("dropping update for ignored", fs[i])
+			}
 			fs[i] = fs[len(fs)-1]
 			fs = fs[:len(fs)-1]
 		} else {
@@ -464,7 +484,10 @@ func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []prot
 
 	for i := 0; i < len(fs); {
 		lamport.Default.Tick(fs[i].Version)
-		if ignores.Match(fs[i].Name) {
+		if ignores != nil && ignores.Match(fs[i].Name) {
+			if debug {
+				l.Debugln("dropping update for ignored", fs[i])
+			}
 			fs[i] = fs[len(fs)-1]
 			fs = fs[:len(fs)-1]
 		} else {
@@ -538,6 +561,7 @@ func (m *Model) ClusterConfig(deviceID protocol.DeviceID, cm protocol.ClusterCon
 					newDeviceCfg := config.DeviceConfiguration{
 						DeviceID:    id,
 						Compression: true,
+						Addresses:   []string{"dynamic"},
 					}
 
 					// The introducers' introducers are also our introducers.
@@ -819,7 +843,7 @@ func (m *Model) deviceWasSeen(deviceID protocol.DeviceID) {
 	m.deviceStatRef(deviceID).WasSeen()
 }
 
-func sendIndexes(conn protocol.Connection, folder string, fs *files.Set, ignores ignore.Patterns) {
+func sendIndexes(conn protocol.Connection, folder string, fs *files.Set, ignores *ignore.Matcher) {
 	deviceID := conn.ID()
 	name := conn.Name()
 	var err error
@@ -844,7 +868,7 @@ func sendIndexes(conn protocol.Connection, folder string, fs *files.Set, ignores
 	}
 }
 
-func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, folder string, fs *files.Set, ignores ignore.Patterns) (uint64, error) {
+func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, folder string, fs *files.Set, ignores *ignore.Matcher) (uint64, error) {
 	deviceID := conn.ID()
 	name := conn.Name()
 	batch := make([]protocol.FileInfo, 0, indexBatchSize)
@@ -862,7 +886,10 @@ func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, fol
 			maxLocalVer = f.LocalVersion
 		}
 
-		if ignores.Match(f.Name) {
+		if ignores != nil && ignores.Match(f.Name) {
+			if debug {
+				l.Debugln("not sending update for ignored", f)
+			}
 			return true
 		}
 
@@ -996,13 +1023,13 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
 	fs, ok := m.folderFiles[folder]
 	dir := m.folderCfgs[folder].Path
 
-	ignores, _ := ignore.Load(filepath.Join(dir, ".stignore"))
+	ignores, _ := ignore.Load(filepath.Join(dir, ".stignore"), m.cfg.Options().CacheIgnoredFiles)
 	m.folderIgnores[folder] = ignores
 
 	w := &scanner.Walker{
 		Dir:          dir,
 		Sub:          sub,
-		Ignores:      ignores,
+		Matcher:      ignores,
 		BlockSize:    protocol.BlockSize,
 		TempNamer:    defTempNamer,
 		CurrentFiler: cFiler{m, folder},
@@ -1062,8 +1089,9 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
 				batch = batch[:0]
 			}
 
-			if ignores.Match(f.Name) {
+			if ignores != nil && ignores.Match(f.Name) {
 				// File has been ignored. Set invalid bit.
+				l.Debugln("setting invalid bit on ignored", f)
 				nf := protocol.FileInfo{
 					Name:     f.Name,
 					Flags:    f.Flags | protocol.FlagInvalid,
@@ -1259,3 +1287,24 @@ func (m *Model) availability(folder string, file string) []protocol.DeviceID {
 func (m *Model) String() string {
 	return fmt.Sprintf("model@%p", m)
 }
+
+func (m *Model) leveldbPanicWorkaround() {
+	// When an inconsistency is detected in leveldb we panic(). This is
+	// appropriate because it should never happen, but currently it does for
+	// some reason. However it only seems to trigger in the asynchronous full-
+	// database scans that happen due to REST and usage-reporting calls. In
+	// those places we defer to this workaround to catch the panic instead of
+	// taking down syncthing.
+
+	// This is just a band-aid and should be removed as soon as we have found
+	// a real root cause.
+
+	if pnc := recover(); pnc != nil {
+		if err, ok := pnc.(error); ok && strings.Contains(err.Error(), "leveldb") {
+			l.Infoln("recovered:", err)
+		} else {
+			// Any non-leveldb error is genuine and should continue panicing.
+			panic(err)
+		}
+	}
+}

internal/model/model_test.go

@@ -390,8 +390,12 @@ func TestIgnores(t *testing.T) {
 	}
 
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
-	m := NewModel(nil, "device", "syncthing", "dev", db)
-	m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"})
+	fcfg := config.FolderConfiguration{ID: "default", Path: "testdata"}
+	cfg := config.Wrap("/tmp", config.Configuration{
+		Folders: []config.FolderConfiguration{fcfg},
+	})
+	m := NewModel(cfg, "device", "syncthing", "dev", db)
+	m.AddFolder(fcfg)
 
 	expected := []string{
 		".*",

internal/model/puller.go

@@ -16,6 +16,7 @@
 package model
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"os"
@@ -23,6 +24,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/AudriusButkevicius/lfu-go"
+
 	"github.com/syncthing/syncthing/internal/config"
 	"github.com/syncthing/syncthing/internal/events"
 	"github.com/syncthing/syncthing/internal/osutil"
@@ -50,7 +53,7 @@ type pullBlockState struct {
 }
 
 // A copyBlocksState is passed to copy routine if the file has blocks to be
-// copied from the original.
+// copied.
 type copyBlocksState struct {
 	*sharedPullerState
 	blocks []protocol.BlockInfo
@@ -62,13 +65,14 @@ var (
 )
 
 type Puller struct {
-	folder      string
-	dir         string
-	scanIntv    time.Duration
-	model       *Model
-	stop        chan struct{}
-	versioner   versioner.Versioner
-	ignorePerms bool
+	folder        string
+	dir           string
+	scanIntv      time.Duration
+	model         *Model
+	stop          chan struct{}
+	versioner     versioner.Versioner
+	ignorePerms   bool
+	lenientMtimes bool
 }
 
 // Serve will run scans and pulls. It will return when Stop()ed or on a
@@ -83,10 +87,12 @@ func (p *Puller) Serve() {
 
 	pullTimer := time.NewTimer(checkPullIntv)
 	scanTimer := time.NewTimer(time.Millisecond) // The first scan should be done immediately.
+	cleanTimer := time.NewTicker(time.Hour)
 
 	defer func() {
 		pullTimer.Stop()
 		scanTimer.Stop()
+		cleanTimer.Stop()
 		// TODO: Should there be an actual FolderStopped state?
 		p.model.setState(p.folder, FolderIdle)
 	}()
@@ -160,6 +166,9 @@ loop:
 						curVer = lv
 					}
 					prevVer = curVer
+					if debug {
+						l.Debugln(p, "next pull in", nextPullIntv)
+					}
 					pullTimer.Reset(nextPullIntv)
 					break
 				}
@@ -170,6 +179,9 @@ loop:
 					// errors preventing us. Flag this with a warning and
 					// wait a bit longer before retrying.
 					l.Warnf("Folder %q isn't making progress - check logs for possible root cause. Pausing puller for %v.", p.folder, pauseIntv)
+					if debug {
+						l.Debugln(p, "next pull in", pauseIntv)
+					}
 					pullTimer.Reset(pauseIntv)
 					break
 				}
@@ -189,11 +201,20 @@ loop:
 				break loop
 			}
 			p.model.setState(p.folder, FolderIdle)
-			scanTimer.Reset(p.scanIntv)
+			if p.scanIntv > 0 {
+				if debug {
+					l.Debugln(p, "next rescan in", p.scanIntv)
+				}
+				scanTimer.Reset(p.scanIntv)
+			}
 			if !initialScanCompleted {
 				l.Infoln("Completed initial scan (rw) of folder", p.folder)
 				initialScanCompleted = true
 			}
+
+		// Clean out old temporaries
+		case <-cleanTimer.C:
+			p.clean()
 		}
 	}
 }
@@ -218,24 +239,25 @@ func (p *Puller) pullerIteration(ncopiers, npullers, nfinishers int) int {
 	copyChan := make(chan copyBlocksState)
 	finisherChan := make(chan *sharedPullerState)
 
-	var wg sync.WaitGroup
+	var copyWg sync.WaitGroup
+	var pullWg sync.WaitGroup
 	var doneWg sync.WaitGroup
 
 	for i := 0; i < ncopiers; i++ {
-		wg.Add(1)
+		copyWg.Add(1)
 		go func() {
 			// copierRoutine finishes when copyChan is closed
-			p.copierRoutine(copyChan, finisherChan)
-			wg.Done()
+			p.copierRoutine(copyChan, pullChan, finisherChan)
+			copyWg.Done()
 		}()
 	}
 
 	for i := 0; i < npullers; i++ {
-		wg.Add(1)
+		pullWg.Add(1)
 		go func() {
 			// pullerRoutine finishes when pullChan is closed
 			p.pullerRoutine(pullChan, finisherChan)
-			wg.Done()
+			pullWg.Done()
 		}()
 	}
 
@@ -259,6 +281,9 @@ func (p *Puller) pullerIteration(ncopiers, npullers, nfinishers int) int {
 	// !!!
 
 	changed := 0
+
+	var deletions []protocol.FileInfo
+
 	files.WithNeed(protocol.LocalDeviceID, func(intf protocol.FileIntf) bool {
 
 		// Needed items are delivered sorted lexicographically. This isn't
@@ -280,19 +305,16 @@ func (p *Puller) pullerIteration(ncopiers, npullers, nfinishers int) int {
 		}
 
 		switch {
-		case protocol.IsDirectory(file.Flags) && protocol.IsDeleted(file.Flags):
-			// A deleted directory
-			p.deleteDir(file)
+		case protocol.IsDeleted(file.Flags):
+			// A deleted file or directory
+			deletions = append(deletions, file)
 		case protocol.IsDirectory(file.Flags):
 			// A new or changed directory
 			p.handleDir(file)
-		case protocol.IsDeleted(file.Flags):
-			// A deleted file
-			p.deleteFile(file)
 		default:
 			// A new or changed file. This is the only case where we do stuff
 			// in the background; the other three are done synchronously.
-			p.handleFile(file, copyChan, pullChan, finisherChan)
+			p.handleFile(file, copyChan, finisherChan)
 		}
 
 		changed++
@@ -300,18 +322,27 @@ func (p *Puller) pullerIteration(ncopiers, npullers, nfinishers int) int {
 	})
 
 	// Signal copy and puller routines that we are done with the in data for
-	// this iteration
+	// this iteration. Wait for them to finish.
 	close(copyChan)
+	copyWg.Wait()
 	close(pullChan)
+	pullWg.Wait()
 
-	// Wait for them to finish, then signal the finisher chan that there will
-	// be no more input.
-	wg.Wait()
+	// Signal the finisher chan that there will be no more input.
 	close(finisherChan)
 
 	// Wait for the finisherChan to finish.
 	doneWg.Wait()
 
+	for i := range deletions {
+		deletion := deletions[len(deletions)-i-1]
+		if deletion.IsDirectory() {
+			p.deleteDir(deletion)
+		} else {
+			p.deleteFile(deletion)
+		}
+	}
+
 	return changed
 }
 
@@ -401,11 +432,15 @@ func (p *Puller) deleteFile(file protocol.FileInfo) {
 
 // handleFile queues the copies and pulls as necessary for a single new or
 // changed file.
-func (p *Puller) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksState, pullChan chan<- pullBlockState, finisherChan chan<- *sharedPullerState) {
+func (p *Puller) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksState, finisherChan chan<- *sharedPullerState) {
 	curFile := p.model.CurrentFolderFile(p.folder, file.Name)
-	copyBlocks, pullBlocks := scanner.BlockDiff(curFile.Blocks, file.Blocks)
 
-	if len(copyBlocks) == len(curFile.Blocks) && len(pullBlocks) == 0 {
+	if len(curFile.Blocks) == len(file.Blocks) {
+		for i := range file.Blocks {
+			if !bytes.Equal(curFile.Blocks[i].Hash, file.Blocks[i].Hash) {
+				goto FilesAreDifferent
+			}
+		}
 		// We are supposed to copy the entire file, and then fetch nothing. We
 		// are only updating metadata, so we don't actually *need* to make the
 		// copy.
@@ -416,11 +451,16 @@ func (p *Puller) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksSt
 		return
 	}
 
+FilesAreDifferent:
+
+	scanner.PopulateOffsets(file.Blocks)
+
 	// Figure out the absolute filenames we need once and for all
 	tempName := filepath.Join(p.dir, defTempNamer.TempName(file.Name))
 	realName := filepath.Join(p.dir, file.Name)
 
-	var reuse bool
+	reused := 0
+	var blocks []protocol.BlockInfo
 
 	// Check for an old temporary file which might have some blocks we could
 	// reuse.
@@ -435,38 +475,25 @@ func (p *Puller) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksSt
 			existingBlocks[block.String()] = true
 		}
 
-		// Since the blocks are already there, we don't need to copy them
-		// nor we need to pull them, hence discard blocks which are already
-		// there, if they are exactly the same...
-		var newCopyBlocks []protocol.BlockInfo
-		for _, block := range copyBlocks {
+		// Since the blocks are already there, we don't need to get them.
+		for _, block := range file.Blocks {
 			_, ok := existingBlocks[block.String()]
 			if !ok {
-				newCopyBlocks = append(newCopyBlocks, block)
+				blocks = append(blocks, block)
 			}
 		}
 
-		var newPullBlocks []protocol.BlockInfo
-		for _, block := range pullBlocks {
-			_, ok := existingBlocks[block.String()]
-			if !ok {
-				newPullBlocks = append(newPullBlocks, block)
-			}
-		}
-
-		// If any blocks could be reused, let the sharedpullerstate know
-		// which flags it is expected to set on the file.
-		// Also update the list of work for the routines.
-		if len(copyBlocks) != len(newCopyBlocks) || len(pullBlocks) != len(newPullBlocks) {
-			reuse = true
-			copyBlocks = newCopyBlocks
-			pullBlocks = newPullBlocks
-		} else {
+		// The sharedpullerstate will know which flags to use when opening the
+		// temp file depending if we are reusing any blocks or not.
+		reused = len(file.Blocks) - len(blocks)
+		if reused == 0 {
 			// Otherwise, discard the file ourselves in order for the
 			// sharedpuller not to panic when it fails to exlusively create a
 			// file which already exists
 			os.Remove(tempName)
 		}
+	} else {
+		blocks = file.Blocks
 	}
 
 	s := sharedPullerState{
@@ -474,43 +501,20 @@ func (p *Puller) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksSt
 		folder:     p.folder,
 		tempName:   tempName,
 		realName:   realName,
-		pullNeeded: len(pullBlocks),
-		reuse:      reuse,
-	}
-	if len(copyBlocks) > 0 {
-		s.copyNeeded = 1
+		copyTotal:  len(blocks),
+		copyNeeded: len(blocks),
+		reused:     reused,
 	}
 
 	if debug {
-		l.Debugf("%v need file %s; copy %d, pull %d, reuse %v", p, file.Name, len(copyBlocks), len(pullBlocks), reuse)
+		l.Debugf("%v need file %s; copy %d, reused %v", p, file.Name, len(blocks), reused)
 	}
 
-	if len(copyBlocks) > 0 {
-		cs := copyBlocksState{
-			sharedPullerState: &s,
-			blocks:            copyBlocks,
-		}
-		copyChan <- cs
-	}
-
-	if len(pullBlocks) > 0 {
-		for _, block := range pullBlocks {
-			ps := pullBlockState{
-				sharedPullerState: &s,
-				block:             block,
-			}
-			pullChan <- ps
-		}
-	}
-
-	if len(pullBlocks) == 0 && len(copyBlocks) == 0 {
-		if !reuse {
-			panic("bug: nothing to do with file?")
-		}
-		// We have a temp file that we can reuse totally. Jump directly to the
-		// finisher stage.
-		finisherChan <- &s
+	cs := copyBlocksState{
+		sharedPullerState: &s,
+		blocks:            blocks,
 	}
+	copyChan <- cs
 }
 
 // shortcutFile sets file mode and modification time, when that's the only
@@ -528,16 +532,24 @@ func (p *Puller) shortcutFile(file protocol.FileInfo) {
 	t := time.Unix(file.Modified, 0)
 	err := os.Chtimes(realName, t, t)
 	if err != nil {
-		l.Infof("Puller (folder %q, file %q): shortcut: %v", p.folder, file.Name, err)
-		return
+		if p.lenientMtimes {
+			// We accept the failure with a warning here and allow the sync to
+			// continue. We'll sync the new mtime back to the other devices later.
+			// If they have the same problem & setting, we might never get in
+			// sync.
+			l.Infof("Puller (folder %q, file %q): shortcut: %v (continuing anyway as requested)", p.folder, file.Name, err)
+		} else {
+			l.Infof("Puller (folder %q, file %q): shortcut: %v", p.folder, file.Name, err)
+			return
+		}
 	}
 
 	p.model.updateLocal(p.folder, file)
 }
 
-// copierRoutine reads pullerStates until the in channel closes and performs
-// the relevant copy.
-func (p *Puller) copierRoutine(in <-chan copyBlocksState, out chan<- *sharedPullerState) {
+// copierRoutine reads copierStates until the in channel closes and performs
+// the relevant copies when possible, or passes it to the puller routine.
+func (p *Puller) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pullBlockState, out chan<- *sharedPullerState) {
 	buf := make([]byte, protocol.BlockSize)
 
 nextFile:
@@ -549,32 +561,70 @@ nextFile:
 			continue nextFile
 		}
 
-		srcFd, err := state.sourceFile()
-		if err != nil {
-			// As above
-			continue nextFile
-		}
+		evictionChan := make(chan lfu.Eviction)
+
+		fdCache := lfu.New()
+		fdCache.UpperBound = 50
+		fdCache.LowerBound = 20
+		fdCache.EvictionChannel = evictionChan
+
+		go func() {
+			for item := range evictionChan {
+				item.Value.(*os.File).Close()
+			}
+		}()
 
 		for _, block := range state.blocks {
 			buf = buf[:int(block.Size)]
 
-			_, err = srcFd.ReadAt(buf, block.Offset)
-			if err != nil {
-				state.earlyClose("src read", err)
-				srcFd.Close()
-				continue nextFile
+			success := p.model.finder.Iterate(block.Hash, func(folder, file string, index uint32) bool {
+				path := filepath.Join(p.model.folderCfgs[folder].Path, file)
+
+				var fd *os.File
+
+				fdi := fdCache.Get(path)
+				if fdi != nil {
+					fd = fdi.(*os.File)
+				} else {
+					fd, err = os.Open(path)
+					if err != nil {
+						return false
+					}
+					fdCache.Set(path, fd)
+				}
+
+				_, err = fd.ReadAt(buf, protocol.BlockSize*int64(index))
+				if err != nil {
+					return false
+				}
+
+				_, err = dstFd.WriteAt(buf, block.Offset)
+				if err != nil {
+					state.earlyClose("dst write", err)
+				}
+				if file == state.file.Name {
+					state.copiedFromOrigin()
+				}
+				return true
+			})
+
+			if state.failed() != nil {
+				break
 			}
 
-			_, err = dstFd.WriteAt(buf, block.Offset)
-			if err != nil {
-				state.earlyClose("dst write", err)
-				srcFd.Close()
-				continue nextFile
+			if !success {
+				state.pullStarted()
+				ps := pullBlockState{
+					sharedPullerState: state.sharedPullerState,
+					block:             block,
+				}
+				pullChan <- ps
+			} else {
+				state.copyDone()
 			}
 		}
-
-		srcFd.Close()
-		state.copyDone()
+		fdCache.Evict(fdCache.Len())
+		close(evictionChan)
 		out <- state.sharedPullerState
 	}
 }
@@ -640,15 +690,13 @@ func (p *Puller) finisherRoutine(in <-chan *sharedPullerState) {
 			// Verify the file against expected hashes
 			fd, err := os.Open(state.tempName)
 			if err != nil {
-				os.Remove(state.tempName)
 				l.Warnln("puller: final:", err)
 				continue
 			}
 			err = scanner.Verify(fd, protocol.BlockSize, state.file.Blocks)
 			fd.Close()
 			if err != nil {
-				os.Remove(state.tempName)
-				l.Warnln("puller: final:", state.file.Name, err)
+				l.Infoln("puller:", state.file.Name, err, "(file changed during pull?)")
 				continue
 			}
 
@@ -656,7 +704,6 @@ func (p *Puller) finisherRoutine(in <-chan *sharedPullerState) {
 			if !p.ignorePerms {
 				err = os.Chmod(state.tempName, os.FileMode(state.file.Flags&0777))
 				if err != nil {
-					os.Remove(state.tempName)
 					l.Warnln("puller: final:", err)
 					continue
 				}
@@ -666,9 +713,16 @@ func (p *Puller) finisherRoutine(in <-chan *sharedPullerState) {
 			t := time.Unix(state.file.Modified, 0)
 			err = os.Chtimes(state.tempName, t, t)
 			if err != nil {
-				os.Remove(state.tempName)
-				l.Warnln("puller: final:", err)
-				continue
+				if p.lenientMtimes {
+					// We accept the failure with a warning here and allow the sync to
+					// continue. We'll sync the new mtime back to the other devices later.
+					// If they have the same problem & setting, we might never get in
+					// sync.
+					l.Infof("Puller (folder %q, file %q): final: %v (continuing anyway as requested)", p.folder, state.file.Name, err)
+				} else {
+					l.Warnln("puller: final:", err)
+					continue
+				}
 			}
 
 			// If we should use versioning, let the versioner archive the old
@@ -677,7 +731,6 @@ func (p *Puller) finisherRoutine(in <-chan *sharedPullerState) {
 			if p.versioner != nil {
 				err = p.versioner.Archive(state.realName)
 				if err != nil {
-					os.Remove(state.tempName)
 					l.Warnln("puller: final:", err)
 					continue
 				}
@@ -686,7 +739,6 @@ func (p *Puller) finisherRoutine(in <-chan *sharedPullerState) {
 			// Replace the original file with the new one
 			err = osutil.Rename(state.tempName, state.realName)
 			if err != nil {
-				os.Remove(state.tempName)
 				l.Warnln("puller: final:", err)
 				continue
 			}

internal/model/puller_test.go

@@ -16,10 +16,13 @@
 package model
 
 import (
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/syncthing/syncthing/internal/config"
 	"github.com/syncthing/syncthing/internal/protocol"
+	"github.com/syncthing/syncthing/internal/scanner"
 
 	"github.com/syndtr/goleveldb/leveldb"
 	"github.com/syndtr/goleveldb/leveldb/storage"
@@ -47,7 +50,7 @@ func TestHandleFile(t *testing.T) {
 	// Copy: 2, 5, 8
 	// Pull: 1, 3, 4, 6, 7
 
-	// Create existing file, and update local index
+	// Create existing file
 	existingFile := protocol.FileInfo{
 		Name:     "filex",
 		Flags:    0,
@@ -65,6 +68,7 @@ func TestHandleFile(t *testing.T) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
 	m := NewModel(config.Wrap("/tmp/test", config.Configuration{}), "device", "syncthing", "dev", db)
 	m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"})
+	// Update index
 	m.updateLocal("default", existingFile)
 
 	p := Puller{
@@ -73,34 +77,20 @@ func TestHandleFile(t *testing.T) {
 		model:  m,
 	}
 
-	copyChan := make(chan copyBlocksState, 1) // Copy chan gets all blocks needed to copy in a wrapper struct
-	pullChan := make(chan pullBlockState, 5)  // Pull chan gets blocks one by one
+	copyChan := make(chan copyBlocksState, 1)
 
-	p.handleFile(requiredFile, copyChan, pullChan, nil)
+	p.handleFile(requiredFile, copyChan, nil)
 
 	// Receive the results
 	toCopy := <-copyChan
-	toPull := []pullBlockState{<-pullChan, <-pullChan, <-pullChan, <-pullChan, <-pullChan}
 
-	select {
-	case <-pullChan:
-		t.Error("Channel not empty!")
-	default:
+	if len(toCopy.blocks) != 8 {
+		t.Errorf("Unexpected count of copy blocks: %d != 8", len(toCopy.blocks))
 	}
 
-	if len(toCopy.blocks) != 3 {
-		t.Errorf("Unexpected count of copy blocks: %d != 3", len(toCopy.blocks))
-	}
-
-	for i, eq := range []int{2, 5, 8} {
-		if string(toCopy.blocks[i].Hash) != string(blocks[eq].Hash) {
-			t.Errorf("Block mismatch: %s != %s", toCopy.blocks[i].String(), blocks[eq].String())
-		}
-	}
-
-	for i, eq := range []int{1, 3, 4, 6, 7} {
-		if string(toPull[i].block.Hash) != string(blocks[eq].Hash) {
-			t.Errorf("Block mismatch: %s != %s", toPull[i].block.String(), blocks[eq].String())
+	for i, block := range toCopy.blocks {
+		if string(block.Hash) != string(blocks[i+1].Hash) {
+			t.Errorf("Block mismatch: %s != %s", block.String(), blocks[i+1].String())
 		}
 	}
 }
@@ -114,7 +104,7 @@ func TestHandleFileWithTemp(t *testing.T) {
 	// Copy: 5, 8
 	// Pull: 1, 6
 
-	// Create existing file, and update local index
+	// Create existing file
 	existingFile := protocol.FileInfo{
 		Name:     "file",
 		Flags:    0,
@@ -132,6 +122,7 @@ func TestHandleFileWithTemp(t *testing.T) {
 	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
 	m := NewModel(config.Wrap("/tmp/test", config.Configuration{}), "device", "syncthing", "dev", db)
 	m.AddFolder(config.FolderConfiguration{ID: "default", Path: "testdata"})
+	// Update index
 	m.updateLocal("default", existingFile)
 
 	p := Puller{
@@ -140,34 +131,122 @@ func TestHandleFileWithTemp(t *testing.T) {
 		model:  m,
 	}
 
-	copyChan := make(chan copyBlocksState, 1) // Copy chan gets all blocks needed to copy in a wrapper struct
-	pullChan := make(chan pullBlockState, 2)  // Pull chan gets blocks one by one
+	copyChan := make(chan copyBlocksState, 1)
 
-	p.handleFile(requiredFile, copyChan, pullChan, nil)
+	p.handleFile(requiredFile, copyChan, nil)
 
 	// Receive the results
 	toCopy := <-copyChan
-	toPull := []pullBlockState{<-pullChan, <-pullChan}
 
-	select {
-	case <-pullChan:
-		t.Error("Channel not empty!")
-	default:
+	if len(toCopy.blocks) != 4 {
+		t.Errorf("Unexpected count of copy blocks: %d != 4", len(toCopy.blocks))
 	}
 
-	if len(toCopy.blocks) != 2 {
-		t.Errorf("Unexpected count of copy blocks: %d != 2", len(toCopy.blocks))
-	}
-
-	for i, eq := range []int{5, 8} {
+	for i, eq := range []int{1, 5, 6, 8} {
 		if string(toCopy.blocks[i].Hash) != string(blocks[eq].Hash) {
 			t.Errorf("Block mismatch: %s != %s", toCopy.blocks[i].String(), blocks[eq].String())
 		}
 	}
+}
 
-	for i, eq := range []int{1, 6} {
-		if string(toPull[i].block.Hash) != string(blocks[eq].Hash) {
-			t.Errorf("Block mismatch: %s != %s", toPull[i].block.String(), blocks[eq].String())
+func TestCopierFinder(t *testing.T) {
+	// After diff between required and existing we should:
+	// Copy: 1, 2, 3, 4, 6, 7, 8
+	// Since there is no existing file, nor a temp file
+
+	// After dropping out blocks found locally:
+	// Pull: 1, 5, 6, 8
+
+	tempFile := filepath.Join("testdata", defTempNamer.TempName("file2"))
+	err := os.Remove(tempFile)
+	if err != nil && !os.IsNotExist(err) {
+		t.Error(err)
+	}
+
+	// Create existing file
+	existingFile := protocol.FileInfo{
+		Name:     defTempNamer.TempName("file"),
+		Flags:    0,
+		Modified: 0,
+		Blocks: []protocol.BlockInfo{
+			blocks[0], blocks[2], blocks[3], blocks[4],
+			blocks[0], blocks[0], blocks[7], blocks[0],
+		},
+	}
+
+	// Create target file
+	requiredFile := existingFile
+	requiredFile.Blocks = blocks[1:]
+	requiredFile.Name = "file2"
+
+	fcfg := config.FolderConfiguration{ID: "default", Path: "testdata"}
+	cfg := config.Configuration{Folders: []config.FolderConfiguration{fcfg}}
+
+	db, _ := leveldb.Open(storage.NewMemStorage(), nil)
+	m := NewModel(config.Wrap("/tmp/test", cfg), "device", "syncthing", "dev", db)
+	m.AddFolder(fcfg)
+	// Update index
+	m.updateLocal("default", existingFile)
+
+	iterFn := func(folder, file string, index uint32) bool {
+		return true
+	}
+
+	// Verify that the blocks we say exist on file, really exist in the db.
+	for _, idx := range []int{2, 3, 4, 7} {
+		if m.finder.Iterate(blocks[idx].Hash, iterFn) == false {
+			t.Error("Didn't find block")
 		}
 	}
+
+	p := Puller{
+		folder: "default",
+		dir:    "testdata",
+		model:  m,
+	}
+
+	copyChan := make(chan copyBlocksState)
+	pullChan := make(chan pullBlockState, 4)
+	finisherChan := make(chan *sharedPullerState, 1)
+
+	// Run a single fetcher routine
+	go p.copierRoutine(copyChan, pullChan, finisherChan)
+
+	p.handleFile(requiredFile, copyChan, finisherChan)
+
+	pulls := []pullBlockState{<-pullChan, <-pullChan, <-pullChan, <-pullChan}
+	finish := <-finisherChan
+
+	select {
+	case <-pullChan:
+		t.Fatal("Finisher channel has data to be read")
+	case <-finisherChan:
+		t.Fatal("Finisher channel has data to be read")
+	default:
+	}
+
+	// Verify that the right blocks went into the pull list
+	for i, eq := range []int{1, 5, 6, 8} {
+		if string(pulls[i].block.Hash) != string(blocks[eq].Hash) {
+			t.Errorf("Block %d mismatch: %s != %s", eq, pulls[i].block.String(), blocks[eq].String())
+		}
+		if string(finish.file.Blocks[eq-1].Hash) != string(blocks[eq].Hash) {
+			t.Errorf("Block %d mismatch: %s != %s", eq, finish.file.Blocks[eq-1].String(), blocks[eq].String())
+		}
+	}
+
+	// Verify that the fetched blocks have actually been written to the temp file
+	blks, err := scanner.HashFile(tempFile, protocol.BlockSize)
+	if err != nil {
+		t.Log(err)
+	}
+
+	for _, eq := range []int{2, 3, 4, 7} {
+		if string(blks[eq-1].Hash) != string(blocks[eq].Hash) {
+			t.Errorf("Block %d mismatch: %s != %s", eq, blks[eq-1].String(), blocks[eq].String())
+		}
+	}
+	finish.fd.Close()
+
+	os.Remove(tempFile)
 }

internal/model/scanner.go

@@ -59,6 +59,10 @@ func (s *Scanner) Serve() {
 				initialScanCompleted = true
 			}
 
+			if s.intv == 0 {
+				return
+			}
+
 			timer.Reset(s.intv)
 		}
 	}

internal/model/sharedpullerstate.go

@@ -31,13 +31,16 @@ type sharedPullerState struct {
 	folder   string
 	tempName string
 	realName string
-	reuse    bool
+	reused   int // Number of blocks reused from temporary file
 
 	// Mutable, must be locked for access
 	err        error      // The first error we hit
 	fd         *os.File   // The fd of the temp file
-	copyNeeded int        // Number of copy actions we expect to happen
-	pullNeeded int        // Number of block pulls we expect to happen
+	copyTotal  int        // Total number of copy actions for the whole job
+	pullTotal  int        // Total number of pull actions for the whole job
+	copyNeeded int        // Number of copy actions still pending
+	pullNeeded int        // Number of block pulls still pending
+	copyOrigin int        // Number of blocks copied from the original file
 	closed     bool       // Set when the file has been closed
 	mut        sync.Mutex // Protects the above
 }
@@ -79,7 +82,7 @@ func (s *sharedPullerState) tempFile() (*os.File, error) {
 
 	// Attempt to create the temp file
 	flags := os.O_WRONLY
-	if !s.reuse {
+	if s.reused == 0 {
 		flags |= os.O_CREATE | os.O_EXCL
 	}
 	fd, err := os.OpenFile(s.tempName, flags, 0644)
@@ -149,7 +152,25 @@ func (s *sharedPullerState) copyDone() {
 	s.mut.Lock()
 	s.copyNeeded--
 	if debug {
-		l.Debugln("sharedPullerState", s.folder, s.file.Name, "copyNeeded ->", s.pullNeeded)
+		l.Debugln("sharedPullerState", s.folder, s.file.Name, "copyNeeded ->", s.copyNeeded)
+	}
+	s.mut.Unlock()
+}
+
+func (s *sharedPullerState) copiedFromOrigin() {
+	s.mut.Lock()
+	s.copyOrigin++
+	s.mut.Unlock()
+}
+
+func (s *sharedPullerState) pullStarted() {
+	s.mut.Lock()
+	s.copyTotal--
+	s.copyNeeded--
+	s.pullTotal++
+	s.pullNeeded++
+	if debug {
+		l.Debugln("sharedPullerState", s.folder, s.file.Name, "pullNeeded start ->", s.pullNeeded)
 	}
 	s.mut.Unlock()
 }
@@ -158,7 +179,7 @@ func (s *sharedPullerState) pullDone() {
 	s.mut.Lock()
 	s.pullNeeded--
 	if debug {
-		l.Debugln("sharedPullerState", s.folder, s.file.Name, "pullNeeded ->", s.pullNeeded)
+		l.Debugln("sharedPullerState", s.folder, s.file.Name, "pullNeeded done ->", s.pullNeeded)
 	}
 	s.mut.Unlock()
 }

internal/protocol/message.go

@@ -54,6 +54,10 @@ func (f FileInfo) IsInvalid() bool {
 	return IsInvalid(f.Flags)
 }
 
+func (f FileInfo) IsDirectory() bool {
+	return IsDirectory(f.Flags)
+}
+
 // Used for unmarshalling a FileInfo structure but skipping the actual block list
 type FileInfoTruncated struct {
 	Name         string // max:8192
@@ -64,6 +68,11 @@ type FileInfoTruncated struct {
 	NumBlocks    uint32
 }
 
+func (f FileInfoTruncated) String() string {
+	return fmt.Sprintf("File{Name:%q, Flags:0%o, Modified:%d, Version:%d, Size:%d, NumBlocks:%d}",
+		f.Name, f.Flags, f.Modified, f.Version, f.Size(), f.NumBlocks)
+}
+
 // Returns a statistical guess on the size, not the exact figure
 func (f FileInfoTruncated) Size() int64 {
 	if IsDeleted(f.Flags) || IsDirectory(f.Flags) {

internal/scanner/blocks.go

@@ -68,6 +68,15 @@ func Blocks(r io.Reader, blocksize int, sizehint int64) ([]protocol.BlockInfo, e
 	return blocks, nil
 }
 
+// Set the Offset field on each block
+func PopulateOffsets(blocks []protocol.BlockInfo) {
+	var offset int64
+	for i := range blocks {
+		blocks[i].Offset = offset
+		offset += int64(blocks[i].Size)
+	}
+}
+
 // BlockDiff returns lists of common and missing (to transform src into tgt)
 // blocks. Both block lists must have been created with the same block size.
 func BlockDiff(src, tgt []protocol.BlockInfo) (have, need []protocol.BlockInfo) {
@@ -75,13 +84,6 @@ func BlockDiff(src, tgt []protocol.BlockInfo) (have, need []protocol.BlockInfo)
 		return nil, nil
 	}
 
-	// Set the Offset field on each target block
-	var offset int64
-	for i := range tgt {
-		tgt[i].Offset = offset
-		offset += int64(tgt[i].Size)
-	}
-
 	if len(tgt) != 0 && len(src) == 0 {
 		// Copy the entire file
 		return nil, tgt

internal/scanner/walk.go

@@ -35,8 +35,8 @@ type Walker struct {
 	Sub string
 	// BlockSize controls the size of the block used when hashing.
 	BlockSize int
-	// List of patterns to ignore
-	Ignores ignore.Patterns
+	// If Matcher is not nil, it is used to identify files to ignore which were specified by the user.
+	Matcher *ignore.Matcher
 	// If TempNamer is not nil, it is used to ignore tempory files when walking.
 	TempNamer TempNamer
 	// If CurrentFiler is not nil, it is queried for the current file before rescanning.
@@ -63,7 +63,7 @@ type CurrentFiler interface {
 // file system. Files are blockwise hashed.
 func (w *Walker) Walk() (chan protocol.FileInfo, error) {
 	if debug {
-		l.Debugln("Walk", w.Dir, w.Sub, w.BlockSize, w.Ignores)
+		l.Debugln("Walk", w.Dir, w.Sub, w.BlockSize, w.Matcher)
 	}
 
 	err := checkDir(w.Dir)
@@ -113,7 +113,8 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
 			return nil
 		}
 
-		if sn := filepath.Base(rn); sn == ".stignore" || sn == ".stversions" || w.Ignores.Match(rn) {
+		if sn := filepath.Base(rn); sn == ".stignore" || sn == ".stversions" ||
+			sn == ".stfolder" || (w.Matcher != nil && w.Matcher.Match(rn)) {
 			// An ignored file
 			if debug {
 				l.Debugln("ignored:", rn)
@@ -151,7 +152,7 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
 				Modified: info.ModTime().Unix(),
 			}
 			if debug {
-				l.Debugln("dir:", f)
+				l.Debugln("dir:", p, f)
 			}
 			fchan <- f
 			return nil
@@ -175,12 +176,16 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
 				flags = protocol.FlagNoPermBits | 0666
 			}
 
-			fchan <- protocol.FileInfo{
+			f := protocol.FileInfo{
 				Name:     rn,
 				Version:  lamport.Default.Tick(0),
 				Flags:    flags,
 				Modified: info.ModTime().Unix(),
 			}
+			if debug {
+				l.Debugln("to hash:", p, f)
+			}
+			fchan <- f
 		}
 
 		return nil

internal/scanner/walk_test.go

@@ -58,7 +58,7 @@ func init() {
 }
 
 func TestWalkSub(t *testing.T) {
-	ignores, err := ignore.Load("testdata/.stignore")
+	ignores, err := ignore.Load("testdata/.stignore", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -67,7 +67,7 @@ func TestWalkSub(t *testing.T) {
 		Dir:       "testdata",
 		Sub:       "dir2",
 		BlockSize: 128 * 1024,
-		Ignores:   ignores,
+		Matcher:   ignores,
 	}
 	fchan, err := w.Walk()
 	var files []protocol.FileInfo
@@ -93,7 +93,7 @@ func TestWalkSub(t *testing.T) {
 }
 
 func TestWalk(t *testing.T) {
-	ignores, err := ignore.Load("testdata/.stignore")
+	ignores, err := ignore.Load("testdata/.stignore", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -102,7 +102,7 @@ func TestWalk(t *testing.T) {
 	w := Walker{
 		Dir:       "testdata",
 		BlockSize: 128 * 1024,
-		Ignores:   ignores,
+		Matcher:   ignores,
 	}
 
 	fchan, err := w.Walk()

internal/upnp/testdata/technicolor.xml

@@ -1,93 +0,0 @@
-<?xml version="1.0"?>
-<root xmlns="urn:schemas-upnp-org:device-1-0">
-<specVersion>
-<major>1</major>
-<minor>0</minor>
-</specVersion>
-<URLBase>http://192.168.1.254:8000</URLBase>
-<device>
-<deviceType>urn:schemas-upnp-org:device:InternetGatewayDevice:1</deviceType>
-<friendlyName>Technicolor TG784n v3 (1321RAWMS)</friendlyName>
-<manufacturer>Technicolor</manufacturer>
-<manufacturerURL>http://www.technicolor.com
-</manufacturerURL>
-<modelDescription>Technicolor Internet Gateway Device</modelDescription>
-<modelName>Technicolor TG</modelName>
-<modelNumber>784n v3</modelNumber>
-<modelURL>http://www.technicolor.com</modelURL>
-<serialNumber>1321RAWMS</serialNumber>
-<UDN>uuid:UPnP_Technicolor TG784n v3-1_A4-B1-E9-D8-F4-78</UDN>
-<presentationURL>/</presentationURL>
-<serviceList>
-<service>
-<serviceType>urn:schemas-upnp-org:service:Layer3Forwarding:1</serviceType>
-<serviceId>urn:upnp-org:serviceId:L3Forwarding1</serviceId>
-<controlURL>/hou74cq4tw9/IGD/upnp/control/igd/layer3f</controlURL>
-<eventSubURL>/hou74cq4tw9/IGD/upnp/event/igd/layer3f</eventSubURL>
-<SCPDURL>/hou74cq4tw9/IGD/upnp/Layer3Forwarding.xml</SCPDURL>
-</service>
-</serviceList>
-<deviceList>
-<device>
-<deviceType>urn:schemas-upnp-org:device:LANDevice:1</deviceType>
-<friendlyName>LANDevice</friendlyName>
-<manufacturer>Technicolor</manufacturer>
-<modelName>Technicolor TG784n v3</modelName>
-<serialNumber>A4-B1-E9-D8-F4-78</serialNumber>
-<UDN>uuid:UPnP_Technicolor TG784n v3-1_A4-B1-E9-D8-F4-78_LD_1</UDN>
-<serviceList>
-<service>
-<serviceType>urn:schemas-upnp-org:service:LANHostConfigManagement:1</serviceType>
-<serviceId>urn:upnp-org:serviceId:LANHostCfg1</serviceId>
-<controlURL>/hou74cq4tw9/IGD/upnp/control/igd/lanhcm_1</controlURL>
-<eventSubURL></eventSubURL>
-<SCPDURL>/hou74cq4tw9/IGD/upnp/LANHostConfigManagement.xml</SCPDURL>
-</service>
-</serviceList>
-</device>
-<device>
-<deviceType>urn:schemas-upnp-org:device:WANDevice:1</deviceType>
-<friendlyName>WANDevice</friendlyName>
-<manufacturer>Technicolor</manufacturer>
-<modelName>Technicolor TG784n v3</modelName>
-<serialNumber>A4-B1-E9-D8-F4-78</serialNumber>
-<UDN>uuid:UPnP_Technicolor TG784n v3-1_A4-B1-E9-D8-F4-78_WD_1</UDN>
-<serviceList>
-<service>
-<serviceType>urn:schemas-upnp-org:service:WANCommonInterfaceConfig:1</serviceType>
-<serviceId>urn:upnp-org:serviceId:WANCommonIFC1</serviceId>
-<controlURL>/hou74cq4tw9/IGD/upnp/control/igd/wancic_1</controlURL>
-<eventSubURL>/hou74cq4tw9/IGD/upnp/event/igd/wancic_1</eventSubURL>
-<SCPDURL>/hou74cq4tw9/IGD/upnp/WANCommonInterfaceConfig.xml</SCPDURL>
-</service>
-</serviceList>
-<deviceList>
-<device>
-<deviceType>urn:schemas-upnp-org:device:WANConnectionDevice:1</deviceType>
-<friendlyName>WANConnectionDevice</friendlyName>
-<manufacturer>Technicolor</manufacturer>
-<modelName>Technicolor TG784n v3</modelName>
-<serialNumber>A4-B1-E9-D8-F4-78</serialNumber>
-<UDN>uuid:UPnP_Technicolor TG784n v3-1_A4-B1-E9-D8-F4-78_WCD_1_1</UDN>
-<serviceList>
-<service>
-<serviceType>urn:schemas-upnp-org:service:WANDSLLinkConfig:1</serviceType>
-<serviceId>urn:upnp-org:serviceId:WANDSLLinkC1</serviceId>
-<controlURL>/hou74cq4tw9/IGD/upnp/control/igd/wandsllc_1_1</controlURL>
-<eventSubURL>/hou74cq4tw9/IGD/upnp/event/igd/wandsllc_1_1</eventSubURL>
-<SCPDURL>/hou74cq4tw9/IGD/upnp/WANDSLLinkConfig.xml</SCPDURL>
-</service>
-<service>
-<serviceType>urn:schemas-upnp-org:service:WANPPPConnection:1</serviceType>
-<serviceId>urn:upnp-org:serviceId:WANPPPConn1</serviceId>
-<controlURL>/hou74cq4tw9/IGD/upnp/control/igd/wanpppc_1_1_1</controlURL>
-<eventSubURL>/hou74cq4tw9/IGD/upnp/event/igd/wanpppc_1_1_1</eventSubURL>
-<SCPDURL>/hou74cq4tw9/IGD/upnp/WANPPPConnection.xml</SCPDURL>
-</service>
-</serviceList>
-</device>
-</deviceList>
-</device>
-</deviceList>
-</device>
-</root>

internal/upnp/upnp.go

@@ -16,7 +16,7 @@
 // Adapted from https://github.com/jackpal/Taipei-Torrent/blob/dd88a8bfac6431c01d959ce3c745e74b8a911793/IGD.go
 // Copyright (c) 2010 Jack Palevich (https://github.com/jackpal/Taipei-Torrent/blob/dd88a8bfac6431c01d959ce3c745e74b8a911793/LICENSE)
 
-// Package upnp implements UPnP Internet Gateway upnpDevice port mappings
+// Package upnp implements UPnP InternetGatewayDevice discovery, querying, and port mapping.
 package upnp
 
 import (
@@ -25,19 +25,29 @@ import (
 	"encoding/xml"
 	"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
 	"net/url"
+	"regexp"
 	"strings"
+	"sync"
 	"time"
 )
 
+// A container for relevant properties of a UPnP InternetGatewayDevice.
 type IGD struct {
+	uuid           string
+	friendlyName   string
+	services       []IGDService
+	url            *url.URL
+	localIPAddress string
+}
+
+// A container for relevant properties of a UPnP service of an IGD.
+type IGDService struct {
 	serviceURL string
-	device     string
-	ourIP      string
+	serviceURN string
 }
 
 type Protocol string
@@ -53,180 +63,352 @@ type upnpService struct {
 }
 
 type upnpDevice struct {
-	DeviceType string        `xml:"deviceType"`
-	Devices    []upnpDevice  `xml:"deviceList>device"`
-	Services   []upnpService `xml:"serviceList>service"`
+	DeviceType   string        `xml:"deviceType"`
+	FriendlyName string        `xml:"friendlyName"`
+	Devices      []upnpDevice  `xml:"deviceList>device"`
+	Services     []upnpService `xml:"serviceList>service"`
 }
 
 type upnpRoot struct {
 	Device upnpDevice `xml:"device"`
 }
 
-func Discover() (*IGD, error) {
+// Discover discovers UPnP InternetGatewayDevices.
+// The order in which the devices appear in the result list is not deterministic.
+func Discover() []*IGD {
+	result := make([]*IGD, 0)
+	l.Infoln("Starting UPnP discovery...")
+
+	timeout := 3
+
+	// Search for InternetGatewayDevice:2 devices
+	result = append(result, discover("urn:schemas-upnp-org:device:InternetGatewayDevice:2", timeout, result)...)
+
+	// Search for InternetGatewayDevice:1 devices
+	// InternetGatewayDevice:2 devices that correctly respond to the IGD:1 request as well will not be re-added to the result list
+	result = append(result, discover("urn:schemas-upnp-org:device:InternetGatewayDevice:1", timeout, result)...)
+
+	if len(result) > 0 && debug {
+		l.Debugln("UPnP discovery result:")
+		for _, resultDevice := range result {
+			l.Debugln("[" + resultDevice.uuid + "]")
+
+			for _, resultService := range resultDevice.services {
+				l.Debugln("* " + resultService.serviceURL)
+			}
+		}
+	}
+
+	suffix := "devices"
+	if len(result) == 1 {
+		suffix = "device"
+	}
+
+	l.Infof("UPnP discovery complete (found %d %s).", len(result), suffix)
+
+	return result
+}
+
+// Search for UPnP InternetGatewayDevices for <timeout> seconds, ignoring responses from any devices listed in knownDevices.
+// The order in which the devices appear in the result list is not deterministic
+func discover(deviceType string, timeout int, knownDevices []*IGD) []*IGD {
 	ssdp := &net.UDPAddr{IP: []byte{239, 255, 255, 250}, Port: 1900}
 
+	tpl := `M-SEARCH * HTTP/1.1
+Host: 239.255.255.250:1900
+St: %s
+Man: "ssdp:discover"
+Mx: %d
+
+`
+	searchStr := fmt.Sprintf(tpl, deviceType, timeout)
+
+	search := []byte(strings.Replace(searchStr, "\n", "\r\n", -1))
+
+	if debug {
+		l.Debugln("Starting discovery of device type " + deviceType + "...")
+	}
+
+	results := make([]*IGD, 0)
+	resultChannel := make(chan *IGD, 8)
+
 	socket, err := net.ListenUDP("udp4", &net.UDPAddr{})
 	if err != nil {
-		return nil, err
+		l.Infoln(err)
+		return results
 	}
-	defer socket.Close()
+	defer socket.Close() // Make sure our socket gets closed
 
-	err = socket.SetDeadline(time.Now().Add(3 * time.Second))
+	err = socket.SetDeadline(time.Now().Add(time.Duration(timeout) * time.Second))
 	if err != nil {
-		return nil, err
-	}
-
-	searchStr := `M-SEARCH * HTTP/1.1
-Host: 239.255.255.250:1900
-St: urn:schemas-upnp-org:device:InternetGatewayDevice:1
-Man: "ssdp:discover"
-Mx: 3
-
-`
-	search := []byte(strings.Replace(searchStr, "\n", "\r\n", -1))
-
-	_, err = socket.WriteTo(search, ssdp)
-	if err != nil {
-		return nil, err
-	}
-
-	resp := make([]byte, 1500)
-	n, _, err := socket.ReadFrom(resp)
-	if err != nil {
-		return nil, err
+		l.Infoln(err)
+		return results
 	}
 
 	if debug {
-		l.Debugln(string(resp[:n]))
+		l.Debugln("Sending search request for device type " + deviceType + "...")
 	}
 
-	reader := bufio.NewReader(bytes.NewBuffer(resp[:n]))
+	var resultWaitGroup sync.WaitGroup
+
+	_, err = socket.WriteTo(search, ssdp)
+	if err != nil {
+		l.Infoln(err)
+		return results
+	}
+
+	if debug {
+		l.Debugln("Listening for UPnP response for device type " + deviceType + "...")
+	}
+
+	// Listen for responses until a timeout is reached
+	for {
+		resp := make([]byte, 1500)
+		n, _, err := socket.ReadFrom(resp)
+		if err != nil {
+			if e, ok := err.(net.Error); !ok || !e.Timeout() {
+				l.Infoln(err) //legitimate error, not a timeout.
+			}
+
+			break
+		} else {
+			// Process results in a separate go routine so we can immediately return to listening for more responses
+			resultWaitGroup.Add(1)
+			go handleSearchResponse(deviceType, knownDevices, resp, n, resultChannel, &resultWaitGroup)
+		}
+	}
+
+	// Wait for all result handlers to finish processing, then close result channel
+	resultWaitGroup.Wait()
+	close(resultChannel)
+
+	// Collect our results from the result handlers using the result channel
+	for result := range resultChannel {
+		results = append(results, result)
+	}
+
+	if debug {
+		l.Debugln("Discovery for device type " + deviceType + " finished.")
+	}
+
+	return results
+}
+
+func handleSearchResponse(deviceType string, knownDevices []*IGD, resp []byte, length int, resultChannel chan<- *IGD, resultWaitGroup *sync.WaitGroup) {
+	defer resultWaitGroup.Done() // Signal when we've finished processing
+
+	if debug {
+		l.Debugln("Handling UPnP response:\n\n" + string(resp[:length]))
+	}
+
+	reader := bufio.NewReader(bytes.NewBuffer(resp[:length]))
 	request := &http.Request{}
 	response, err := http.ReadResponse(reader, request)
 	if err != nil {
-		return nil, err
+		l.Infoln(err)
+		return
 	}
 
-	if response.Header.Get("St") != "urn:schemas-upnp-org:device:InternetGatewayDevice:1" {
-		return nil, errors.New("no igd")
+	respondingDeviceType := response.Header.Get("St")
+	if respondingDeviceType != deviceType {
+		l.Infoln("Unrecognized UPnP device of type " + respondingDeviceType)
+		return
 	}
 
-	locURL := response.Header.Get("Location")
-	if locURL == "" {
-		return nil, errors.New("no location")
+	deviceDescriptionLocation := response.Header.Get("Location")
+	if deviceDescriptionLocation == "" {
+		l.Infoln("Invalid IGD response: no location specified.")
+		return
 	}
 
-	serviceURL, device, err := getServiceURL(locURL)
+	deviceDescriptionURL, err := url.Parse(deviceDescriptionLocation)
+
 	if err != nil {
-		return nil, err
+		l.Infoln("Invalid IGD location: " + err.Error())
 	}
 
-	// Figure out our IP number, on the network used to reach the IGD. We
-	// do this in a fairly roundabout way by connecting to the IGD and
+	deviceUSN := response.Header.Get("USN")
+	if deviceUSN == "" {
+		l.Infoln("Invalid IGD response: USN not specified.")
+		return
+	}
+
+	deviceUUID := strings.TrimLeft(strings.Split(deviceUSN, "::")[0], "uuid:")
+	matched, err := regexp.MatchString("[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}", deviceUUID)
+	if !matched {
+		l.Infoln("Invalid IGD response: invalid device UUID " + deviceUUID)
+		return
+	}
+
+	// Don't re-add devices that are already known
+	for _, knownDevice := range knownDevices {
+		if deviceUUID == knownDevice.uuid {
+			if debug {
+				l.Debugln("Ignoring known device with UUID " + deviceUUID)
+			}
+			return
+		}
+	}
+
+	response, err = http.Get(deviceDescriptionLocation)
+	if err != nil {
+		l.Infoln(err)
+		return
+	}
+	defer response.Body.Close()
+
+	if response.StatusCode >= 400 {
+		l.Infoln(errors.New(response.Status))
+		return
+	}
+
+	var upnpRoot upnpRoot
+	err = xml.NewDecoder(response.Body).Decode(&upnpRoot)
+	if err != nil {
+		l.Infoln(err)
+		return
+	}
+
+	services, err := getServiceDescriptions(deviceDescriptionLocation, upnpRoot.Device)
+	if err != nil {
+		l.Infoln(err)
+		return
+	}
+
+	// Figure out our IP number, on the network used to reach the IGD.
+	// We do this in a fairly roundabout way by connecting to the IGD and
 	// checking the address of the local end of the socket. I'm open to
 	// suggestions on a better way to do this...
-	ourIP, err := localIP(locURL)
+	localIPAddress, err := localIP(deviceDescriptionURL)
 	if err != nil {
-		return nil, err
+		l.Infoln(err)
+		return
 	}
 
 	igd := &IGD{
-		serviceURL: serviceURL,
-		device:     device,
-		ourIP:      ourIP,
+		uuid:           deviceUUID,
+		friendlyName:   upnpRoot.Device.FriendlyName,
+		url:            deviceDescriptionURL,
+		services:       services,
+		localIPAddress: localIPAddress,
+	}
+
+	resultChannel <- igd
+
+	if debug {
+		l.Debugln("Finished handling of UPnP response.")
 	}
-	return igd, nil
 }
 
-func localIP(tgt string) (string, error) {
-	url, err := url.Parse(tgt)
-	if err != nil {
-		return "", err
-	}
-
+func localIP(url *url.URL) (string, error) {
 	conn, err := net.Dial("tcp", url.Host)
 	if err != nil {
 		return "", err
 	}
 	defer conn.Close()
 
-	ourIP, _, err := net.SplitHostPort(conn.LocalAddr().String())
+	localIPAddress, _, err := net.SplitHostPort(conn.LocalAddr().String())
 	if err != nil {
 		return "", err
 	}
 
-	return ourIP, nil
+	return localIPAddress, nil
 }
 
-func getChildDevice(d upnpDevice, deviceType string) (upnpDevice, bool) {
+func getChildDevices(d upnpDevice, deviceType string) []upnpDevice {
+	result := make([]upnpDevice, 0)
 	for _, dev := range d.Devices {
 		if dev.DeviceType == deviceType {
-			return dev, true
+			result = append(result, dev)
 		}
 	}
-	return upnpDevice{}, false
+	return result
 }
 
-func getChildService(d upnpDevice, serviceType string) (upnpService, bool) {
+func getChildServices(d upnpDevice, serviceType string) []upnpService {
+	result := make([]upnpService, 0)
 	for _, svc := range d.Services {
 		if svc.ServiceType == serviceType {
-			return svc, true
+			result = append(result, svc)
 		}
 	}
-	return upnpService{}, false
+	return result
 }
 
-func getServiceURL(rootURL string) (string, string, error) {
-	r, err := http.Get(rootURL)
-	if err != nil {
-		return "", "", err
+func getServiceDescriptions(rootURL string, device upnpDevice) ([]IGDService, error) {
+	result := make([]IGDService, 0)
+
+	if device.DeviceType == "urn:schemas-upnp-org:device:InternetGatewayDevice:1" {
+		descriptions := getIGDServices(rootURL, device,
+			"urn:schemas-upnp-org:device:WANDevice:1",
+			"urn:schemas-upnp-org:device:WANConnectionDevice:1",
+			[]string{"urn:schemas-upnp-org:service:WANIPConnection:1", "urn:schemas-upnp-org:service:WANPPPConnection:1"})
+
+		result = append(result, descriptions...)
+	} else if device.DeviceType == "urn:schemas-upnp-org:device:InternetGatewayDevice:2" {
+		descriptions := getIGDServices(rootURL, device,
+			"urn:schemas-upnp-org:device:WANDevice:2",
+			"urn:schemas-upnp-org:device:WANConnectionDevice:2",
+			[]string{"urn:schemas-upnp-org:service:WANIPConnection:2", "urn:schemas-upnp-org:service:WANPPPConnection:1"})
+
+		result = append(result, descriptions...)
+	} else {
+		return result, errors.New("[" + rootURL + "] Malformed root device description: not an InternetGatewayDevice.")
 	}
-	defer r.Body.Close()
-	if r.StatusCode >= 400 {
-		return "", "", errors.New(r.Status)
+
+	if len(result) < 1 {
+		return result, errors.New("[" + rootURL + "] Malformed device description: no compatible service descriptions found.")
+	} else {
+		return result, nil
 	}
-	return getServiceURLReader(rootURL, r.Body)
 }
 
-func getServiceURLReader(rootURL string, r io.Reader) (string, string, error) {
-	var upnpRoot upnpRoot
-	err := xml.NewDecoder(r).Decode(&upnpRoot)
-	if err != nil {
-		return "", "", err
+func getIGDServices(rootURL string, device upnpDevice, wanDeviceURN string, wanConnectionURN string, serviceURNs []string) []IGDService {
+	result := make([]IGDService, 0)
+
+	devices := getChildDevices(device, wanDeviceURN)
+
+	if len(devices) < 1 {
+		l.Infoln("[" + rootURL + "] Malformed InternetGatewayDevice description: no WANDevices specified.")
+		return result
 	}
 
-	dev := upnpRoot.Device
-	if dev.DeviceType != "urn:schemas-upnp-org:device:InternetGatewayDevice:1" {
-		return "", "", errors.New("No InternetGatewayDevice")
+	for _, device := range devices {
+		connections := getChildDevices(device, wanConnectionURN)
+
+		if len(connections) < 1 {
+			l.Infoln("[" + rootURL + "] Malformed " + wanDeviceURN + " description: no WANConnectionDevices specified.")
+		}
+
+		for _, connection := range connections {
+			for _, serviceURN := range serviceURNs {
+				services := getChildServices(connection, serviceURN)
+
+				if len(services) < 1 && debug {
+					l.Debugln("[" + rootURL + "] No services of type " + serviceURN + " found on connection.")
+				}
+
+				for _, service := range services {
+					if len(service.ControlURL) == 0 {
+						l.Infoln("[" + rootURL + "] Malformed " + service.ServiceType + " description: no control URL.")
+					} else {
+						u, _ := url.Parse(rootURL)
+						replaceRawPath(u, service.ControlURL)
+
+						if debug {
+							l.Debugln("[" + rootURL + "] Found " + service.ServiceType + " with URL " + u.String())
+						}
+
+						service := IGDService{serviceURL: u.String(), serviceURN: service.ServiceType}
+
+						result = append(result, service)
+					}
+				}
+			}
+		}
 	}
 
-	dev, ok := getChildDevice(dev, "urn:schemas-upnp-org:device:WANDevice:1")
-	if !ok {
-		return "", "", errors.New("No WANDevice")
-	}
-
-	dev, ok = getChildDevice(dev, "urn:schemas-upnp-org:device:WANConnectionDevice:1")
-	if !ok {
-		return "", "", errors.New("No WANConnectionDevice")
-	}
-
-	device := "urn:schemas-upnp-org:service:WANIPConnection:1"
-	svc, ok := getChildService(dev, device)
-	if !ok {
-		device = "urn:schemas-upnp-org:service:WANPPPConnection:1"
-	}
-	svc, ok = getChildService(dev, device)
-	if !ok {
-		return "", "", errors.New("No WANIPConnection nor WANPPPConnection")
-	}
-
-	if len(svc.ControlURL) == 0 {
-		return "", "", errors.New("no controlURL")
-	}
-
-	u, _ := url.Parse(rootURL)
-	replaceRawPath(u, svc.ControlURL)
-	return u.String(), device, nil
+	return result
 }
 
 func replaceRawPath(u *url.URL, rp string) {
@@ -245,17 +427,19 @@ func replaceRawPath(u *url.URL, rp string) {
 	u.RawQuery = q
 }
 
-func soapRequest(url, device, function, message string) error {
-	tpl := `<?xml version="1.0" ?>
+func soapRequest(url, device, function, message string) ([]byte, error) {
+	tpl := `	<?xml version="1.0" ?>
 	<s:Envelope xmlns:s="http://schemas.xmlsoap.org/soap/envelope/" s:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/">
 	<s:Body>%s</s:Body>
 	</s:Envelope>
 `
+	var resp []byte
+
 	body := fmt.Sprintf(tpl, message)
 
 	req, err := http.NewRequest("POST", url, strings.NewReader(body))
 	if err != nil {
-		return err
+		return resp, err
 	}
 	req.Header.Set("Content-Type", `text/xml; charset="utf-8"`)
 	req.Header.Set("User-Agent", "syncthing/1.0")
@@ -266,30 +450,91 @@ func soapRequest(url, device, function, message string) error {
 
 	if debug {
 		l.Debugln(req.Header.Get("SOAPAction"))
-		l.Debugln(body)
+		l.Debugln("SOAP Request:\n\n" + body)
 	}
 
 	r, err := http.DefaultClient.Do(req)
 	if err != nil {
-		return err
+		return resp, err
 	}
 
+	resp, _ = ioutil.ReadAll(r.Body)
 	if debug {
-		resp, _ := ioutil.ReadAll(r.Body)
-		l.Debugln(string(resp))
+		l.Debugln("SOAP Response:\n\n" + string(resp) + "\n")
 	}
 
 	r.Body.Close()
 
 	if r.StatusCode >= 400 {
-		return errors.New(function + ": " + r.Status)
+		return resp, errors.New(function + ": " + r.Status)
 	}
 
+	return resp, nil
+}
+
+// Add a port mapping to all relevant services on the specified InternetGatewayDevice.
+// Port mapping will fail and return an error if action is fails for _any_ of the relevant services.
+// For this reason, it is generally better to configure port mapping for each individual service instead. 
+func (n *IGD) AddPortMapping(protocol Protocol, externalPort, internalPort int, description string, timeout int) error {
+	for _, service := range n.services {
+		err := service.AddPortMapping(n.localIPAddress, protocol, externalPort, internalPort, description, timeout)
+		if err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
-func (n *IGD) AddPortMapping(protocol Protocol, externalPort, internalPort int, description string, timeout int) error {
-	tpl := `<u:AddPortMapping xmlns:u="urn:schemas-upnp-org:service:WANIPConnection:1">
+// Delete a port mapping from all relevant services on the specified InternetGatewayDevice.
+// Port mapping will fail and return an error if action is fails for _any_ of the relevant services.
+// For this reason, it is generally better to configure port mapping for each individual service instead. 
+func (n *IGD) DeletePortMapping(protocol Protocol, externalPort int) error {
+	for _, service := range n.services {
+		err := service.DeletePortMapping(protocol, externalPort)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// The InternetGatewayDevice's UUID.
+func (n *IGD) UUID() string {
+	return n.uuid
+}
+
+// The InternetGatewayDevice's friendly name.
+func (n *IGD) FriendlyName() string {
+	return n.friendlyName
+}
+
+// The InternetGatewayDevice's friendly identifier (friendly name + IP address).
+func (n *IGD) FriendlyIdentifier() string {
+	return "'" + n.FriendlyName() + "' (" + strings.Split(n.URL().Host, ":")[0] + ")"
+}
+
+// The URL of the InternetGatewayDevice's root device description.
+func (n *IGD) URL() *url.URL {
+	return n.url
+}
+
+type soapGetExternalIPAddressResponseEnvelope struct {
+	XMLName xml.Name
+	Body    soapGetExternalIPAddressResponseBody `xml:"Body"`
+}
+
+type soapGetExternalIPAddressResponseBody struct {
+	XMLName                      xml.Name
+	GetExternalIPAddressResponse getExternalIPAddressResponse `xml:"GetExternalIPAddressResponse"`
+}
+
+type getExternalIPAddressResponse struct {
+	NewExternalIPAddress string `xml:"NewExternalIPAddress"`
+}
+
+// Add a port mapping to the specified IGD service.
+func (s *IGDService) AddPortMapping(localIPAddress string, protocol Protocol, externalPort, internalPort int, description string, timeout int) error {
+	tpl := `<u:AddPortMapping xmlns:u="%s">
 	<NewRemoteHost></NewRemoteHost>
 	<NewExternalPort>%d</NewExternalPort>
 	<NewProtocol>%s</NewProtocol>
@@ -298,21 +543,55 @@ func (n *IGD) AddPortMapping(protocol Protocol, externalPort, internalPort int,
 	<NewEnabled>1</NewEnabled>
 	<NewPortMappingDescription>%s</NewPortMappingDescription>
 	<NewLeaseDuration>%d</NewLeaseDuration>
-	</u:AddPortMapping>
-	`
+	</u:AddPortMapping>`
+	body := fmt.Sprintf(tpl, s.serviceURN, externalPort, protocol, internalPort, localIPAddress, description, timeout)
 
-	body := fmt.Sprintf(tpl, externalPort, protocol, internalPort, n.ourIP, description, timeout)
-	return soapRequest(n.serviceURL, n.device, "AddPortMapping", body)
+	_, err := soapRequest(s.serviceURL, s.serviceURN, "AddPortMapping", body)
+	if err != nil {
+		return err
+	}
+
+	return nil
 }
 
-func (n *IGD) DeletePortMapping(protocol Protocol, externalPort int) (err error) {
-	tpl := `<u:DeletePortMapping xmlns:u="urn:schemas-upnp-org:service:WANIPConnection:1">
+// Delete a port mapping from the specified IGD service.
+func (s *IGDService) DeletePortMapping(protocol Protocol, externalPort int) error {
+	tpl := `<u:DeletePortMapping xmlns:u="%s">
 	<NewRemoteHost></NewRemoteHost>
 	<NewExternalPort>%d</NewExternalPort>
 	<NewProtocol>%s</NewProtocol>
-	</u:DeletePortMapping>
-	`
+	</u:DeletePortMapping>`
+	body := fmt.Sprintf(tpl, s.serviceURN, externalPort, protocol)
 
-	body := fmt.Sprintf(tpl, externalPort, protocol)
-	return soapRequest(n.serviceURL, n.device, "DeletePortMapping", body)
+	_, err := soapRequest(s.serviceURL, s.serviceURN, "DeletePortMapping", body)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Query the IGD service for its external IP address.
+// Returns nil if the external IP address is invalid or undefined, along with any relevant errors
+func (s *IGDService) GetExternalIPAddress() (net.IP, error) {
+	tpl := `<u:GetExternalIPAddress xmlns:u="%s" />`
+
+	body := fmt.Sprintf(tpl, s.serviceURN)
+
+	response, err := soapRequest(s.serviceURL, s.serviceURN, "GetExternalIPAddress", body)
+
+	if err != nil {
+		return nil, err
+	}
+
+	envelope := &soapGetExternalIPAddressResponseEnvelope{}
+	err = xml.Unmarshal(response, envelope)
+	if err != nil {
+		return nil, err
+	}
+
+	result := net.ParseIP(envelope.Body.GetExternalIPAddressResponse.NewExternalIPAddress)
+
+	return result, nil
 }

internal/upnp/upnp_test.go

@@ -16,17 +16,27 @@
 package upnp
 
 import (
-	"os"
+	"encoding/xml"
 	"testing"
 )
 
-func TestGetTechnicolorRootURL(t *testing.T) {
-	r, _ := os.Open("testdata/technicolor.xml")
-	_, action, err := getServiceURLReader("http://localhost:1234/", r)
+func TestExternalIPParsing(t *testing.T) {
+	soap_response :=
+		[]byte(`<s:Envelope xmlns:s="http://schemas.xmlsoap.org/soap/envelope/" s:encodingStyle="http://schemas.xmlsoap.org/soap/encoding/">
+		<s:Body>
+			<u:GetExternalIPAddressResponse xmlns:u="urn:schemas-upnp-org:service:WANIPConnection:1">
+			<NewExternalIPAddress>1.2.3.4</NewExternalIPAddress>
+			</u:GetExternalIPAddressResponse>
+		</s:Body>
+		</s:Envelope>`)
+
+	envelope := &soapGetExternalIPAddressResponseEnvelope{}
+	err := xml.Unmarshal(soap_response, envelope)
 	if err != nil {
-		t.Fatal(err)
+		t.Error(err)
 	}
-	if action != "urn:schemas-upnp-org:service:WANPPPConnection:1" {
-		t.Error("Unexpected action", action)
+
+	if envelope.Body.GetExternalIPAddressResponse.NewExternalIPAddress != "1.2.3.4" {
+		t.Error("Parse of SOAP request failed.")
 	}
 }

test/common_test.go

@@ -324,7 +324,7 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) {
 		}
 
 		rn, _ := filepath.Rel(dir, path)
-		if rn == "." {
+		if rn == "." || rn == ".stfolder" {
 			return nil
 		}
 

test/h1/config.xml

@@ -1,25 +1,27 @@
-<configuration version="5">
+<configuration version="6">
     <folder id="default" path="s1" ro="false" rescanIntervalS="10" ignorePerms="false">
         <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
         <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></device>
         <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
         <versioning></versioning>
+        <lenientMtimes>false</lenientMtimes>
     </folder>
     <folder id="s12" path="s12-1" ro="false" rescanIntervalS="10" ignorePerms="false">
         <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
         <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></device>
         <versioning></versioning>
+        <lenientMtimes>false</lenientMtimes>
     </folder>
-    <device id="EJHMPAQ-OGCVORE-ISB4IS3-SYYVJXF-TKJGLTU-66DIQPF-GJ5D2GX-GQ3OWQK" name="s4" compression="true">
+    <device id="EJHMPAQ-OGCVORE-ISB4IS3-SYYVJXF-TKJGLTU-66DIQPF-GJ5D2GX-GQ3OWQK" name="s4" compression="true" introducer="false">
         <address>127.0.0.1:22004</address>
     </device>
-    <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1" compression="true">
+    <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1" compression="true" introducer="false">
         <address>127.0.0.1:22001</address>
     </device>
-    <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU" name="s2" compression="true">
+    <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU" name="s2" compression="true" introducer="false">
         <address>127.0.0.1:22002</address>
     </device>
-    <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU" name="s3" compression="true">
+    <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU" name="s3" compression="true" introducer="false">
         <address>127.0.0.1:22003</address>
     </device>
     <gui enabled="true" tls="false">
@@ -35,7 +37,6 @@
         <localAnnounceEnabled>true</localAnnounceEnabled>
         <localAnnouncePort>21025</localAnnouncePort>
         <localAnnounceMCAddr>[ff32::5222]:21026</localAnnounceMCAddr>
-        <parallelRequests>16</parallelRequests>
         <maxSendKbps>50000</maxSendKbps>
         <maxRecvKbps>50000</maxRecvKbps>
         <reconnectionIntervalS>5</reconnectionIntervalS>
@@ -45,5 +46,8 @@
         <upnpRenewalMinutes>30</upnpRenewalMinutes>
         <urAccepted>-1</urAccepted>
         <restartOnWakeup>true</restartOnWakeup>
+        <autoUpgradeIntervalH>12</autoUpgradeIntervalH>
+        <keepTemporariesH>24</keepTemporariesH>
+        <cacheIgnoredFiles>true</cacheIgnoredFiles>
     </options>
 </configuration>

test/h2/config.xml

@@ -1,27 +1,30 @@
-<configuration version="5">
+<configuration version="6">
     <folder id="default" path="s2" ro="false" rescanIntervalS="15" ignorePerms="false">
         <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
         <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></device>
         <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
         <versioning></versioning>
+        <lenientMtimes>false</lenientMtimes>
     </folder>
     <folder id="s12" path="s12-2" ro="false" rescanIntervalS="15" ignorePerms="false">
         <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
         <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></device>
         <versioning></versioning>
+        <lenientMtimes>false</lenientMtimes>
     </folder>
     <folder id="s23" path="s23-2" ro="false" rescanIntervalS="15" ignorePerms="false">
         <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></device>
         <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
         <versioning></versioning>
+        <lenientMtimes>false</lenientMtimes>
     </folder>
-    <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1" compression="true">
+    <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1" compression="true" introducer="false">
         <address>127.0.0.1:22001</address>
     </device>
-    <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU" name="s2" compression="true">
+    <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU" name="s2" compression="true" introducer="false">
         <address>127.0.0.1:22002</address>
     </device>
-    <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU" name="s3" compression="true">
+    <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU" name="s3" compression="true" introducer="false">
         <address>127.0.0.1:22003</address>
     </device>
     <gui enabled="true" tls="false">
@@ -35,7 +38,6 @@
         <localAnnounceEnabled>true</localAnnounceEnabled>
         <localAnnouncePort>21025</localAnnouncePort>
         <localAnnounceMCAddr>[ff32::5222]:21026</localAnnounceMCAddr>
-        <parallelRequests>16</parallelRequests>
         <maxSendKbps>0</maxSendKbps>
         <maxRecvKbps>0</maxRecvKbps>
         <reconnectionIntervalS>5</reconnectionIntervalS>
@@ -45,5 +47,8 @@
         <upnpRenewalMinutes>30</upnpRenewalMinutes>
         <urAccepted>-1</urAccepted>
         <restartOnWakeup>true</restartOnWakeup>
+        <autoUpgradeIntervalH>12</autoUpgradeIntervalH>
+        <keepTemporariesH>24</keepTemporariesH>
+        <cacheIgnoredFiles>true</cacheIgnoredFiles>
     </options>
 </configuration>

test/h3/config.xml

@@ -1,8 +1,9 @@
-<configuration version="5">
+<configuration version="6">
     <folder id="s23" path="s23-3" ro="false" rescanIntervalS="20" ignorePerms="false">
         <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></device>
         <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
         <versioning></versioning>
+        <lenientMtimes>false</lenientMtimes>
     </folder>
     <folder id="default" path="s3" ro="false" rescanIntervalS="20" ignorePerms="false">
         <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
@@ -11,14 +12,15 @@
         <versioning type="simple">
             <param key="keep" val="5"></param>
         </versioning>
+        <lenientMtimes>false</lenientMtimes>
     </folder>
-    <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1" compression="true">
+    <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1" compression="true" introducer="false">
         <address>127.0.0.1:22001</address>
     </device>
-    <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU" name="s2" compression="true">
+    <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU" name="s2" compression="true" introducer="false">
         <address>127.0.0.1:22002</address>
     </device>
-    <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU" name="s3" compression="true">
+    <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU" name="s3" compression="true" introducer="false">
         <address>127.0.0.1:22003</address>
     </device>
     <gui enabled="true" tls="false">
@@ -32,7 +34,6 @@
         <localAnnounceEnabled>false</localAnnounceEnabled>
         <localAnnouncePort>21025</localAnnouncePort>
         <localAnnounceMCAddr>[ff32::5222]:21026</localAnnounceMCAddr>
-        <parallelRequests>16</parallelRequests>
         <maxSendKbps>0</maxSendKbps>
         <maxRecvKbps>0</maxRecvKbps>
         <reconnectionIntervalS>5</reconnectionIntervalS>
@@ -42,5 +43,8 @@
         <upnpRenewalMinutes>30</upnpRenewalMinutes>
         <urAccepted>-1</urAccepted>
         <restartOnWakeup>true</restartOnWakeup>
+        <autoUpgradeIntervalH>12</autoUpgradeIntervalH>
+        <keepTemporariesH>24</keepTemporariesH>
+        <cacheIgnoredFiles>true</cacheIgnoredFiles>
     </options>
 </configuration>

test/h4/config.xml

@@ -1,28 +1,30 @@
-<configuration version="5">
+<configuration version="6">
     <folder id="unique" path="s4" ro="false" rescanIntervalS="60" ignorePerms="false">
         <device id="EJHMPAQ-OGCVORE-ISB4IS3-SYYVJXF-TKJGLTU-66DIQPF-GJ5D2GX-GQ3OWQK"></device>
         <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
         <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU"></device>
         <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU"></device>
         <versioning></versioning>
+        <lenientMtimes>false</lenientMtimes>
     </folder>
     <folder id="default" path="s4d" ro="false" rescanIntervalS="60" ignorePerms="false">
         <device id="EJHMPAQ-OGCVORE-ISB4IS3-SYYVJXF-TKJGLTU-66DIQPF-GJ5D2GX-GQ3OWQK"></device>
         <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU"></device>
         <versioning></versioning>
+        <lenientMtimes>false</lenientMtimes>
     </folder>
-    <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1" compression="true">
+    <device id="EJHMPAQ-OGCVORE-ISB4IS3-SYYVJXF-TKJGLTU-66DIQPF-GJ5D2GX-GQ3OWQK" name="s4" compression="true" introducer="false">
+        <address>dynamic</address>
+    </device>
+    <device id="I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU" name="s1" compression="true" introducer="false">
         <address>127.0.0.1:22001</address>
     </device>
-    <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU" name="s2" compression="true">
+    <device id="JMFJCXB-GZDE4BN-OCJE3VF-65GYZNU-AIVJRET-3J6HMRQ-AUQIGJO-FKNHMQU" name="s2" compression="true" introducer="false">
         <address>127.0.0.1:22002</address>
     </device>
-    <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU" name="s3" compression="true">
+    <device id="373HSRP-QLPNLIE-JYKZVQF-P4PKZ63-R2ZE6K3-YD442U2-JHBGBQG-WWXAHAU" name="s3" compression="true" introducer="false">
         <address>127.0.0.1:22003</address>
     </device>
-    <device id="EJHMPAQ-OGCVORE-ISB4IS3-SYYVJXF-TKJGLTU-66DIQPF-GJ5D2GX-GQ3OWQK" name="s4" compression="true">
-        <address>dynamic</address>
-    </device>
     <gui enabled="true" tls="false">
         <address>127.0.0.1:8084</address>
         <apikey>abc123</apikey>
@@ -34,7 +36,6 @@
         <localAnnounceEnabled>false</localAnnounceEnabled>
         <localAnnouncePort>21025</localAnnouncePort>
         <localAnnounceMCAddr>[ff32::5222]:21026</localAnnounceMCAddr>
-        <parallelRequests>16</parallelRequests>
         <maxSendKbps>0</maxSendKbps>
         <maxRecvKbps>0</maxRecvKbps>
         <reconnectionIntervalS>10</reconnectionIntervalS>
@@ -44,5 +45,8 @@
         <upnpRenewalMinutes>30</upnpRenewalMinutes>
         <urAccepted>-1</urAccepted>
         <restartOnWakeup>true</restartOnWakeup>
+        <autoUpgradeIntervalH>12</autoUpgradeIntervalH>
+        <keepTemporariesH>24</keepTemporariesH>
+        <cacheIgnoredFiles>true</cacheIgnoredFiles>
     </options>
 </configuration>

test/test-delupd.sh

@@ -30,7 +30,7 @@ go build json.go
 start() {
 	echo "Starting..."
 	for i in 1 2 3 ; do
-		STTRACE=model,scanner STPROFILER=":909$i" syncthing -home "h$i" > "$i.out" 2>&1 &
+		STTRACE=model,scanner STPROFILER=":909$i" ../bin/syncthing -home "h$i" > "$i.out" 2>&1 &
 	done
 }
 
@@ -64,7 +64,7 @@ testConvergence() {
 
 	for i in 1 2 3 12-1 12-2 23-2 23-3; do
 		pushd "s$i" >/dev/null
-		../md5r -l | sort | grep -v .stversions > ../md5-$i
+		../md5r -l | sort | grep -v .stversions | grep -v .stfolder > ../md5-$i
 		popd >/dev/null
 	done
 
@@ -112,6 +112,7 @@ alterFiles() {
 			set +o pipefail
 			find . -type f \
 				| grep -v timechanged \
+				| grep -v .stfolder \
 				| sort -k 1.16 \
 				| head -n "$todelete" \
 				| xargs rm -f
@@ -126,7 +127,7 @@ alterFiles() {
 		chmod 500 ro-test
 		touch "timechanged-$i"
 
-		../md5r -l | sort | grep -v .stversions > ../md5-$i
+		../md5r -l | sort | grep -v .stversions | grep -v .stfolder > ../md5-$i
 		popd >/dev/null
 	done
 
@@ -157,7 +158,7 @@ done
 echo "MD5-summing..."
 for i in 1 12-2 23-3 ; do
 	pushd "s$i" >/dev/null
-	../md5r -l | sort > ../md5-$i
+	../md5r -l | grep -v .stfolder | sort > ../md5-$i
 	popd >/dev/null
 done
 

test/test-folders.sh

@@ -26,8 +26,8 @@ go build json.go
 
 start() {
 	echo "Starting..."
-	STTRACE=model,scanner STPROFILER=":9091" syncthing -home "f1" > 1.out 2>&1 &
-	STTRACE=model,scanner STPROFILER=":9092" syncthing -home "f2" > 2.out 2>&1 &
+	STTRACE=model,scanner STPROFILER=":9091" ../bin/syncthing -home "f1" > 1.out 2>&1 &
+	STTRACE=model,scanner STPROFILER=":9092" ../bin/syncthing -home "f2" > 2.out 2>&1 &
 	sleep 1
 }
 

test/test-merge.sh

@@ -30,7 +30,7 @@ go build json.go
 start() {
 	echo "Starting..."
 	for i in 1 2 3 4 ; do
-		STTRACE=files,model,puller,versioner STPROFILER=":909$i" syncthing -home "h$i" > "$i.out" 2>&1 &
+		STTRACE=files,model,puller,versioner STPROFILER=":909$i" ../bin/syncthing -home "h$i" > "$i.out" 2>&1 &
 	done
 }
 
@@ -43,9 +43,9 @@ stop() {
 
 clean() {
 	if [[ $(uname -s) == "Linux" ]] ; then
-		grep -v .stversions | grep -v utf8-nfd
+		grep -v .stversions | grep -v .stfolder | grep -v utf8-nfd
 	else
-		grep -v .stversions
+		grep -v .stversions | grep -v .stfolder
 	fi
 }