Fix UPnP line endings (ref #211)

"Simple File Versioning" -> "File Versioning"

beacon/beacon.go

@@ -102,9 +102,7 @@ func (b *Beacon) writer() {
 				if debug {
 					l.Debugln(err)
 				}
-				return
-			}
-			if debug {
+			} else if debug {
 				l.Debugf("sent %d bytes to %s", len(bs), dst)
 			}
 		}

cmd/syncthing/main.go

@@ -203,9 +203,9 @@ func main() {
 		l.FatalErr(err)
 		cfg.GUI.Address = fmt.Sprintf("127.0.0.1:%d", port)
 
-		port, err = getFreePort("", 22000)
+		port, err = getFreePort("0.0.0.0", 22000)
 		l.FatalErr(err)
-		cfg.Options.ListenAddress = []string{fmt.Sprintf(":%d", port)}
+		cfg.Options.ListenAddress = []string{fmt.Sprintf("0.0.0.0:%d", port)}
 
 		saveConfig()
 		l.Infof("Edit %s to taste or use the GUI\n", cfgFile)
@@ -695,6 +695,7 @@ func expandTilde(p string) string {
 		return getHomeDir()
 	}
 
+	p = filepath.FromSlash(p)
 	if !strings.HasPrefix(p, fmt.Sprintf("~%c", os.PathSeparator)) {
 		return p
 	}

config/config.go

@@ -244,7 +244,7 @@ func Load(rd io.Reader, myID string) (Configuration, error) {
 		repo := &cfg.Repositories[i]
 
 		if len(repo.Directory) == 0 {
-			repo.Invalid = "empty directory"
+			repo.Invalid = "no directory configured"
 			continue
 		}
 

discover/discover.go

@@ -187,9 +187,8 @@ func (d *Discoverer) sendExternalAnnouncements() {
 	} else {
 		buf = d.announcementPkt()
 	}
-	var errCounter = 0
 
-	for errCounter < maxErrors {
+	for {
 		var ok bool
 
 		if debug {
@@ -201,11 +200,8 @@ func (d *Discoverer) sendExternalAnnouncements() {
 			if debug {
 				l.Debugln("discover: warning:", err)
 			}
-			errCounter++
 			ok = false
 		} else {
-			errCounter = 0
-
 			// Verify that the announce server responds positively for our node ID
 
 			time.Sleep(1 * time.Second)
@@ -214,7 +210,6 @@ func (d *Discoverer) sendExternalAnnouncements() {
 				l.Debugln("discover: external lookup check:", res)
 			}
 			ok = len(res) > 0
-
 		}
 
 		d.extAnnounceOKmut.Lock()
@@ -227,7 +222,6 @@ func (d *Discoverer) sendExternalAnnouncements() {
 			time.Sleep(60 * time.Second)
 		}
 	}
-	l.Warnf("Global discovery: %v: stopping due to too many errors: %v", remote, err)
 }
 
 func (d *Discoverer) recvAnnouncements() {

gui/app.js

@@ -648,6 +648,12 @@ syncthing.filter('shortPath', function () {
     }
 });
 
+syncthing.filter('clean', function () {
+    return function (input) {
+        return encodeURIComponent(input).replace(/%/g, '');
+    }
+});
+
 syncthing.directive('optionEditor', function () {
     return {
         restrict: 'C',
@@ -680,3 +686,25 @@ syncthing.directive('uniqueRepo', function() {
         }
     };
 });
+
+syncthing.directive('validNodeid', function() {
+    return {
+        require: 'ngModel',
+        link: function(scope, elm, attrs, ctrl) {
+            ctrl.$parsers.unshift(function(viewValue) {
+                if (scope.editingExisting) {
+                    // we shouldn't validate
+                    ctrl.$setValidity('validNodeid', true);
+                } else {
+                    var cleaned = viewValue.replace(/ /g, '').replace(/-/g, '').toUpperCase().trim();
+                    if (cleaned.match(/^[A-Z2-7]{52}$/)) {
+                        ctrl.$setValidity('validNodeid', true);
+                    } else {
+                        ctrl.$setValidity('validNodeid', false);
+                    }
+                }
+                return viewValue;
+            });
+        }
+    };
+});

gui/index.html

@@ -60,6 +60,7 @@
     }
 
     .table th {
+      white-space:nowrap;
       font-weight: 400;
     }
 
@@ -124,13 +125,13 @@
           <div class="panel panel-{{repoClass(repo.ID)}}" ng-repeat="repo in repoList()">
             <div class="panel-heading">
               <h3 class="panel-title">
-                <a data-toggle="collapse" data-parent="#repositories" href="#repo-{{repo.ID}}">
+                <a data-toggle="collapse" data-parent="#repositories" href="#repo-{{repo.ID | clean}}">
                   <span class="glyphicon glyphicon-hdd"></span> {{repo.Directory | shortPath}}
                   <span class="pull-right hidden-xs">{{repoStatus(repo.ID)}}</span>
                 </a>
               </h3>
             </div>
-            <div id="repo-{{repo.ID}}" class="panel-collapse collapse">
+            <div id="repo-{{repo.ID | clean}}" class="panel-collapse collapse">
               <div class="panel-body">
                 <div class="table-responsive">
                   <table class="table table-condensed table-striped">
@@ -143,6 +144,10 @@
                         <th><span class="glyphicon glyphicon-folder-open"></span>&emsp;Folder</th>
                         <td class="text-right">{{repo.Directory}}</td>
                       </tr>
+                      <tr ng-if="model[repo.ID].invalid">
+                        <th><span class="glyphicon glyphicon-warning-sign"></span>&emsp;Error</th>
+                        <td class="text-right">{{model[repo.ID].invalid}}</td>
+                      </tr>
                       <tr>
                         <th><span class="glyphicon glyphicon-comment"></span>&emsp;Synchronization</th>
                         <td class="text-right">{{repoStatus(repo.ID)}}</td>
@@ -194,10 +199,10 @@
           <div class="panel panel-default" ng-repeat="nodeCfg in [thisNode()]">
             <div class="panel-heading">
               <h3 class="panel-title">
-                <a data-toggle="collapse" data-parent="#nodes" href="#node-{{nodeCfg.NodeID}}"><span class="glyphicon glyphicon-home"></span> {{nodeName(nodeCfg)}}</a>
+                <a data-toggle="collapse" data-parent="#nodes" href="#node-{{nodeCfg.NodeID | clean}}"><span class="glyphicon glyphicon-home"></span> {{nodeName(nodeCfg)}}</a>
               </h3>
             </div>
-            <div id="node-{{nodeCfg.NodeID}}" class="panel-collapse collapse in">
+            <div id="node-{{nodeCfg.NodeID | clean}}" class="panel-collapse collapse in">
               <div class="panel-body">
                 <div class="table-responsive">
                   <table class="table table-condensed table-striped">
@@ -414,13 +419,14 @@
           <form role="form" name="nodeEditor">
             <div class="form-group" ng-class="{'has-error': nodeEditor.nodeID.$invalid && nodeEditor.nodeID.$dirty}">
               <label for="nodeID">Node ID</label>
-              <input ng-if="!editingExisting" name="nodeID" id="nodeID" class="form-control text-monospace" type="text" ng-model="currentNode.NodeID" required></input>
+              <input ng-if="!editingExisting" name="nodeID" id="nodeID" class="form-control text-monospace" type="text" ng-model="currentNode.NodeID" required valid-nodeid></input>
               <div ng-if="editingExisting" class="well well-sm text-monospace">{{currentNode.NodeID | chunkID}}</div>
               <p class="help-block">
                 <span ng-if="nodeEditor.nodeID.$valid || nodeEditor.nodeID.$pristine">The node ID to enter here can be found in the "Edit > Show ID" dialog on the other node. Spaces and dashes are optional (ignored).
                   <span ng-show="!editingExisting">When adding a new node, keep in mind that <em>this node</em> must be added on the other side too.</span>
                 </span>
                 <span ng-if="nodeEditor.nodeID.$error.required && nodeEditor.nodeID.$dirty">The node ID cannot be blank.</span>
+                <span ng-if="nodeEditor.nodeID.$error.validNodeid && nodeEditor.nodeID.$dirty">The entered node ID does not look valid. It should be a 52 character string consisting of letters and numbers, with spaces and dashes being optional.</span>
               </p>
             </div>
             <div class="form-group">
@@ -459,11 +465,12 @@
               <div class="col-md-12">
                 <div class="form-group" ng-class="{'has-error': repoEditor.repoID.$invalid && repoEditor.repoID.$dirty}">
                   <label for="repoID">Repository ID</label>
-                  <input name="repoID" placeholder="documents" ng-disabled="editingExisting" id="repoID" class="form-control" type="text" ng-model="currentRepo.ID" required unique-repo></input>
+                  <input name="repoID" placeholder="documents" ng-disabled="editingExisting" id="repoID" class="form-control" type="text" ng-model="currentRepo.ID" required unique-repo ng-pattern="/^[a-zaZ0-9-_.]{1,64}$/"></input>
                   <p class="help-block">
                     <span ng-if="repoEditor.repoID.$valid || repoEditor.repoID.$pristine">Short identifier for the repository. Must be the same on all cluster nodes.</span>
                     <span ng-if="repoEditor.repoID.$error.uniqueRepo">The repository ID must be unique.</span>
                     <span ng-if="repoEditor.repoID.$error.required && repoEditor.repoID.$dirty">The repository ID cannot be blank.</span>
+                    <span ng-if="repoEditor.repoID.$error.pattern && repoEditor.repoID.$dirty">The repository ID must be a short identifier (64 characters or less) consisting of letters, numbers and the the <code>-_.</code> characters only.</span>
                   </p>
                 </div>
                 <div class="form-group" ng-class="{'has-error': repoEditor.repoPath.$invalid && repoEditor.repoPath.$dirty}">
@@ -508,7 +515,7 @@
                 <div class="form-group">
                   <div class="checkbox">
                     <label>
-                      <input type="checkbox" ng-model="currentRepo.simpleFileVersioning"> Simple File Versioning
+                      <input type="checkbox" ng-model="currentRepo.simpleFileVersioning"> File Versioning
                     </label>
                   </div>
                   <p class="help-block">Files are moved to date stamped versions in a <code>.stversions</code> folder when replaced or deleted by syncthing.</p>

model/blockqueue.go

@@ -16,6 +16,7 @@ type bqBlock struct {
 	file  scanner.File
 	block scanner.Block   // get this block from the network
 	copy  []scanner.Block // copy these blocks from the old version of the file
+	first bool
 	last  bool
 }
 
@@ -47,24 +48,30 @@ func (q *blockQueue) addBlock(a bqAdd) {
 			return
 		}
 	}
+
+	l := len(a.need)
+
 	if len(a.have) > 0 {
 		// First queue a copy operation
 		q.queued = append(q.queued, bqBlock{
-			file: a.file,
-			copy: a.have,
+			file:  a.file,
+			copy:  a.have,
+			first: true,
+			last:  l == 0,
 		})
 	}
+
 	// Queue the needed blocks individually
-	l := len(a.need)
 	for i, b := range a.need {
 		q.queued = append(q.queued, bqBlock{
 			file:  a.file,
 			block: b,
+			first: len(a.have) == 0 && i == 0,
 			last:  i == l-1,
 		})
 	}
 
-	if l == 0 {
+	if len(a.need)+len(a.have) == 0 {
 		// If we didn't have anything to fetch, queue an empty block with the "last" flag set to close the file.
 		q.queued = append(q.queued, bqBlock{
 			file: a.file,

model/model.go

@@ -275,7 +275,8 @@ func (m *Model) Index(nodeID string, repo string, fs []protocol.FileInfo) {
 	if r, ok := m.repoFiles[repo]; ok {
 		r.Replace(id, files)
 	} else {
-		l.Warnf("Index from %s for nonexistant repo %q; dropping", nodeID, repo)
+		l.Warnf("Index from %s for unexpected repo %q; verify configuration", nodeID, repo)
+
 	}
 	m.rmut.RUnlock()
 }
@@ -628,7 +629,10 @@ func (m *Model) ScanRepos() {
 	for _, repo := range repos {
 		repo := repo
 		go func() {
-			m.ScanRepo(repo)
+			err := m.ScanRepo(repo)
+			if err != nil {
+				invalidateRepo(m.cfg, repo, err)
+			}
 			wg.Done()
 		}()
 	}

model/puller.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"os"
 	"path/filepath"
+	"runtime"
 	"time"
 
 	"github.com/calmh/syncthing/buffers"
@@ -258,7 +259,7 @@ func (p *puller) fixupDirectories() {
 			return nil
 		}
 
-		if !scanner.PermsEqual(cur.Flags, uint32(info.Mode())) {
+		if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(cur.Flags) && !scanner.PermsEqual(cur.Flags, uint32(info.Mode())) {
 			err := os.Chmod(path, os.FileMode(cur.Flags)&os.ModePerm)
 			if err != nil {
 				l.Warnf("Restoring folder flags: %q: %v", path, err)
@@ -274,7 +275,10 @@ func (p *puller) fixupDirectories() {
 			t := time.Unix(cur.Modified, 0)
 			err := os.Chtimes(path, t, t)
 			if err != nil {
-				l.Warnf("Restoring folder modtime: %q: %v", path, err)
+				if runtime.GOOS != "windows" {
+					// https://code.google.com/p/go/issues/detail?id=8090
+					l.Warnf("Restoring folder modtime: %q: %v", path, err)
+				}
 			} else {
 				changed++
 				if debug {
@@ -369,6 +373,29 @@ func (p *puller) handleBlock(b bqBlock) bool {
 		return true
 	}
 
+	if len(b.copy) > 0 && len(b.copy) == len(b.file.Blocks) && b.last {
+		// We are supposed to copy the entire file, and then fetch nothing.
+		// We don't actually need to make the copy.
+		if debug {
+			l.Debugln("taking shortcut:", f)
+		}
+		fp := filepath.Join(p.repoCfg.Directory, f.Name)
+		t := time.Unix(f.Modified, 0)
+		err := os.Chtimes(fp, t, t)
+		if debug && err != nil {
+			l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
+		}
+		if !p.repoCfg.IgnorePerms && protocol.HasPermissionBits(f.Flags) {
+			err = os.Chmod(fp, os.FileMode(f.Flags&0777))
+			if debug && err != nil {
+				l.Debugf("pull: error: %q / %q: %v", p.repoCfg.ID, f.Name, err)
+			}
+		}
+
+		p.model.updateLocal(p.repoCfg.ID, f)
+		return true
+	}
+
 	of, ok := p.openFiles[f.Name]
 	of.done = b.last
 

model/util.go

@@ -81,17 +81,8 @@ func compareClusterConfig(local, remote protocol.ClusterConfigMessage) error {
 					}
 				}
 			}
-		} else {
-			return ClusterConfigMismatch(fmt.Errorf("remote is missing repository %q", repo))
 		}
 	}
 
-	for repo := range rm {
-		if _, ok := lm[repo]; !ok {
-			return ClusterConfigMismatch(fmt.Errorf("remote has extra repository %q", repo))
-		}
-
-	}
-
 	return nil
 }

model/util_test.go

@@ -20,24 +20,6 @@ var testcases = []struct {
 		remote: protocol.ClusterConfigMessage{ClientName: "c", ClientVersion: "d"},
 		err:    "",
 	},
-	{
-		local: protocol.ClusterConfigMessage{
-			Repositories: []protocol.Repository{
-				{ID: "foo"},
-			},
-		},
-		remote: protocol.ClusterConfigMessage{ClientName: "c", ClientVersion: "d"},
-		err:    `remote is missing repository "foo"`,
-	},
-	{
-		local: protocol.ClusterConfigMessage{ClientName: "c", ClientVersion: "d"},
-		remote: protocol.ClusterConfigMessage{
-			Repositories: []protocol.Repository{
-				{ID: "foo"},
-			},
-		},
-		err: `remote has extra repository "foo"`,
-	},
 	{
 		local: protocol.ClusterConfigMessage{
 			Repositories: []protocol.Repository{
@@ -53,38 +35,6 @@ var testcases = []struct {
 		},
 		err: "",
 	},
-	{
-		local: protocol.ClusterConfigMessage{
-			Repositories: []protocol.Repository{
-				{ID: "quux"},
-				{ID: "foo"},
-				{ID: "bar"},
-			},
-		},
-		remote: protocol.ClusterConfigMessage{
-			Repositories: []protocol.Repository{
-				{ID: "bar"},
-				{ID: "quux"},
-			},
-		},
-		err: `remote is missing repository "foo"`,
-	},
-	{
-		local: protocol.ClusterConfigMessage{
-			Repositories: []protocol.Repository{
-				{ID: "quux"},
-				{ID: "bar"},
-			},
-		},
-		remote: protocol.ClusterConfigMessage{
-			Repositories: []protocol.Repository{
-				{ID: "bar"},
-				{ID: "foo"},
-				{ID: "quux"},
-			},
-		},
-		err: `remote has extra repository "foo"`,
-	},
 	{
 		local: protocol.ClusterConfigMessage{
 			Repositories: []protocol.Repository{

protocol/debug.go

@@ -0,0 +1,13 @@
+package protocol
+
+import (
+	"os"
+	"strings"
+
+	"github.com/calmh/syncthing/logger"
+)
+
+var (
+	debug = strings.Contains(os.Getenv("STTRACE"), "protocol") || os.Getenv("STTRACE") == "all"
+	l     = logger.DefaultLogger
+)

protocol/protocol.go

@@ -92,8 +92,8 @@ type asyncResult struct {
 }
 
 const (
-	pingTimeout  = 4 * time.Minute
-	pingIdleTime = 5 * time.Minute
+	pingTimeout  = 300 * time.Second
+	pingIdleTime = 600 * time.Second
 )
 
 func NewConnection(nodeID string, reader io.Reader, writer io.Writer, receiver Model) Connection {
@@ -476,11 +476,27 @@ func (c *rawConnection) pingerLoop() {
 	for {
 		select {
 		case <-ticker:
+			if d := time.Since(c.xr.LastRead()); d < pingIdleTime {
+				if debug {
+					l.Debugln(c.id, "ping skipped after rd", d)
+				}
+				continue
+			}
+			if d := time.Since(c.xw.LastWrite()); d < pingIdleTime {
+				if debug {
+					l.Debugln(c.id, "ping skipped after wr", d)
+				}
+				continue
+			}
 			go func() {
+				if debug {
+					l.Debugln(c.id, "ping ->")
+				}
 				rc <- c.ping()
 			}()
 			select {
 			case ok := <-rc:
+				l.Debugln(c.id, "<- pong")
 				if !ok {
 					c.close(fmt.Errorf("ping failure"))
 				}

scanner/walk.go

@@ -167,7 +167,7 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 				} else {
 					var flags uint32 = protocol.FlagDirectory
 					if w.IgnorePerms {
-						flags |= protocol.FlagNoPermBits
+						flags |= protocol.FlagNoPermBits | 0777
 					} else {
 						flags |= uint32(info.Mode() & os.ModePerm)
 					}
@@ -284,10 +284,12 @@ func (w *Walker) ignoreFile(patterns map[string][]string, file string) bool {
 }
 
 func checkDir(dir string) error {
-	if info, err := os.Stat(dir); err != nil {
+	if info, err := os.Lstat(dir); err != nil {
 		return err
 	} else if !info.IsDir() {
 		return errors.New(dir + ": not a directory")
+	} else if debug {
+		l.Debugln("checkDir", dir, info)
 	}
 	return nil
 }

upnp/upnp.go

@@ -60,14 +60,14 @@ func Discover() (*IGD, error) {
 		return nil, err
 	}
 
-	search := []byte(`
-M-SEARCH * HTTP/1.1
+	searchStr := `M-SEARCH * HTTP/1.1
 Host: 239.255.255.250:1900
 St: urn:schemas-upnp-org:device:InternetGatewayDevice:1
 Man: "ssdp:discover"
 Mx: 3
 
-`)
+`
+	search := []byte(strings.Replace(searchStr, "\n", "\r\n", -1))
 
 	_, err = socket.WriteTo(search, ssdp)
 	if err != nil {

xdr/reader.go

@@ -3,15 +3,17 @@ package xdr
 import (
 	"errors"
 	"io"
+	"time"
 )
 
 var ErrElementSizeExceeded = errors.New("element size exceeded")
 
 type Reader struct {
-	r   io.Reader
-	tot int
-	err error
-	b   [8]byte
+	r    io.Reader
+	tot  int
+	err  error
+	b    [8]byte
+	last time.Time
 }
 
 func NewReader(r io.Reader) *Reader {
@@ -44,6 +46,7 @@ func (r *Reader) ReadBytesMaxInto(max int, dst []byte) []byte {
 	if r.err != nil {
 		return nil
 	}
+	r.last = time.Now()
 	l := int(r.ReadUint32())
 	if r.err != nil {
 		return nil
@@ -74,6 +77,7 @@ func (r *Reader) ReadUint16() uint16 {
 	if r.err != nil {
 		return 0
 	}
+	r.last = time.Now()
 	_, r.err = io.ReadFull(r.r, r.b[:4])
 	r.tot += 4
 	v := uint16(r.b[1]) | uint16(r.b[0])<<8
@@ -88,6 +92,7 @@ func (r *Reader) ReadUint32() uint32 {
 	if r.err != nil {
 		return 0
 	}
+	r.last = time.Now()
 	n, r.err = io.ReadFull(r.r, r.b[:4])
 	if n < 4 {
 		return 0
@@ -105,6 +110,7 @@ func (r *Reader) ReadUint64() uint64 {
 	if r.err != nil {
 		return 0
 	}
+	r.last = time.Now()
 	n, r.err = io.ReadFull(r.r, r.b[:8])
 	r.tot += n
 	v := uint64(r.b[7]) | uint64(r.b[6])<<8 | uint64(r.b[5])<<16 | uint64(r.b[4])<<24 |
@@ -122,3 +128,7 @@ func (r *Reader) Tot() int {
 func (r *Reader) Error() error {
 	return r.err
 }
+
+func (r *Reader) LastRead() time.Time {
+	return r.last
+}

xdr/writer.go

@@ -1,6 +1,9 @@
 package xdr
 
-import "io"
+import (
+	"io"
+	"time"
+)
 
 func pad(l int) int {
 	d := l % 4
@@ -13,10 +16,11 @@ func pad(l int) int {
 var padBytes = []byte{0, 0, 0}
 
 type Writer struct {
-	w   io.Writer
-	tot int
-	err error
-	b   [8]byte
+	w    io.Writer
+	tot  int
+	err  error
+	b    [8]byte
+	last time.Time
 }
 
 func NewWriter(w io.Writer) *Writer {
@@ -34,6 +38,7 @@ func (w *Writer) WriteBytes(bs []byte) (int, error) {
 		return 0, w.err
 	}
 
+	w.last = time.Now()
 	w.WriteUint32(uint32(len(bs)))
 	if w.err != nil {
 		return 0, w.err
@@ -65,6 +70,7 @@ func (w *Writer) WriteUint16(v uint16) (int, error) {
 		return 0, w.err
 	}
 
+	w.last = time.Now()
 	if debug {
 		dl.Debugf("wr uint16=%d", v)
 	}
@@ -85,6 +91,7 @@ func (w *Writer) WriteUint32(v uint32) (int, error) {
 		return 0, w.err
 	}
 
+	w.last = time.Now()
 	if debug {
 		dl.Debugf("wr uint32=%d", v)
 	}
@@ -105,6 +112,7 @@ func (w *Writer) WriteUint64(v uint64) (int, error) {
 		return 0, w.err
 	}
 
+	w.last = time.Now()
 	if debug {
 		dl.Debugf("wr uint64=%d", v)
 	}
@@ -131,3 +139,7 @@ func (w *Writer) Tot() int {
 func (w *Writer) Error() error {
 	return w.err
 }
+
+func (w *Writer) LastWrite() time.Time {
+	return w.last
+}