Reimplement quick startup scan

auto/doc.go

@@ -0,0 +1,2 @@
+// Package auto contains auto generated files for web assets.
+package auto

buffers/buffers.go

@@ -1,3 +1,4 @@
+// Package buffers manages a set of reusable byte buffers.
 package buffers
 
 const (

cid/cid.go

@@ -1,3 +1,4 @@
+// Package cid provides a manager for mappings between node ID:s and connection ID:s.
 package cid
 
 type Map struct {

cmd/syncthing/gui_solaris.go

@@ -0,0 +1,92 @@
+//+build solaris
+
+package main
+
+import (
+	"encoding/binary"
+	"fmt"
+	"os"
+	"time"
+)
+
+type id_t int32
+type ulong_t uint32
+
+type timestruc_t struct {
+	Tv_sec  int64
+	Tv_nsec int64
+}
+
+func (tv timestruc_t) Nano() int64 {
+	return tv.Tv_sec*1e9 + tv.Tv_nsec
+}
+
+type prusage_t struct {
+	Pr_lwpid    id_t        /* lwp id.  0: process or defunct */
+	Pr_count    int32       /* number of contributing lwps */
+	Pr_tstamp   timestruc_t /* real time stamp, time of read() */
+	Pr_create   timestruc_t /* process/lwp creation time stamp */
+	Pr_term     timestruc_t /* process/lwp termination time stamp */
+	Pr_rtime    timestruc_t /* total lwp real (elapsed) time */
+	Pr_utime    timestruc_t /* user level CPU time */
+	Pr_stime    timestruc_t /* system call CPU time */
+	Pr_ttime    timestruc_t /* other system trap CPU time */
+	Pr_tftime   timestruc_t /* text page fault sleep time */
+	Pr_dftime   timestruc_t /* data page fault sleep time */
+	Pr_kftime   timestruc_t /* kernel page fault sleep time */
+	Pr_ltime    timestruc_t /* user lock wait sleep time */
+	Pr_slptime  timestruc_t /* all other sleep time */
+	Pr_wtime    timestruc_t /* wait-cpu (latency) time */
+	Pr_stoptime timestruc_t /* stopped time */
+	Pr_minf     ulong_t     /* minor page faults */
+	Pr_majf     ulong_t     /* major page faults */
+	Pr_nswap    ulong_t     /* swaps */
+	Pr_inblk    ulong_t     /* input blocks */
+	Pr_oublk    ulong_t     /* output blocks */
+	Pr_msnd     ulong_t     /* messages sent */
+	Pr_mrcv     ulong_t     /* messages received */
+	Pr_sigs     ulong_t     /* signals received */
+	Pr_vctx     ulong_t     /* voluntary context switches */
+	Pr_ictx     ulong_t     /* involuntary context switches */
+	Pr_sysc     ulong_t     /* system calls */
+	Pr_ioch     ulong_t     /* chars read and written */
+
+}
+
+func solarisPrusage(pid int, rusage *prusage_t) error {
+	fd, err := os.Open(fmt.Sprintf("/proc/%d/usage", pid))
+	if err != nil {
+		return err
+	}
+	err = binary.Read(fd, binary.LittleEndian, rusage)
+	fd.Close()
+	return err
+}
+
+func init() {
+	go trackCPUUsage()
+}
+
+func trackCPUUsage() {
+	var prevUsage int64
+	var prevTime = time.Now().UnixNano()
+	var rusage prusage_t
+	var pid = os.Getpid()
+	for {
+		time.Sleep(10 * time.Second)
+		err := solarisPrusage(pid, &rusage)
+		if err != nil {
+			warnln(err)
+			continue
+		}
+		curTime := time.Now().UnixNano()
+		timeDiff := curTime - prevTime
+		curUsage := rusage.Pr_utime.Nano() + rusage.Pr_stime.Nano()
+		usageDiff := curUsage - prevUsage
+		cpuUsageLock.Lock()
+		cpuUsagePercent = 100 * float64(usageDiff) / float64(timeDiff)
+		cpuUsageLock.Unlock()
+		prevTime = curTime
+		prevUsage = curUsage
+	}
+}

cmd/syncthing/main.go

@@ -41,19 +41,23 @@ var (
 
 const (
 	usage      = "syncthing [options]"
-	extraUsage = `The following environemnt variables can be set to facilitate debugging:
+	extraUsage = `The following enviroment variables are interpreted by syncthing:
 
- STPROFILER  Set to a listen address such as "127.0.0.1:9090" to start the
-             profiler with HTTP access.
+ STNORESTART  Do not attempt to restart when requested to, instead just exit.
+              Set this variable when running under a service manager such as
+              runit, launchd, etc.
 
- STTRACE     A comma separated string of facilities to trace. The valid
-             facility strings:
-             - "scanner"  (the file change scanner)
-             - "discover" (the node discovery package)
-             - "net"      (connecting and disconnecting, sent/received messages)
-             - "idx"      (index sending and receiving)
-             - "need"     (file need calculations)
-             - "pull"     (file pull activity)`
+ STPROFILER   Set to a listen address such as "127.0.0.1:9090" to start the
+              profiler with HTTP access.
+
+ STTRACE      A comma separated string of facilities to trace. The valid
+              facility strings:
+              - "scanner"  (the file change scanner)
+              - "discover" (the node discovery package)
+              - "net"      (connecting and disconnecting, network messages)
+              - "idx"      (index sending and receiving)
+              - "need"     (file need calculations)
+              - "pull"     (file pull activity)`
 )
 
 func main() {
@@ -235,8 +239,9 @@ func main() {
 		IgnoreFile:     ".stignore",
 		FollowSymlinks: cfg.Options.FollowSymlinks,
 		BlockSize:      BlockSize,
-		Suppressor:     sup,
 		TempNamer:      defTempNamer,
+		Suppressor:     sup,
+		CurrentFiler:   m,
 	}
 	updateLocalModel(m, w)
 
@@ -299,6 +304,12 @@ func main() {
 
 func restart() {
 	infoln("Restarting")
+	if os.Getenv("SMF_FMRI") != "" || os.Getenv("STNORESTART") != "" {
+		// Solaris SMF
+		infoln("Service manager detected; exit instead of restart")
+		os.Exit(0)
+	}
+
 	env := os.Environ()
 	if len(os.Getenv("STRESTART")) == 0 {
 		env = append(env, "STRESTART=1")

cmd/syncthing/model.go

@@ -162,7 +162,9 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
 
 	var tot int64
 	for _, f := range m.global {
-		tot += f.Size
+		if f.Flags&protocol.FlagDeleted == 0 {
+			tot += f.Size
+		}
 	}
 
 	var res = make(map[string]ConnectionInfo)
@@ -178,7 +180,7 @@ func (m *Model) ConnectionStats() map[string]ConnectionInfo {
 
 		var have int64
 		for _, f := range m.remote[node] {
-			if f.Equals(m.global[f.Name]) {
+			if f.Equals(m.global[f.Name]) && f.Flags&protocol.FlagDeleted == 0 {
 				have += f.Size
 			}
 		}
@@ -241,8 +243,10 @@ func (m *Model) InSyncSize() (files, bytes int64) {
 
 	for n, f := range m.local {
 		if gf, ok := m.global[n]; ok && f.Equals(gf) {
-			files++
-			bytes += f.Size
+			if f.Flags&protocol.FlagDeleted == 0 {
+				files++
+				bytes += f.Size
+			}
 		}
 	}
 
@@ -393,7 +397,7 @@ func (m *Model) Request(nodeID, repo, name string, offset int64, size int) ([]by
 		warnf("SECURITY (nonexistent file) REQ(in): %s: %q o=%d s=%d", nodeID, name, offset, size)
 		return nil, ErrNoSuchFile
 	}
-	if lf.Flags&protocol.FlagInvalid != 0 {
+	if lf.Suppressed {
 		return nil, ErrInvalid
 	}
 
@@ -476,6 +480,14 @@ func (m *Model) SeedLocal(fs []protocol.FileInfo) {
 	m.recomputeNeedForGlobal()
 }
 
+// Implements scanner.CurrentFiler
+func (m *Model) CurrentFile(file string) scanner.File {
+	m.lmut.RLock()
+	f := m.local[file]
+	m.lmut.RUnlock()
+	return f
+}
+
 // ConnectedTo returns true if we are connected to the named node.
 func (m *Model) ConnectedTo(nodeID string) bool {
 	m.pmut.RLock()
@@ -806,7 +818,7 @@ func (m *Model) recomputeNeedForFile(gf scanner.File, toAdd []addOrder, toDelete
 	m.lmut.RUnlock()
 
 	if !ok || gf.NewerThan(lf) {
-		if gf.Flags&protocol.FlagInvalid != 0 {
+		if gf.Suppressed {
 			// Never attempt to sync invalid files
 			return toAdd, toDelete
 		}
@@ -885,12 +897,13 @@ func fileFromFileInfo(f protocol.FileInfo) scanner.File {
 		offset += int64(b.Size)
 	}
 	return scanner.File{
-		Name:     f.Name,
-		Size:     offset,
-		Flags:    f.Flags,
-		Modified: f.Modified,
-		Version:  f.Version,
-		Blocks:   blocks,
+		Name:       f.Name,
+		Size:       offset,
+		Flags:      f.Flags &^ protocol.FlagInvalid,
+		Modified:   f.Modified,
+		Version:    f.Version,
+		Blocks:     blocks,
+		Suppressed: f.Flags&protocol.FlagInvalid != 0,
 	}
 }
 
@@ -902,11 +915,15 @@ func fileInfoFromFile(f scanner.File) protocol.FileInfo {
 			Hash: b.Hash,
 		}
 	}
-	return protocol.FileInfo{
+	pf := protocol.FileInfo{
 		Name:     f.Name,
 		Flags:    f.Flags,
 		Modified: f.Modified,
 		Version:  f.Version,
 		Blocks:   blocks,
 	}
+	if f.Suppressed {
+		pf.Flags |= protocol.FlagInvalid
+	}
+	return pf
 }

discover/doc.go

@@ -0,0 +1,2 @@
+// Package discover implements the node discovery protocol.
+package discover

files/set.go

@@ -1,3 +1,4 @@
+// Package fileset provides a set type to track local/remote files with newness checks.
 package fileset
 
 import "sync"

integration/.gitignore

@@ -1,5 +1,8 @@
-files-*
-conf-*
+s1
+s2
+s3
 md5-*
 genfiles
 md5r
+json
+*.idx.gz

integration/genfiles.go

@@ -8,6 +8,7 @@ import (
 	mr "math/rand"
 	"os"
 	"path"
+	"time"
 )
 
 func name() string {
@@ -38,5 +39,10 @@ func main() {
 		rand.Reader.Read(b)
 		p1 := path.Join(p0, n)
 		ioutil.WriteFile(p1, b, 0644)
+
+		os.Chmod(p1, os.FileMode(mr.Intn(0777)|0400))
+
+		t := time.Now().Add(-time.Duration(mr.Intn(30*86400)) * time.Second)
+		os.Chtimes(p1, t, t)
 	}
 }

integration/h1/cert.pem

@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID3jCCAkigAwIBAgIBADALBgkqhkiG9w0BAQUwFDESMBAGA1UEAxMJc3luY3Ro
+aW5nMB4XDTE0MDMxNDA3MDA1M1oXDTQ5MTIzMTIzNTk1OVowFDESMBAGA1UEAxMJ
+c3luY3RoaW5nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEArDOcd5ft
+R7SnalxF1ckU3lDQpgfMIPhFDU//4dvdSSFevrMuVDTbUYhyCfGtg/g+F5TmKhZg
+E2peYhllITupz5MP7OHGaO2GHf2XnUDD4QUO3E+KVAUw7dyFSwy09esqApVLzH3+
+ov+QXyyzmRWPsJe9u18BHU1Hob/RmBhS9m2CAJgzN6EJ8KGjApiW3iR8lD/hjVyi
+IVde8IRD6qYHEJYiPJuziTVcQpCblVYxTz3ScmmT190/O9UvViIpcOPQdwgOdewP
+NNMK35c9Edt0AH5flYp6jgrja9NkLQJ3+KOiro6yl9IUS5w87GMxI8qzI8SgCAZZ
+pYSoLbu1FJPvxV4p5eHwuprBCwmFYZWw6Y7rqH0sN52C+3TeObJCMNP9ilPadqRI
++G0Q99TCaloeR022x33r/8D8SIn3FP35zrlFM+DvqlxoS6glbNb/Bj3p9vN0XONO
+RCuynOGe9F/4h/DaNnrbrRWqJOxBsZTsbbcJaKATfWU/Z9GcC+pUpPRhAgMBAAGj
+PzA9MA4GA1UdDwEB/wQEAwIAoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUH
+AwIwDAYDVR0TAQH/BAIwADALBgkqhkiG9w0BAQUDggGBAFF8dklGoC43fMrUZfb4
+6areRWG8quO6cSX6ATzRQVJ8WJ5VcC7OJk8/FeiYA+wcvUJ/1Zm/VHMYugtOz5M8
+CrWAF1r9D3Xfe5D8qfrEOYG2XjxD2nFHCnkbY4fP+SMSuXaDs7ixQnzw0UFh1wsV
+9Jy/QrgXFAIFZtu1Nz+rrvoAgw24gkDhY3557MbmYfmfPsJ8cw+WJ845sxGMPFF2
+c+5EN0jiSm0AwZK11BMJda36ke829UZctDkopbGEg1peydDR5LiyhiTAPtWn7uT/
+PkzHYLuaECAkVbWC3bZLocMGOP6F1pG+BMr00NJgVy05ASQzi4FPjcZQNNY8s69R
+ZgoCIBaJZq3ti1EsZQ1H0Ynm2c2NMVKdj4czoy8a9ZC+DCuhG7EV5Foh20VhCWgA
+RfPhlHVJthuimsWBx39X85gjSBR017uk0AxOJa6pzh/b/RPCRtUfX8EArInS3XCf
+RvRtdrnBZNI3tiREopZGt0SzgDZUs4uDVBUX8HnHzyFJrg==
+-----END CERTIFICATE-----

integration/h1/config.xml

@@ -0,0 +1,30 @@
+<configuration version="1">
+    <repository directory="s1">
+        <node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA" name="s1">
+            <address>127.0.0.1:22001</address>
+        </node>
+        <node id="JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ" name="s2">
+            <address>127.0.0.1:22002</address>
+        </node>
+        <node id="373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA" name="s3">
+            <address>127.0.0.1:22003</address>
+        </node>
+    </repository>
+    <options>
+        <listenAddress>127.0.0.1:22001</listenAddress>
+        <readOnly>false</readOnly>
+        <allowDelete>true</allowDelete>
+        <followSymlinks>true</followSymlinks>
+        <guiEnabled>true</guiEnabled>
+        <guiAddress>127.0.0.1:8081</guiAddress>
+        <globalAnnounceServer>announce.syncthing.net:22025</globalAnnounceServer>
+        <globalAnnounceEnabled>false</globalAnnounceEnabled>
+        <localAnnounceEnabled>false</localAnnounceEnabled>
+        <parallelRequests>16</parallelRequests>
+        <maxSendKbps>0</maxSendKbps>
+        <rescanIntervalS>60</rescanIntervalS>
+        <reconnectionIntervalS>5</reconnectionIntervalS>
+        <maxChangeKbps>1000</maxChangeKbps>
+        <startBrowser>false</startBrowser>
+    </options>
+</configuration>

integration/h1/key.pem

@@ -0,0 +1,39 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIG5AIBAAKCAYEArDOcd5ftR7SnalxF1ckU3lDQpgfMIPhFDU//4dvdSSFevrMu
+VDTbUYhyCfGtg/g+F5TmKhZgE2peYhllITupz5MP7OHGaO2GHf2XnUDD4QUO3E+K
+VAUw7dyFSwy09esqApVLzH3+ov+QXyyzmRWPsJe9u18BHU1Hob/RmBhS9m2CAJgz
+N6EJ8KGjApiW3iR8lD/hjVyiIVde8IRD6qYHEJYiPJuziTVcQpCblVYxTz3ScmmT
+190/O9UvViIpcOPQdwgOdewPNNMK35c9Edt0AH5flYp6jgrja9NkLQJ3+KOiro6y
+l9IUS5w87GMxI8qzI8SgCAZZpYSoLbu1FJPvxV4p5eHwuprBCwmFYZWw6Y7rqH0s
+N52C+3TeObJCMNP9ilPadqRI+G0Q99TCaloeR022x33r/8D8SIn3FP35zrlFM+Dv
+qlxoS6glbNb/Bj3p9vN0XONORCuynOGe9F/4h/DaNnrbrRWqJOxBsZTsbbcJaKAT
+fWU/Z9GcC+pUpPRhAgMBAAECggGAL8+Unc/c3Y/W+7zq1tShqqgdhjub/XtxEKUp
+kngNFITjXWc6cb7LNfQAVap4Vq/R7ZI15XGY80sRMYODhJqgJzXZshdtkyx/lEwY
+kFyvBgb1fU3IRlO6phAYIiJBDBZi75ysEvbYgEEcwJAUvWgzIQDAeQmDsbMHNG2h
+r+zw++Kjua6IaeWYcOsv60Safsr6m96wrSMPENrFTVor0TaPt5c3okRIsMvT9ddY
+mzn3Lt0nVQTjO4f+SoqCPhP2FZXqksfKlZlKlr6BLxXGt6b49OrLSXM5eQXIcIZn
+ZDRsO24X5z8156qPgM9cA8oNEjuSdnArUTreBOsTwNoSpf24Qadsv/uTZlaHM19V
+q6zQvkjH3ERcOpixmg48TKdIj8cPYxezvcbNqSbZmdyQuaVlgDbUxwYI8A4IhhWl
+6xhwpX3qPDgw/QHIEngFIWfiIfCk11EPY0SN4cGO6f1rLYug8kqxMPuIQ5Jz9Hhx
+eFSRnr/fWoJcVYG6bMDKn9YWObQBAoHBAM8NahsLbjl8mdT43LH1Od1tDmDch+0Y
+JM7TgiIN/GM3piZSpGMOFqToLAqvY+Gf3l4sPgNs10cqdPAEpMk8MJ/IXGmbKq38
+iVmMaqHTQorCxyUbc54q9AbFU4HKv//F6ZN6K1wSaJt2RBeZpYI+MyBXr5baFiBZ
+ddXtXlqoEcCFyNR0DhlXrlZPs+cnyM2ZDp++lpn9Wfy+zkv36+NWpAkXVnARjxdF
+l6M+L7OlurYAWiyJE4uHUjawAM82i5+w8QKBwQDU6RCN6/AMmVrYqPy+7QcnAq67
+tPDv25gzVExeMKLBAMoz1TkMS+jIF1NMp3cYg5GbLqvx8Qd27fjFbWe/GPeZvlgL
+qdQI/T8J60dHAySMeOFOB2QWXhI1kwh0b2X0SDkTgfdJBKGdrKVcLTuLyVE24exu
+yRc8cXpYwBtVkXNBYFd7XEM+tC4b1khO23OJXHJUen9+hgsmn8/zUjASAoq3+Zly
+J+OHwwXcDcTFLeok3kX3A9NuqIV/Fa9DOGYlenECgcEAvO1onDTZ5uqjE4nhFyDE
+JB+WtxuDi/wz2eV1IM3SNlZY7S8LgLciQmb3iOhxIzdVGGkWTNnLtcwv17LlCho5
+5BJXAKXtU8TTLzrJMdArL6J7RIi//tsCwAreH9h5SVG1yDP5zJGfkftgNoikVSuc
+Sy63sdZdyjbXJtTo+5/QUvPARNuA4e73zRn89jd/Kts2VNz7XpemvND+PKOEQnSU
+SRdab/gVsQ53RyU/MZVPwTKhFXIeu3pGsk/27RzAWn6BAoHBAMIRYwaKDffd/SHJ
+/v+lHEThvBXa21c26ae36hhc6q1UI/tVGrfrpVZldIdFilgs7RbvVsmksvIj/gMv
+M0bL4j0gdC7FcUF0XPaUoBbJdZIZSP0P3ZpJyv1MdYN0WxFsl6IBcD79WrdXPC8m
+B8XmDgIhsppU77onkaa+DOxVNSJdR8BpG95W7ERxcN14SPrm6ku4kOfqFNXzC+C1
+hJ2V9Y22lLiqRUplaLzpS/eTX36VoF6E/T87mtt5D5UNHoaA8QKBwH5sRqZXoatU
+X+vw1MHU5eptMwG7LXR0gw2xmvG3cCN4hbnnBp5YaXlWPiIMmaWhpvschgBIo1TP
+qGWUpMEETGES18NenLBym+tWIXlfuyZH3B4NUi4kItiZaKb09LzmTjFvzdfQzun4
+HzIeigTNBDHdS0rdicNIn83QLZ4pJaOZJHq79+mFYkp+9It7UUoWsws6DGl/qX8o
+0cj4NmJB6QiJa1QCzrGkaajbtThbFoQal9Twk2h3jHgJzX3FbwCpLw==
+-----END RSA PRIVATE KEY-----

integration/h2/cert.pem

@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID3jCCAkigAwIBAgIBADALBgkqhkiG9w0BAQUwFDESMBAGA1UEAxMJc3luY3Ro
+aW5nMB4XDTE0MDMxNDA3MDEwNFoXDTQ5MTIzMTIzNTk1OVowFDESMBAGA1UEAxMJ
+c3luY3RoaW5nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAsIV0syyR
+O56BvIOro4bIqB6iFJsNc4zX8MiM4QPTWgqGlYwsKSVmNppTdlACZCJIqyzoscrF
+qJPto8/e2Fc3oaTdEREGIs7cmc7LSXfot/mAgPpy71SVWtb7xNmXro2JJPZjRBCS
+pl1ulPug+/8w7fSKQdLMjh4Hp2YlwVBfVu0bYEEW+7Vl9PZVTv+NbTqXYvYVc9R6
+QFIbN/njWAuo2wpjJlY7vqNnSYZyskAaaAC17fFJkVQKKblTeTk1C9PxTmVTB1j9
+yOoD3+V/6IrTYKXdTHGJ1MqdieTHj1jHXe5TOeSB+Hjgq4tr25mPfQ4ixXqDqIcx
+5390DAjInuSKNUJ5pqiFrVe9eIDmySZCg5/JIL3c8phy6g1bxiJN14+Dn0om/0+9
+UrHK8LVzWMmtFRVycWVUYmARWFY3EE10k0RXU2HtzmjfnBkRrl13b0ExizlA1qJ3
+3ngxF5rNEDSMpwf4og5uYOjRUPYuvCL9XtQKr254NFO/sg/qqPV4hFWTAgMBAAGj
+PzA9MA4GA1UdDwEB/wQEAwIAoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUH
+AwIwDAYDVR0TAQH/BAIwADALBgkqhkiG9w0BAQUDggGBAAZSU08zAzyuGqKqqU/c
+Pr+xML8oKiJqko5pb3ETDQC+uVw+qUHwiGYsvHI1cih4ix+tKvf+Yaiizp/35VkP
+qwls3a4ljq1Ww0Sf7J87QX0DumYpBGOfoCpmV4MacyjLhpLRKRGZHwIbOeFsmEu9
+oO38co+GvDy4CiAt3tuOdjBNs0gNOAdTTxqgm97raB9oXeg2i4Fb4MCT4UBUdXLM
+ZNLCifza+PWkBxmfBORvlKGeJBruLpXHBWnWEigZSLXIFjn3JJUy4fKd+/JMp063
+8Pjo6zUOckBCH8Lv90vzfrmdlQK555jWpcebN0l9neESEXw19l0OlqkJGVTr6JKq
+w5kjiL4eP7kpKKwCezhDSX3jf4P36wdF8MpOUBxVqfM+Oh5tHIcZctnurhYV7rXs
+jR70FMqWjHBmwemsXGrObNVt8c75yB+19U6DAulr2RhRw5GD74U1znP00eGZ8TJf
+RN1FYilUPCawMYeQoB8WIn9So7zIm0MfOl4KXNWDX02+Kw==
+-----END CERTIFICATE-----

integration/h2/config.xml

@@ -0,0 +1,30 @@
+<configuration version="1">
+    <repository directory="s2">
+        <node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA" name="s1">
+            <address>127.0.0.1:22001</address>
+        </node>
+        <node id="JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ" name="s2">
+            <address>127.0.0.1:22002</address>
+        </node>
+        <node id="373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA" name="s3">
+            <address>127.0.0.1:22003</address>
+        </node>
+    </repository>
+    <options>
+        <listenAddress>127.0.0.2:22002</listenAddress>
+        <readOnly>false</readOnly>
+        <allowDelete>true</allowDelete>
+        <followSymlinks>true</followSymlinks>
+        <guiEnabled>true</guiEnabled>
+        <guiAddress>127.0.0.1:8082</guiAddress>
+        <globalAnnounceServer>announce.syncthing.net:22025</globalAnnounceServer>
+        <globalAnnounceEnabled>false</globalAnnounceEnabled>
+        <localAnnounceEnabled>false</localAnnounceEnabled>
+        <parallelRequests>16</parallelRequests>
+        <maxSendKbps>0</maxSendKbps>
+        <rescanIntervalS>60</rescanIntervalS>
+        <reconnectionIntervalS>5</reconnectionIntervalS>
+        <maxChangeKbps>1000</maxChangeKbps>
+        <startBrowser>false</startBrowser>
+    </options>
+</configuration>

integration/h2/key.pem

@@ -0,0 +1,39 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIG5AIBAAKCAYEAsIV0syyRO56BvIOro4bIqB6iFJsNc4zX8MiM4QPTWgqGlYws
+KSVmNppTdlACZCJIqyzoscrFqJPto8/e2Fc3oaTdEREGIs7cmc7LSXfot/mAgPpy
+71SVWtb7xNmXro2JJPZjRBCSpl1ulPug+/8w7fSKQdLMjh4Hp2YlwVBfVu0bYEEW
++7Vl9PZVTv+NbTqXYvYVc9R6QFIbN/njWAuo2wpjJlY7vqNnSYZyskAaaAC17fFJ
+kVQKKblTeTk1C9PxTmVTB1j9yOoD3+V/6IrTYKXdTHGJ1MqdieTHj1jHXe5TOeSB
++Hjgq4tr25mPfQ4ixXqDqIcx5390DAjInuSKNUJ5pqiFrVe9eIDmySZCg5/JIL3c
+8phy6g1bxiJN14+Dn0om/0+9UrHK8LVzWMmtFRVycWVUYmARWFY3EE10k0RXU2Ht
+zmjfnBkRrl13b0ExizlA1qJ33ngxF5rNEDSMpwf4og5uYOjRUPYuvCL9XtQKr254
+NFO/sg/qqPV4hFWTAgMBAAECggGAH6SMuuGuVyWe1BA2YGX06k4zd8Yjryb8Pql0
+t5Fb/bQNVBmAgQ+3NuqLM5Y8F38dz7GJNPXIYOPDoa3NoLJhwpQvHLQUiYDTgq7T
+OiRIj1ImevhqSgS7kUEgeLUYv62XfAy+1qCx6Siuff5taT7hooZHkm0bRg6UCKoC
+8phZvtdaJPMGD7EAydyuhi7BR2dNY+wBBHZ+Q7F0N6CP5GSSrFE8XM7wfsgD5+Y2
+AUYEdchK1JCAQ5DxEXGrSPu8SpZ/SuhMjLc3/JDwB8SZPT0C1jX7YMeUiPONy9VK
+J6Fdnl0FMhS9VJHocL4o5IU9OLoahAcpq/Z25arm9z7yyxUoO4nVUAl3H9N7+N7A
+cwpbSgMld15bQ9iPV8MCB/eVKzfgLbWuhpZr6h6oJF9pgIq9DDCK/mc9KYzSGd1J
+dOVuizi0dMYS+iOJRFR3kIrNW7dGCecniigZfrrprqqkycl7823VTi0zIU4CHbDm
+ypu/b8sbs+h6mHN71muWAlmChz3hAoHBAN5Cm3ZZeQJj/p7Kb3sn6WAXlRqnnDz9
+fJDaa3788o4VQ3ie4odDNzALF7bHhYnfovXWrh/4XGkjiW98GPczpEFEdYF9gCGO
+mAaHV/unvtjbGF7Wk3xjgaXwPeKXGU8vZrQ4y41u5eZWpA1fwSK3T+AQ79t4R2jr
+kRgFz7iIJ8iQGleI+F9X4PRjhoOSsdaUkJRB6pxvxcsiYIKxDi7VScTx8iD0pgwn
+tgcQ0do1A0ZQsnJMtBnfIj1/J0sSMHEE9QKBwQDLUVyLmVjv0apqDxnNCw0laIFm
+ofp7S/q4pXfDDg3SqrM05Wgm4CHijzKzoqvFLILQvI004LShRcNXTMAsAbbIRzcY
+YbEOYytHB+k9WfEjAFJkNM4qB4w8erELKwnvjflodLgBw9k97cybhYIZCnwWIIHp
+SwXPT9AI5Ck8E6wifo1nWjpgMZtg5PaH6yfGa+o+ahmetKoU+4ENzVeU95XuKbVa
+x/6UW+wNbPqo3oEfV/K25U6WGHoGfX2X8wn/m2cCgcEAnlABXi5i9GH3ZnG5MJcA
+M3L4wNCsiADirmb19LEFsFDTC2LY5hHpiG4OSSIbK1bBQ6zTwG/umvE2HtPdEI+X
+KuoxbLfRAZYJEXVsJROZ6+s7k6nxycMzANh7rB+GZpHT7QEbdDWOyh/ioKgY8Lpz
+yZ0mzEQDUWehpOPWzpElDUYfjURB7d+xm0Ic+TEPPVH7Ha9KBn3S/FsTNWQaPx+r
+eP4BQpoggD30+VlwsKXcHES0ppeeHWODhxxAB8f/+zDVAoHBALJY3GVYTruPn30J
+YgiK+S0nTttImwAs1fHCtBtV6KozMp/j3Ei9svuZwU/yEdsUAGw5+WO4+Lm/CGs7
+2BbCKiPk1F9+0mFcfEoCloZKr0uUrLFZ4L7dgBZNSaASUNTiJTWLrR1fPuEkB6ck
+pcpxeAew3ERYmvAPgt1JxyH737Mib8eJTkuzOCj2r4rqrClR4Fh/mZmtwMRHGh2R
+UpJJ3CreS0cmyBo7yAS+4+HdzEZCT5Y/73+aWO/4hIMVnl+pYQKBwCpUb85zm5zg
+UnZ8nBS22FLGTcvBs8hbyXUtioSNadNteuqk6jsN2F+Pwsh6eHbVHW4Lu9j6Gn+J
+S1ss/ztgGkErvQF/9DpxMeYt01FpvZaUJthThQVQ9xvr9i7utgthtdspNvQ0fux1
+9Xg2fhLnDz707PUt7OhmVW7d+XOfoc19mYZlN0IOHsqMUMphIW97Lp5QWlZXxr23
+Zrv2j5mTvv3Fq2TRDNfz5dwijFMvv7kpGfHA1950ZIbobQvYYsoC7A==
+-----END RSA PRIVATE KEY-----

integration/h3/cert.pem

@@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID3jCCAkigAwIBAgIBADALBgkqhkiG9w0BAQUwFDESMBAGA1UEAxMJc3luY3Ro
+aW5nMB4XDTE0MDMxNDA3MDExMVoXDTQ5MTIzMTIzNTk1OVowFDESMBAGA1UEAxMJ
+c3luY3RoaW5nMIIBojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAypziXAfA
+GG5mEwTIo+dUt3a4SxKnCVWfTXq7RF5ukCdz3jRRRoqy2ujbW5oe7qmqELTUzkJU
+eJkuCyvLIhVsKgt2r1peThy5+3/kUst9Faitnke1/jYtWwSf7WmK9TQBN/tFxdgR
+SM0LpdvsIJQqB5eodR51KrsjkJcKzEHeE2IQA8EKD1LcfXz8p22zdaZ4CNhz7mgF
+ghkRT6NINDZiMtm6R4qXqYyW1MNlYLw+DqXvE2eCQMQOcqI7SSZTeS3eSTP5NQum
+AmaLKn3JZQ9G6Ldn0VSWSPlXqVSaFTI9LnYICTDa4Nj4+L6idwzjmBkxPr8vBjlj
+15eo6xcU4fKqh2/xHIcZYpas1bwo5ljGdcfRm4L1zpxDY+nJFIgRv0Ndpltve+T8
+o3qiqFvIMiphFUtdb2pzgaTi8FW9SHMRFiuj6sNLI0Yb9u8BoN/QsWZnFh6fQoHW
+PKtkJrPZGw4GMMTHeEQe/eLW9VAH7ywPyc2hSzSri44aR44s4ErSV+XfAgMBAAGj
+PzA9MA4GA1UdDwEB/wQEAwIAoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUH
+AwIwDAYDVR0TAQH/BAIwADALBgkqhkiG9w0BAQUDggGBAJshUcNfAveVE115S/NQ
+h+HfqahB3Gw69KmSWgFNXAEyXwOftmz9p2gL6DwfbU1SkbakU/YLJ3/mP4Q69EmX
+axzr108hUJ2qxH091IfKPCuTDtLsFY7g0PI+SBzADFSkoFe79aF/PSjaZk6fjvEl
+44PXzhxqqnSS+9lXQk+DIS6/jSVuX1XYvkkdHRXd9d7nw42sONDdSX8oQAUvGOeR
+CBleWB4AqS6HH2zFVIzRvjDsSg2wUXQjcqMqef7zBMpuWKTIQf+uaXWsEuPK2a3V
+IPwreWGwkxqGTToLagqOdAhgN+owvTJS9PaqxFqlxp8ypIYBnZ5VL5G5p115C17/
+xVENRxX0/oB99L5HGRhAuIGE4LhXq858DzmTZFcwm3fHragPKS/MMI71Z6U5RH2p
+VouosWzxAEx3Piq+ccenPLt6VPO5iFn+ppoBfwAFtNKyTrXcTQm33oJWFz+NSMAl
+vCq5H4c9x5CoWDpX2bdjpnZ3xcSj84fPYpkKBI52eZAvcw==
+-----END CERTIFICATE-----

integration/h3/config.xml

@@ -0,0 +1,30 @@
+<configuration version="1">
+    <repository directory="s3">
+        <node id="I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA" name="s1">
+            <address>127.0.0.1:22001</address>
+        </node>
+        <node id="JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ" name="s2">
+            <address>127.0.0.1:22002</address>
+        </node>
+        <node id="373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA" name="s3">
+            <address>127.0.0.1:22003</address>
+        </node>
+    </repository>
+    <options>
+        <listenAddress>127.0.0.1:22003</listenAddress>
+        <readOnly>false</readOnly>
+        <allowDelete>true</allowDelete>
+        <followSymlinks>true</followSymlinks>
+        <guiEnabled>true</guiEnabled>
+        <guiAddress>127.0.0.1:8083</guiAddress>
+        <globalAnnounceServer>announce.syncthing.net:22025</globalAnnounceServer>
+        <globalAnnounceEnabled>false</globalAnnounceEnabled>
+        <localAnnounceEnabled>false</localAnnounceEnabled>
+        <parallelRequests>16</parallelRequests>
+        <maxSendKbps>0</maxSendKbps>
+        <rescanIntervalS>60</rescanIntervalS>
+        <reconnectionIntervalS>5</reconnectionIntervalS>
+        <maxChangeKbps>1000</maxChangeKbps>
+        <startBrowser>false</startBrowser>
+    </options>
+</configuration>

integration/h3/key.pem

@@ -0,0 +1,39 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIG5AIBAAKCAYEAypziXAfAGG5mEwTIo+dUt3a4SxKnCVWfTXq7RF5ukCdz3jRR
+Roqy2ujbW5oe7qmqELTUzkJUeJkuCyvLIhVsKgt2r1peThy5+3/kUst9Faitnke1
+/jYtWwSf7WmK9TQBN/tFxdgRSM0LpdvsIJQqB5eodR51KrsjkJcKzEHeE2IQA8EK
+D1LcfXz8p22zdaZ4CNhz7mgFghkRT6NINDZiMtm6R4qXqYyW1MNlYLw+DqXvE2eC
+QMQOcqI7SSZTeS3eSTP5NQumAmaLKn3JZQ9G6Ldn0VSWSPlXqVSaFTI9LnYICTDa
+4Nj4+L6idwzjmBkxPr8vBjlj15eo6xcU4fKqh2/xHIcZYpas1bwo5ljGdcfRm4L1
+zpxDY+nJFIgRv0Ndpltve+T8o3qiqFvIMiphFUtdb2pzgaTi8FW9SHMRFiuj6sNL
+I0Yb9u8BoN/QsWZnFh6fQoHWPKtkJrPZGw4GMMTHeEQe/eLW9VAH7ywPyc2hSzSr
+i44aR44s4ErSV+XfAgMBAAECggGBALolpuXsjPUlQIyKoZfMag3gefMnIOW3j5NM
+hg6LP8MbLB3jLSTFOwtaUmZ3U6HrqP6OVNFnKVpfSWkkBA29ZtG+FH2IZgoX5FsH
+JgtXPwWOImy/75mtxr/PoOsrQ2qCK/h116WsHD0pfWEVi3xnA7JUCIYJxJXMtyEZ
+U+dTQKfIOXRpf0eS1lZIZYuhgvY9Shy+WMyZLy5Wv4vONQEbUd7sIHOoBizUqKKz
+HkngyJcGpn7KY2YDek6hdByBpME5fFRo1DmXPeE+pTycDBP3C55JTB6v6kGkqxAl
+hTFK+x8tU9N+d+Wmuvfdm4MdQ1CnLjv+TL01OTYJx4nTDRk8PkVxVfrXdSY/LFib
+4mhghDPglvPHyFljbwkdiah60dAd2dIUhZwax79oxliyV6Ivf3teB5bvvrWIsMCX
+harESDSC6sgkHRGb05u1uYOPpsnAVOmi6CVgsKQtTmRZ6tqBA/O97RBWj92ROJi+
+0PqFs+cLXvpd4u36sI0MeWYO7Eko2QKBwQDo1Mc/fUsquniFrkjjXnGC8Q+tUpRC
+/TS2vjP5zFlgIOD8fSOoNF4VwFapCoG8l5NsOZrFOYvzHKZRtuBHLL0xdAOuxzge
+72mJYDsfO+vwOtg8xMrdtlJqDOnQWU2NVgOB+xSQKILEOm422BRyXbOM3hylsL7Y
+GPSTdqst9jcLODd6VFY4xq9VDebO63z5Ku62K+BerUJsUfrs64ES6U//Pivq4c06
+57BQq1AbVXqd8q9aInOsSQ1nJwg28PuBREUCgcEA3sZQRJSpktsdsnlvk4Nvf+KK
+OdYEAa2hPcflWDn/ZiEzaP0NR5s6NFSYhzGWVAX39lPUuHJ9MZ17CO4GqMojbXJc
+hneWi7+SWNqZn1rxdCl8FU7wJccX479G45lEL0aIN91i+9K0ZFGZg198EQcpX4kt
+s8J3O3n81rLnHg4bgMhJ6BGRIAxtYwmdOSfTmdyewVJfNUookswjC0sD5JDL/jd1
+Yi4CeaFOqRFVY+4ge+whYz9OcfBBj4kBl1OYoq3TAoHAZe35CJd+l8coykVhjYCk
+KxIDrfpQA/+72yDrujk3C3l5ZbAXMyUx503b4odCAuFM3f1d/2fRF+579ZwdFavF
+a+gBULvQmuJvDoA9gdAG25W3Yus4cNXHwLvEhL0D0ZNNV9MmznGdxfBandH9KZBR
+8aKvYe49rndGY2R5TlbTBCtpRjmzwYlh6td6Ky42+RJNjR0qTeiGAsvlEWGMkU3p
+ArIiIeMWqOoTa02EdWL3mjxLfidFArC8mGMjGoJnaNENAoHBAIQ9ZnJ/aPXS+hry
+uQfw0qQwXuscHr68SeW5nmuz6ea/OJxO8q4Z+AAOY4iFJ/5ymJHxi2l+FND58YoI
+eY2CiGs0orXzkTsdmgsCoISW4JOa+JxRgn56Y7T+217JoU8K6Ft5IIPpvMl8Ist4
+R9Z30Nh0PtvhSRPWQv9TrZwKtcrJmg2XN/W1Ss2qbFj8SkCgVODfO8MPZWxWn0rZ
+2dK5HU6nrxd7xl5bIa18q5qpRUEql1spvjAmdVR1+KrSpd2TnwKBwDGptuY2og9y
+PeEipbGrxww44DMFp6026xn0r8NXSlylD8Rihm9z2wQ0vFPjG1VHjupcQYI5bQLB
+piYt1PyPzCs27WrnXU6eEFjhVP4CzZHFKHkPeW6uBt7XbHpR14hxJ54FLN33Rewr
+Vt6xys+K3OHFzgDfBnhtvROYHtNQENiyJ3rMwRZcxXROmR/zj0cVl1R8wdE3XCwA
+rFPm/n3hzKv2OG3KX0KHh1bH0eXPVhIEmPcgz8RgL3uPGkqh8H+sgQ==
+-----END RSA PRIVATE KEY-----

integration/json.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"strconv"
+	"strings"
+)
+
+func main() {
+	log.SetFlags(0)
+	flag.Parse()
+	path := strings.Split(flag.Arg(0), "/")
+
+	var obj map[string]interface{}
+	dec := json.NewDecoder(os.Stdin)
+	dec.UseNumber()
+	dec.Decode(&obj)
+
+	var v interface{} = obj
+	for _, p := range path {
+		switch tv := v.(type) {
+		case map[string]interface{}:
+			v = tv[p]
+		case []interface{}:
+			i, err := strconv.Atoi(p)
+			if err != nil {
+				log.Fatal(err)
+			}
+			v = tv[i]
+		default:
+			return // Silence is golden
+		}
+	}
+	fmt.Println(v)
+}

integration/md5r.go

@@ -9,7 +9,10 @@ import (
 	"path/filepath"
 )
 
+var long bool
+
 func main() {
+	flag.BoolVar(&long, "l", false, "Long output")
 	flag.Parse()
 	args := flag.Args()
 
@@ -37,7 +40,12 @@ func walker(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
-		fmt.Printf("%s  %s\n", sum, path)
+		if long {
+			fmt.Printf("%s  %s 0%03o %d\n", sum, path, info.Mode(), info.ModTime().Unix())
+
+		} else {
+			fmt.Printf("%s  %s\n", sum, path)
+		}
 	}
 
 	return nil

integration/test.sh

@@ -1,74 +1,94 @@
 #!/bin/bash
 
-rm -rf files-* conf-* md5-*
+export STNORESTART=1
 
-extraopts=""
-p=$(pwd)
+id1=I6KAH7666SLLL5PFXSOAUFJCDZYAOMLEKCP2GB3BV5RQST3PSROA
+id2=JMFJCXBGZDE4BOCJE3VF65GYZNAIVJRET3J6HMRAUQIGJOFKNHMQ
+id3=373HSRPQLPNLIJYKZVQFP4PKZ6R2ZE6K3YD442UJHBGBQGWWXAHA
 
 go build genfiles.go
 go build md5r.go
+go build json.go
 
-echo "Setting up (keys)..."
-i1=$(syncthing --home conf-1 2>&1 | awk '/My ID/ {print $7}')
-echo $i1
-i2=$(syncthing --home conf-2 2>&1 | awk '/My ID/ {print $7}')
-echo $i2
-i3=$(syncthing --home conf-3 2>&1 | awk '/My ID/ {print $7}')
-echo $i3
-
-echo "Setting up (files)..."
-for i in 1 2 3 ; do
-	cat >conf-$i/syncthing.ini <<EOT
-[repository]
-dir = $p/files-$i
-
-[nodes]
-$i1 = 127.0.0.1:22001
-$i2 = 127.0.0.1:22002
-$i3 = 127.0.0.1:22003
-
-[settings]
-gui-enabled = false
-listen-address = :2200$i
-EOT
-
-	mkdir files-$i
-	pushd files-$i >/dev/null
-	../genfiles -maxexp 21 -files 400
-	touch empty-$i
-	../md5r > ../md5-$i
-	popd >/dev/null
-done
-
-echo "Starting..."
-for i in 1 2 3 ; do
-	sleep 1
-	syncthing --home conf-$i $extraopts &
-done
-
-cat md5-* | sort > md5-tot
-while true ; do
-	read
-	echo Verifying...
-
-	conv=0
+testConvergence() {
+	echo "Starting..."
 	for i in 1 2 3 ; do
-		pushd files-$i >/dev/null
-		../md5r | sort > ../md5-$i
-		popd >/dev/null
-		if ! cmp md5-$i md5-tot >/dev/null ; then
-			echo $i unconverged
-		else
-			conv=$((conv + 1))
-			echo $i converged
+		sleep 1
+		syncthing -home "h$i" &
+	done
+
+	while true ; do
+		sleep 5
+		s1comp=$(curl -s "http://localhost:8082/rest/connections" | ./json "$id1/Completion")
+		s2comp=$(curl -s "http://localhost:8083/rest/connections" | ./json "$id2/Completion")
+		s3comp=$(curl -s "http://localhost:8081/rest/connections" | ./json "$id3/Completion")
+		s1comp=${s1comp:-0}
+		s2comp=${s2comp:-0}
+		s3comp=${s3comp:-0}
+		tot=$(($s1comp + $s2comp + $s3comp))
+		echo $tot / 300
+		if [[ $tot == 300 ]] ; then
+			echo "Stopping..."
+			pkill syncthing
+			break
 		fi
 	done
 
-	if [[ $conv == 3 ]] ; then
-		kill %1
-		kill %2
-		kill %3
-		exit
-	fi
+	echo "Verifying..."
+	cat md5-* | sort | uniq > md5-tot
+
+	for i in 1 2 3 ; do
+		pushd "s$i" >/dev/null
+		../md5r -l | sort > ../md5-$i
+		popd >/dev/null
+		if ! cmp "md5-$i" md5-tot >/dev/null ; then
+			echo "Fail: instance $i unconverged"
+			diff -u md5-tot "md5-$i"
+			exit
+		else
+			echo "OK: instance $i converged"
+		fi
+	done
+}
+
+echo "Setting up files..."
+for i in 1 2 3 ; do
+	rm -f h$i/*.idx.gz
+	rm -rf "s$i"
+	mkdir "s$i"
+	pushd "s$i" >/dev/null
+	echo "  $i: random nonoverlapping"
+	../genfiles -maxexp 22 -files 600
+	echo "  $i: empty file"
+	touch "empty-$i"
+	echo "  $i: common file"
+	dd if=/dev/urandom of=common bs=1000 count=1000 2>/dev/null
+	popd >/dev/null
 done
 
+# instance 1 common file should be the newest, the other should disappear
+sleep 2
+touch "s1/common"
+
+echo "MD5-summing..."
+for i in 1 2 3 ; do
+	pushd "s$i" >/dev/null
+	../md5r -l > ../md5-$i
+	popd >/dev/null
+done
+grep -v common md5-2 > t ; mv t md5-2
+grep -v common md5-3 > t ; mv t md5-3
+
+testConvergence
+
+echo "Add and remove random files..."
+for i in 1 2 3 ; do
+	pushd "s$i" >/dev/null
+	rm -rf */?[02468ace]
+	../genfiles -maxexp 22 -files 600
+	../md5r -l > ../md5-$i
+	popd >/dev/null
+done
+
+testConvergence
+

protocol/doc.go

@@ -0,0 +1,2 @@
+// Package protocol implements the Block Exchange Protocol.
+package protocol

protocol/protocol.go

@@ -26,8 +26,8 @@ const (
 )
 
 const (
-	FlagDeleted = 1 << 12
-	FlagInvalid = 1 << 13
+	FlagDeleted uint32 = 1 << 12
+	FlagInvalid        = 1 << 13
 )
 
 var (

scanner/doc.go

@@ -0,0 +1,2 @@
+// Package scanner implements a file system scanner and hasher.
+package scanner

scanner/file.go

@@ -3,12 +3,13 @@ package scanner
 import "fmt"
 
 type File struct {
-	Name     string
-	Flags    uint32
-	Modified int64
-	Version  uint32
-	Size     int64
-	Blocks   []Block
+	Name       string
+	Flags      uint32
+	Modified   int64
+	Version    uint32
+	Size       int64
+	Blocks     []Block
+	Suppressed bool
 }
 
 func (f File) String() string {

scanner/walk.go

@@ -9,8 +9,6 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
-
-	"github.com/calmh/syncthing/protocol"
 )
 
 type Walker struct {
@@ -25,10 +23,13 @@ type Walker struct {
 	IgnoreFile string
 	// If TempNamer is not nil, it is used to ignore tempory files when walking.
 	TempNamer TempNamer
+	// If CurrentFiler is not nil, it is queried for the current file before rescanning.
+	CurrentFiler CurrentFiler
 	// If Suppressor is not nil, it is queried for supression of modified files.
+	// Suppressed files will be returned with empty metadata and the Suppressed flag set.
+	// Requires CurrentFiler to be set.
 	Suppressor Suppressor
 
-	previous   map[string]File // file name -> last seen file state
 	suppressed map[string]bool // file name -> suppression status
 }
 
@@ -44,6 +45,11 @@ type Suppressor interface {
 	Suppress(name string, fi os.FileInfo) bool
 }
 
+type CurrentFiler interface {
+	// CurrentFile returns the file as seen at last scan.
+	CurrentFile(name string) File
+}
+
 // Walk returns the list of files found in the local repository by scanning the
 // file system. Files are blockwise hashed.
 func (w *Walker) Walk() (files []File, ignore map[string][]string) {
@@ -95,8 +101,7 @@ func (w *Walker) CleanTempFiles() {
 }
 
 func (w *Walker) lazyInit() {
-	if w.previous == nil {
-		w.previous = make(map[string]File)
+	if w.suppressed == nil {
 		w.suppressed = make(map[string]bool)
 	}
 }
@@ -171,40 +176,30 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 		}
 
 		if info.Mode()&os.ModeType == 0 {
-			modified := info.ModTime().Unix()
-			pf := w.previous[rn]
-
-			if pf.Modified == modified {
-				if nf := uint32(info.Mode()); nf != pf.Flags {
+			if w.CurrentFiler != nil {
+				cf := w.CurrentFiler.CurrentFile(rn)
+				if cf.Modified == info.ModTime().Unix() {
 					if debug {
-						dlog.Println("new flags:", rn)
+						dlog.Println("unchanged:", rn)
 					}
-					pf.Flags = nf
-					pf.Version++
-					w.previous[rn] = pf
-				} else if debug {
-					dlog.Println("unchanged:", rn)
+					*res = append(*res, cf)
+					return nil
 				}
-				*res = append(*res, pf)
-				return nil
-			}
 
-			if w.Suppressor != nil && w.Suppressor.Suppress(rn, info) {
-				if debug {
-					dlog.Println("suppressed:", rn)
+				if w.Suppressor != nil && w.Suppressor.Suppress(rn, info) {
+					if debug {
+						dlog.Println("suppressed:", rn)
+					}
+					if !w.suppressed[rn] {
+						w.suppressed[rn] = true
+						log.Printf("INFO: Changes to %q are being temporarily suppressed because it changes too frequently.", p)
+					}
+					cf.Suppressed = true
+					*res = append(*res, cf)
+				} else if w.suppressed[rn] {
+					log.Printf("INFO: Changes to %q are no longer suppressed.", p)
+					delete(w.suppressed, rn)
 				}
-				if !w.suppressed[rn] {
-					w.suppressed[rn] = true
-					log.Printf("INFO: Changes to %q are being temporarily suppressed because it changes too frequently.", p)
-				}
-				f := pf
-				f.Flags = protocol.FlagInvalid
-				f.Blocks = nil
-				*res = append(*res, f)
-				return nil
-			} else if w.suppressed[rn] {
-				log.Printf("INFO: Changes to %q are no longer suppressed.", p)
-				delete(w.suppressed, rn)
 			}
 
 			fd, err := os.Open(p)
@@ -232,10 +227,9 @@ func (w *Walker) walkAndHashFiles(res *[]File, ign map[string][]string) filepath
 				Name:     rn,
 				Size:     info.Size(),
 				Flags:    uint32(info.Mode()),
-				Modified: modified,
+				Modified: info.ModTime().Unix(),
 				Blocks:   blocks,
 			}
-			w.previous[rn] = f
 			*res = append(*res, f)
 		}
 

xdr/doc.go

@@ -0,0 +1,2 @@
+// Package xdr implements an XDR (RFC 4506) encoder/decoder.
+package xdr