lib/ignore: Match directory contents for patterns ending in / (fixes #3639)

Appends "**" to patterns with a terminal slash, so that directory
contents are ignored, but not the directory itself.

AUTHORS

@@ -87,6 +87,7 @@ Sergey Mishin (ralder) <ralder@yandex.ru>
 Stefan Kuntz (Stefan-Code) <stefan.github@gmail.com> <Stefan.github@gmail.com>
 Stefan Tatschner (rumpelsepp) <stefan@sevenbyte.org> <rumpelsepp@sevenbyte.org>
 Tim Abell (timabell) <tim@timwise.co.uk>
+Tim Howes (timhowes) <timhowes@berkeley.edu>
 Tobias Nygren (tnn2) <tnn@nygren.pp.se>
 Tomas Cerveny (kozec) <kozec@kozec.com>
 Tully Robinson (tojrobinson) <tully@tojr.org>

NICKS

@@ -101,6 +101,7 @@ snnd <dw@risu.io>
 Stefan-Code <stefan.github@gmail.com>
 Stefan-Code <Stefan.github@gmail.com>
 timabell <tim@timwise.co.uk>
+timhowes <timhowes@berkeley.edu>
 tnn2 <tnn@nygren.pp.se>
 tojrobinson <tully@tojr.org>
 tpng <benny.tpng@gmail.com>

cmd/syncthing/gui.go

@@ -53,6 +53,7 @@ type apiService struct {
 	statics            *staticsServer
 	model              modelIntf
 	eventSub           events.BufferedSubscription
+	diskEventSub       events.BufferedSubscription
 	discoverer         discover.CachingMux
 	connectionsService connectionsIntf
 	fss                *folderSummaryService
@@ -113,7 +114,7 @@ type connectionsIntf interface {
 	Status() map[string]interface{}
 }
 
-func newAPIService(id protocol.DeviceID, cfg configIntf, httpsCertFile, httpsKeyFile, assetDir string, m modelIntf, eventSub events.BufferedSubscription, discoverer discover.CachingMux, connectionsService connectionsIntf, errors, systemLog logger.Recorder) *apiService {
+func newAPIService(id protocol.DeviceID, cfg configIntf, httpsCertFile, httpsKeyFile, assetDir string, m modelIntf, eventSub events.BufferedSubscription, diskEventSub events.BufferedSubscription, discoverer discover.CachingMux, connectionsService connectionsIntf, errors, systemLog logger.Recorder) *apiService {
 	service := &apiService{
 		id:                 id,
 		cfg:                cfg,
@@ -122,6 +123,7 @@ func newAPIService(id protocol.DeviceID, cfg configIntf, httpsCertFile, httpsKey
 		statics:            newStaticsServer(cfg.GUI().Theme, assetDir),
 		model:              m,
 		eventSub:           eventSub,
+		diskEventSub:       diskEventSub,
 		discoverer:         discoverer,
 		connectionsService: connectionsService,
 		systemConfigMut:    sync.NewMutex(),
@@ -229,7 +231,8 @@ func (s *apiService) Serve() {
 	getRestMux.HandleFunc("/rest/db/need", s.getDBNeed)                          // folder [perpage] [page]
 	getRestMux.HandleFunc("/rest/db/status", s.getDBStatus)                      // folder
 	getRestMux.HandleFunc("/rest/db/browse", s.getDBBrowse)                      // folder [prefix] [dirsonly] [levels]
-	getRestMux.HandleFunc("/rest/events", s.getEvents)                           // since [limit]
+	getRestMux.HandleFunc("/rest/events", s.getIndexEvents)                      // since [limit]
+	getRestMux.HandleFunc("/rest/events/disk", s.getDiskEvents)                  // since [limit]
 	getRestMux.HandleFunc("/rest/stats/device", s.getDeviceStats)                // -
 	getRestMux.HandleFunc("/rest/stats/folder", s.getFolderStats)                // -
 	getRestMux.HandleFunc("/rest/svc/deviceid", s.getDeviceID)                   // id
@@ -765,11 +768,13 @@ func (s *apiService) postSystemConfig(w http.ResponseWriter, r *http.Request) {
 	// Activate and save
 
 	if err := s.cfg.Replace(to); err != nil {
+		l.Warnln("Replacing config:", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
 	if err := s.cfg.Save(); err != nil {
+		l.Warnln("Saving config:", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
@@ -991,15 +996,22 @@ func (s *apiService) postDBIgnores(w http.ResponseWriter, r *http.Request) {
 	s.getDBIgnores(w, r)
 }
 
-func (s *apiService) getEvents(w http.ResponseWriter, r *http.Request) {
+func (s *apiService) getIndexEvents(w http.ResponseWriter, r *http.Request) {
+	s.fss.gotEventRequest()
+	s.getEvents(w, r, s.eventSub)
+}
+
+func (s *apiService) getDiskEvents(w http.ResponseWriter, r *http.Request) {
+	s.getEvents(w, r, s.diskEventSub)
+}
+
+func (s *apiService) getEvents(w http.ResponseWriter, r *http.Request, eventSub events.BufferedSubscription) {
 	qs := r.URL.Query()
 	sinceStr := qs.Get("since")
 	limitStr := qs.Get("limit")
 	since, _ := strconv.Atoi(sinceStr)
 	limit, _ := strconv.Atoi(limitStr)
 
-	s.fss.gotEventRequest()
-
 	// Flush before blocking, to indicate that we've received the request and
 	// that it should not be retried. Must set Content-Type header before
 	// flushing.
@@ -1007,7 +1019,7 @@ func (s *apiService) getEvents(w http.ResponseWriter, r *http.Request) {
 	f := w.(http.Flusher)
 	f.Flush()
 
-	evs := s.eventSub.Since(since, nil)
+	evs := eventSub.Since(since, nil)
 	if 0 < limit && limit < len(evs) {
 		evs = evs[len(evs)-limit:]
 	}
@@ -1316,7 +1328,7 @@ func addressIsLocalhost(addr string) bool {
 		// There was no port, so we assume the address was just a hostname
 		host = addr
 	}
-	switch host {
+	switch strings.ToLower(host) {
 	case "127.0.0.1", "::1", "localhost":
 		return true
 	default:

cmd/syncthing/gui_test.go

@@ -70,7 +70,7 @@ func TestStopAfterBrokenConfig(t *testing.T) {
 	}
 	w := config.Wrap("/dev/null", cfg)
 
-	srv := newAPIService(protocol.LocalDeviceID, w, "../../test/h1/https-cert.pem", "../../test/h1/https-key.pem", "", nil, nil, nil, nil, nil, nil)
+	srv := newAPIService(protocol.LocalDeviceID, w, "../../test/h1/https-cert.pem", "../../test/h1/https-key.pem", "", nil, nil, nil, nil, nil, nil, nil)
 	srv.started = make(chan string)
 
 	sup := suture.NewSimple("test")
@@ -469,6 +469,7 @@ func startHTTP(cfg *mockedConfig) (string, error) {
 	httpsKeyFile := "../../test/h1/https-key.pem"
 	assetDir := "../../gui"
 	eventSub := new(mockedEventSub)
+	diskEventSub := new(mockedEventSub)
 	discoverer := new(mockedCachingMux)
 	connections := new(mockedConnections)
 	errorLog := new(mockedLoggerRecorder)
@@ -477,7 +478,7 @@ func startHTTP(cfg *mockedConfig) (string, error) {
 
 	// Instantiate the API service
 	svc := newAPIService(protocol.LocalDeviceID, cfg, httpsCertFile, httpsKeyFile, assetDir, model,
-		eventSub, discoverer, connections, errorLog, systemLog)
+		eventSub, diskEventSub, discoverer, connections, errorLog, systemLog)
 	svc.started = addrChan
 
 	// Actually start the API service
@@ -832,9 +833,11 @@ func TestAddressIsLocalhost(t *testing.T) {
 	}{
 		// These are all valid localhost addresses
 		{"localhost", true},
+		{"LOCALHOST", true},
 		{"::1", true},
 		{"127.0.0.1", true},
 		{"localhost:8080", true},
+		{"LOCALHOST:8000", true},
 		{"[::1]:8080", true},
 		{"127.0.0.1:8080", true},
 

cmd/syncthing/main.go

@@ -23,6 +23,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"runtime/debug"
 	"runtime/pprof"
 	"sort"
 	"strconv"
@@ -41,6 +42,7 @@ import (
 	"github.com/syncthing/syncthing/lib/osutil"
 	"github.com/syncthing/syncthing/lib/protocol"
 	"github.com/syncthing/syncthing/lib/rand"
+	"github.com/syncthing/syncthing/lib/sha256"
 	"github.com/syncthing/syncthing/lib/symlinks"
 	"github.com/syncthing/syncthing/lib/tlsutil"
 	"github.com/syncthing/syncthing/lib/upgrade"
@@ -166,6 +168,11 @@ are mostly useful for developers. Use with care.
 
  STNOUPGRADE       Disable automatic upgrades.
 
+ STHASHING         Select the SHA256 hashing package to use. Possible values
+                   are "standard" for the Go standard library implementation,
+                   "minio" for the github.com/minio/sha256-simd implementation,
+                   and blank (the default) for auto detection.
+
  GOMAXPROCS        Set the maximum number of CPU cores to use. Defaults to all
                    available CPU cores.
 
@@ -274,6 +281,9 @@ func parseCommandLineOptions() RuntimeOptions {
 }
 
 func main() {
+	// We want all (our) goroutines in panic traces.
+	debug.SetTraceback("all")
+
 	options := parseCommandLineOptions()
 	l.SetFlags(options.logFlags)
 
@@ -542,6 +552,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 	// events. The LocalChangeDetected event might overwhelm the event
 	// receiver in some situations so we will not subscribe to it here.
 	apiSub := events.NewBufferedSubscription(events.Default.Subscribe(events.AllEvents&^events.LocalChangeDetected), 1000)
+	diskSub := events.NewBufferedSubscription(events.Default.Subscribe(events.LocalChangeDetected), 1000)
 
 	if len(os.Getenv("GOMAXPROCS")) == 0 {
 		runtime.GOMAXPROCS(runtime.NumCPU())
@@ -567,7 +578,9 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 
 	l.Infoln(LongVersion)
 	l.Infoln("My ID:", myID)
-	printHashRate()
+
+	sha256.SelectAlgo()
+	sha256.Report()
 
 	// Emit the Starting event, now that we know who we are.
 
@@ -740,7 +753,7 @@ func syncthingMain(runtimeOptions RuntimeOptions) {
 
 	// GUI
 
-	setupGUI(mainService, cfg, m, apiSub, cachedDiscovery, connectionsService, errors, systemLog, runtimeOptions)
+	setupGUI(mainService, cfg, m, apiSub, diskSub, cachedDiscovery, connectionsService, errors, systemLog, runtimeOptions)
 
 	if runtimeOptions.cpuProfile {
 		f, err := os.Create(fmt.Sprintf("cpu-%d.pprof", os.Getpid()))
@@ -840,22 +853,6 @@ func setupSignalHandling() {
 	}()
 }
 
-// printHashRate prints the hashing performance in MB/s, formatting it with
-// appropriate precision for the value, i.e. 182 MB/s, 18 MB/s, 1.8 MB/s, 0.18
-// MB/s.
-func printHashRate() {
-	hashRate := cpuBench(3, 100*time.Millisecond)
-
-	decimals := 0
-	if hashRate < 1 {
-		decimals = 2
-	} else if hashRate < 10 {
-		decimals = 1
-	}
-
-	l.Infof("Single thread hash performance is ~%.*f MB/s", decimals, hashRate)
-}
-
 func loadConfig() (*config.Wrapper, error) {
 	cfgFile := locations[locConfigFile]
 	cfg, err := config.Load(cfgFile, myID)
@@ -932,7 +929,7 @@ func startAuditing(mainService *suture.Supervisor) {
 	l.Infoln("Audit log in", auditFile)
 }
 
-func setupGUI(mainService *suture.Supervisor, cfg *config.Wrapper, m *model.Model, apiSub events.BufferedSubscription, discoverer discover.CachingMux, connectionsService *connections.Service, errors, systemLog logger.Recorder, runtimeOptions RuntimeOptions) {
+func setupGUI(mainService *suture.Supervisor, cfg *config.Wrapper, m *model.Model, apiSub events.BufferedSubscription, diskSub events.BufferedSubscription, discoverer discover.CachingMux, connectionsService *connections.Service, errors, systemLog logger.Recorder, runtimeOptions RuntimeOptions) {
 	guiCfg := cfg.GUI()
 
 	if !guiCfg.Enabled {
@@ -943,7 +940,7 @@ func setupGUI(mainService *suture.Supervisor, cfg *config.Wrapper, m *model.Mode
 		l.Warnln("Insecure admin access is enabled.")
 	}
 
-	api := newAPIService(myID, cfg, locations[locHTTPSCertFile], locations[locHTTPSKeyFile], runtimeOptions.assetDir, m, apiSub, discoverer, connectionsService, errors, systemLog)
+	api := newAPIService(myID, cfg, locations[locHTTPSCertFile], locations[locHTTPSKeyFile], runtimeOptions.assetDir, m, apiSub, diskSub, discoverer, connectionsService, errors, systemLog)
 	cfg.Subscribe(api)
 	mainService.Add(api)
 

cmd/syncthing/monitor.go

@@ -178,6 +178,22 @@ func copyStderr(stderr io.Reader, dst io.Writer) {
 		if panicFd == nil {
 			dst.Write([]byte(line))
 
+			if strings.Contains(line, "SIGILL") {
+				l.Warnln(`
+*******************************************************************************
+* Crash due to illegal instruction detected. This is most likely due to a CPU *
+* incompatibility with the high performance hashing package. Switching to the *
+* standard hashing package instead. Please report this issue at:              *
+*                                                                             *
+*   https://github.com/syncthing/syncthing/issues                             *
+*                                                                             *
+* Include the details of your CPU.                                            *
+*******************************************************************************
+`)
+				os.Setenv("STHASHING", "standard")
+				return
+			}
+
 			if strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:") {
 				panicFd, err = os.Create(timestampedLoc(locPanicLog))
 				if err != nil {

cmd/syncthing/usage_report.go

@@ -18,12 +18,11 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/sha256-simd"
-
 	"github.com/syncthing/syncthing/lib/config"
 	"github.com/syncthing/syncthing/lib/dialer"
 	"github.com/syncthing/syncthing/lib/model"
 	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/sha256"
 	"github.com/syncthing/syncthing/lib/upgrade"
 	"github.com/thejerf/suture"
 )

etc/linux-systemd/system/syncthing-resume.service

@@ -5,7 +5,7 @@ After=suspend.target
 
 [Service]
 Type=oneshot
-ExecStart=/usr/bin/pkill -HUP -x syncthing
+ExecStart=-/usr/bin/pkill -HUP -x syncthing
 
 [Install]
 WantedBy=suspend.target

etc/linux-systemd/user/syncthing.service

@@ -1,7 +1,6 @@
 [Unit]
 Description=Syncthing - Open Source Continuous File Synchronization
 Documentation=man:syncthing(1)
-After=network.target
 Wants=syncthing-inotify.service
 
 [Service]

gui/default/syncthing/core/aboutModalView.html

@@ -12,7 +12,7 @@
     <p translate>Copyright &copy; 2014-2016 the following Contributors:</p>
     <div class="row">
       <div class="col-md-12" id="contributor-list">
-Jakob Borg, Audrius Butkevicius, Alexander Graf, Anderson Mesquita, Antony Male, Ben Schulz, Caleb Callaway, Daniel Harte, Lars K.W. Gohlke, Lode Hoste, Michael Ploujnikov, Philippe Schommers, Ryan Sullivan, Sergey Mishin, Stefan Tatschner, Aaron Bieber, Adam Piggott, Alessandro G., Alexandre Viau, Andrew Dunham, Andrey D, Antoine Lamielle, Arthur Axel fREW Schmidt, Bart De Vries, Ben Curthoys, Ben Sidhom, Benny Ng, Brandon Philips, Brendan Long, Brian R. Becker, Carsten Hagemann, Cathryne Linenweaver, Cedric Staniewski, Chris Howie, Chris Joel, Colin Kennedy, Daniel Bergmann, Daniel Martí, David Rimmer, Denis A., Dennis Wilson, Dominik Heidler, Elias Jarlebring, Emil Hessman, Erik Meitner, Federico Castagnini, Felix Ableitner, Felix Unterpaintner, Francois-Xavier Gsell, Frank Isemann, Gilli Sigurdsson, Jaakko Hannikainen, Jacek Szafarkiewicz, Jake Peterson, James Patterson, Jaroslav Malec, Jens Diemer, Jochen Voss, Johan Vromans, Karol Różycki, Kelong Cong, Ken'ichi Kamada, Kevin Allen, Laurent Etiemble, Lord Landon Agahnim, Majed Abdulaziz, Marc Laporte, Marc Pujol, Marcin Dziadus, Mateusz Naściszewski, Matt Burke, Max Schulze, Michael Jephcote, Michael Tilli, Nate Morrison, Pascal Jungblut, Peter Hoeg, Phill Luby, Piotr Bejda, Scott Klupfel, Stefan Kuntz, Tim Abell, Tobias Nygren, Tomas Cerveny, Tully Robinson, Tyler Brazier, Veeti Paananen, Victor Buinsky, Vil Brekin, William A. Kennington III, Wulf Weich, Yannic A.
+Jakob Borg, Audrius Butkevicius, Alexander Graf, Anderson Mesquita, Antony Male, Ben Schulz, Caleb Callaway, Daniel Harte, Lars K.W. Gohlke, Lode Hoste, Michael Ploujnikov, Philippe Schommers, Ryan Sullivan, Sergey Mishin, Stefan Tatschner, Aaron Bieber, Adam Piggott, Alessandro G., Alexandre Viau, Andrew Dunham, Andrey D, Antoine Lamielle, Arthur Axel fREW Schmidt, Bart De Vries, Ben Curthoys, Ben Sidhom, Benny Ng, Brandon Philips, Brendan Long, Brian R. Becker, Carsten Hagemann, Cathryne Linenweaver, Cedric Staniewski, Chris Howie, Chris Joel, Colin Kennedy, Daniel Bergmann, Daniel Martí, David Rimmer, Denis A., Dennis Wilson, Dominik Heidler, Elias Jarlebring, Emil Hessman, Erik Meitner, Federico Castagnini, Felix Ableitner, Felix Unterpaintner, Francois-Xavier Gsell, Frank Isemann, Gilli Sigurdsson, Jaakko Hannikainen, Jacek Szafarkiewicz, Jake Peterson, James Patterson, Jaroslav Malec, Jens Diemer, Jochen Voss, Johan Vromans, Karol Różycki, Kelong Cong, Ken'ichi Kamada, Kevin Allen, Laurent Etiemble, Lord Landon Agahnim, Majed Abdulaziz, Marc Laporte, Marc Pujol, Marcin Dziadus, Mateusz Naściszewski, Matt Burke, Max Schulze, Michael Jephcote, Michael Tilli, Nate Morrison, Pascal Jungblut, Peter Hoeg, Phill Luby, Piotr Bejda, Scott Klupfel, Stefan Kuntz, Tim Abell, Tim Howes, Tobias Nygren, Tomas Cerveny, Tully Robinson, Tyler Brazier, Veeti Paananen, Victor Buinsky, Vil Brekin, William A. Kennington III, Wulf Weich, Yannic A.
     </div>
     </div>
     <hr/>

gui/default/syncthing/core/syncthingController.js

@@ -654,7 +654,7 @@ angular.module('syncthing.core')
             if (state === 'error') {
                 return 'stopped'; // legacy, the state is called "stopped" in the GUI
             }
-            if (state === 'idle' && $scope.model[folderCfg.id].needFiles > 0) {
+            if (state === 'idle' && $scope.model[folderCfg.id].needFiles + $scope.model[folderCfg.id].needDeletes > 0) {
                 return 'outofsync';
             }
             if (state === 'scanning') {

lib/config/wrapper.go

@@ -304,15 +304,18 @@ func (w *Wrapper) Device(id protocol.DeviceID) (DeviceConfiguration, bool) {
 func (w *Wrapper) Save() error {
 	fd, err := osutil.CreateAtomic(w.path, 0600)
 	if err != nil {
+		l.Debugln("CreateAtomic:", err)
 		return err
 	}
 
 	if err := w.cfg.WriteXML(fd); err != nil {
+		l.Debugln("WriteXML:", err)
 		fd.Close()
 		return err
 	}
 
 	if err := fd.Close(); err != nil {
+		l.Debugln("Close:", err)
 		return err
 	}
 

lib/ignore/ignore.go

@@ -345,7 +345,7 @@ func parseIgnoreFile(fd io.Reader, currentFile string, seen map[string]bool) ([]
 		case strings.HasSuffix(line, "/**"):
 			err = addPattern(line)
 		case strings.HasSuffix(line, "/"):
-			err = addPattern(line)
+			err = addPattern(line + "**")
 		default:
 			err = addPattern(line)
 			if err == nil {

lib/ignore/ignore_test.go

@@ -718,3 +718,22 @@ func TestIssue3174(t *testing.T) {
 		t.Error("Should match")
 	}
 }
+
+func TestIssue3639(t *testing.T) {
+	stignore := `
+	foo/
+	`
+	pats := New(true)
+	err := pats.Parse(bytes.NewBufferString(stignore), ".stignore")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !pats.Match("foo/bar").IsIgnored() {
+		t.Error("Should match 'foo/bar'")
+	}
+
+	if pats.Match("foo").IsIgnored() {
+		t.Error("Should not match 'foo'")
+	}
+}

lib/model/model.go

@@ -1465,12 +1465,12 @@ func sendIndexTo(minSequence int64, conn protocol.Connection, folder string, fs
 func (m *Model) updateLocalsFromScanning(folder string, fs []protocol.FileInfo) {
 	m.updateLocals(folder, fs)
 
-	// Fire the LocalChangeDetected event to notify listeners about local
-	// updates.
 	m.fmut.RLock()
-	path := m.folderCfgs[folder].Path()
+	folderCfg := m.folderCfgs[folder]
 	m.fmut.RUnlock()
-	m.localChangeDetected(folder, path, fs)
+
+	// Fire the LocalChangeDetected event to notify listeners about local updates.
+	m.localChangeDetected(folderCfg, fs)
 }
 
 func (m *Model) updateLocalsFromPulling(folder string, fs []protocol.FileInfo) {
@@ -1500,9 +1500,8 @@ func (m *Model) updateLocals(folder string, fs []protocol.FileInfo) {
 	})
 }
 
-func (m *Model) localChangeDetected(folder, path string, files []protocol.FileInfo) {
-	// For windows paths, strip unwanted chars from the front
-	path = strings.Replace(path, `\\?\`, "", 1)
+func (m *Model) localChangeDetected(folderCfg config.FolderConfiguration, files []protocol.FileInfo) {
+	path := strings.Replace(folderCfg.Path(), `\\?\`, "", 1)
 
 	for _, file := range files {
 		objType := "file"
@@ -1526,14 +1525,16 @@ func (m *Model) localChangeDetected(folder, path string, files []protocol.FileIn
 			action = "deleted"
 		}
 
-		// The full file path, adjusted to the local path separator character.
+		// The full file path, adjusted to the local path separator character.  Also
+		// for windows paths, strip unwanted chars from the front.
 		path := filepath.Join(path, filepath.FromSlash(file.Name))
 
 		events.Default.Log(events.LocalChangeDetected, map[string]string{
-			"folder": folder,
-			"action": action,
-			"type":   objType,
-			"path":   path,
+			"folderID": folderCfg.ID,
+			"label":    folderCfg.Label,
+			"action":   action,
+			"type":     objType,
+			"path":     path,
 		})
 	}
 }

lib/protocol/bep_extensions.go

@@ -12,9 +12,8 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/minio/sha256-simd"
-
 	"github.com/syncthing/syncthing/lib/rand"
+	"github.com/syncthing/syncthing/lib/sha256"
 )
 
 const (

lib/protocol/deviceid.go

@@ -11,7 +11,7 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/minio/sha256-simd"
+	"github.com/syncthing/syncthing/lib/sha256"
 
 	"github.com/calmh/luhn"
 )

lib/scanner/blocks.go

@@ -11,9 +11,8 @@ import (
 	"fmt"
 	"io"
 
-	"github.com/minio/sha256-simd"
-
 	"github.com/syncthing/syncthing/lib/protocol"
+	"github.com/syncthing/syncthing/lib/sha256"
 )
 
 var SHA256OfNothing = []uint8{0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55}

lib/sha256/sha256.go

@@ -0,0 +1,136 @@
+// Copyright (C) 2016 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package sha256
+
+import (
+	"crypto/rand"
+	cryptoSha256 "crypto/sha256"
+	"fmt"
+	"hash"
+	"os"
+	"time"
+
+	minioSha256 "github.com/minio/sha256-simd"
+	"github.com/syncthing/syncthing/lib/logger"
+)
+
+var l = logger.DefaultLogger.NewFacility("sha256", "SHA256 hashing package")
+
+const (
+	benchmarkingIterations = 3
+	benchmarkingDuration   = 150 * time.Millisecond
+	defaultImpl            = "crypto/sha256"
+	minioImpl              = "minio/sha256-simd"
+)
+
+const (
+	BlockSize = cryptoSha256.BlockSize
+	Size      = cryptoSha256.Size
+)
+
+// May be switched out for another implementation
+var (
+	New    = cryptoSha256.New
+	Sum256 = cryptoSha256.Sum256
+)
+
+var (
+	selectedImpl = defaultImpl
+	cryptoPerf   float64
+	minioPerf    float64
+)
+
+func SelectAlgo() {
+	switch os.Getenv("STHASHING") {
+	case "":
+		// When unset, probe for the fastest implementation.
+		benchmark()
+		if minioPerf > cryptoPerf {
+			selectMinio()
+		}
+
+	case "minio":
+		// When set to "minio", use that. Benchmark anyway to be able to
+		// present the difference.
+		benchmark()
+		selectMinio()
+
+	default:
+		// When set to anything else, such as "standard", use the default Go
+		// implementation. Benchmark that anyway, so we can report something
+		// useful in Report(). Make sure not to touch the minio
+		// implementation as it may be disabled for incompatibility reasons.
+		cryptoPerf = cpuBenchOnce(benchmarkingIterations*benchmarkingDuration, cryptoSha256.New)
+	}
+}
+
+// Report prints a line with the measured hash performance rates for the
+// selected and alternate implementation.
+func Report() {
+	var otherImpl string
+	var selectedRate, otherRate float64
+
+	switch selectedImpl {
+	case defaultImpl:
+		selectedRate = cryptoPerf
+		otherRate = minioPerf
+		otherImpl = minioImpl
+
+	case minioImpl:
+		selectedRate = minioPerf
+		otherRate = cryptoPerf
+		otherImpl = defaultImpl
+	}
+
+	l.Infof("Single thread hash performance is %s using %s (%s using %s).", formatRate(selectedRate), selectedImpl, formatRate(otherRate), otherImpl)
+}
+
+func selectMinio() {
+	New = minioSha256.New
+	Sum256 = minioSha256.Sum256
+	selectedImpl = minioImpl
+}
+
+func benchmark() {
+	// Interleave the tests to achieve some sort of fairness if the CPU is
+	// just in the process of spinning up to full speed.
+	for i := 0; i < benchmarkingIterations; i++ {
+		if perf := cpuBenchOnce(benchmarkingDuration, cryptoSha256.New); perf > cryptoPerf {
+			cryptoPerf = perf
+		}
+		if perf := cpuBenchOnce(benchmarkingDuration, minioSha256.New); perf > minioPerf {
+			minioPerf = perf
+		}
+	}
+}
+
+func cpuBenchOnce(duration time.Duration, newFn func() hash.Hash) float64 {
+	chunkSize := 100 * 1 << 10
+	h := newFn()
+	bs := make([]byte, chunkSize)
+	rand.Reader.Read(bs)
+
+	t0 := time.Now()
+	b := 0
+	for time.Since(t0) < duration {
+		h.Write(bs)
+		b += chunkSize
+	}
+	h.Sum(nil)
+	d := time.Since(t0)
+	return float64(int(float64(b)/d.Seconds()/(1<<20)*100)) / 100
+}
+
+func formatRate(rate float64) string {
+	decimals := 0
+	if rate < 1 {
+		decimals = 2
+	} else if rate < 10 {
+		decimals = 1
+	}
+	return fmt.Sprintf("%.*f MB/s", decimals, rate)
+}

lib/signature/signature.go

@@ -20,7 +20,7 @@ import (
 	"io"
 	"math/big"
 
-	"github.com/minio/sha256-simd"
+	"github.com/syncthing/syncthing/lib/sha256"
 )
 
 // GenerateKeys returns a new key pair, with the private and public key

lib/upnp/upnp.go

@@ -108,19 +108,21 @@ func Discover(renewal, timeout time.Duration) []nat.Device {
 		close(resultChan)
 	}()
 
+	seenResults := make(map[string]bool)
 nextResult:
 	for result := range resultChan {
-		for _, existingResult := range results {
-			if existingResult.ID() == result.ID() {
-				l.Debugf("Skipping duplicate result %s with services:", result.uuid)
-				for _, service := range result.services {
-					l.Debugf("* [%s] %s", service.ID, service.URL)
-				}
-				continue nextResult
+		if seenResults[result.ID()] {
+			l.Debugf("Skipping duplicate result %s with services:", result.uuid)
+			for _, service := range result.services {
+				l.Debugf("* [%s] %s", service.ID, service.URL)
 			}
+			continue nextResult
 		}
 
+		result := result // Reallocate as we need to keep a pointer
 		results = append(results, &result)
+		seenResults[result.ID()] = true
+
 		l.Debugf("UPnP discovery result %s with services:", result.uuid)
 		for _, service := range result.services {
 			l.Debugf("* [%s] %s", service.ID, service.URL)

test/http_test.go

@@ -27,6 +27,7 @@ var jsonEndpoints = []string{
 	"/rest/db/status?folder=default",
 	"/rest/db/browse?folder=default",
 	"/rest/events?since=-1&limit=5",
+	"/rest/events/disk?since=-1&limit=5",
 	"/rest/stats/device",
 	"/rest/stats/folder",
 	"/rest/svc/deviceid?id=I6KAH76-66SLLLB-5PFXSOA-UFJCDZC-YAOMLEK-CP2GB32-BV5RQST-3PSROAU",