lib/fs: Properly handle case insensitive systems (fixes #1787, fixes #2739, fixes #5708)

With this change we emulate a case sensitive filesystem on top of
insensitive filesystems. This means we correctly pick up case-only renames
and throw a case conflict error when there would be multiple files differing
only in case.

This safety check has a small performance hit (about 20% more filesystem
operations when scanning for changes). The new advanced folder option
`caseSensitiveFS` can be used to disable the safety checks, retaining the
previous behavior on systems known to be fully case sensitive.

Co-authored-by: Jakob Borg <jakob@kastelo.net>

test/transfer-bench_test.go

@@ -10,42 +10,42 @@ package integration
 
 import (
 	"log"
+	"math/rand"
 	"os"
 	"testing"
 	"time"
+
+	"github.com/syncthing/syncthing/lib/rc"
 )
 
 func TestBenchmarkTransferManyFiles(t *testing.T) {
-	benchmarkTransfer(t, 10000, 15)
+	setupAndBenchmarkTransfer(t, 10000, 15)
 }
 
 func TestBenchmarkTransferLargeFile1G(t *testing.T) {
-	benchmarkTransfer(t, 1, 30)
+	setupAndBenchmarkTransfer(t, 1, 30)
 }
 func TestBenchmarkTransferLargeFile2G(t *testing.T) {
-	benchmarkTransfer(t, 1, 31)
+	setupAndBenchmarkTransfer(t, 1, 31)
 }
 func TestBenchmarkTransferLargeFile4G(t *testing.T) {
-	benchmarkTransfer(t, 1, 32)
+	setupAndBenchmarkTransfer(t, 1, 32)
 }
 func TestBenchmarkTransferLargeFile8G(t *testing.T) {
-	benchmarkTransfer(t, 1, 33)
+	setupAndBenchmarkTransfer(t, 1, 33)
 }
 func TestBenchmarkTransferLargeFile16G(t *testing.T) {
-	benchmarkTransfer(t, 1, 34)
+	setupAndBenchmarkTransfer(t, 1, 34)
 }
 func TestBenchmarkTransferLargeFile32G(t *testing.T) {
-	benchmarkTransfer(t, 1, 35)
+	setupAndBenchmarkTransfer(t, 1, 35)
 }
 
-func benchmarkTransfer(t *testing.T, files, sizeExp int) {
-	log.Println("Cleaning...")
-	err := removeAll("s1", "s2", "h1/index*", "h2/index*")
-	if err != nil {
-		t.Fatal(err)
-	}
+func setupAndBenchmarkTransfer(t *testing.T, files, sizeExp int) {
+	cleanBenchmarkTransfer(t)
 
 	log.Println("Generating files...")
+	var err error
 	if files == 1 {
 		// Special case. Generate one file with the specified size exactly.
 		var fd *os.File
@@ -57,13 +57,39 @@ func benchmarkTransfer(t *testing.T, files, sizeExp int) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		err = generateOneFile(fd, "s1/onefile", 1<<uint(sizeExp))
+		err = generateOneFile(fd, "s1/onefile", 1<<uint(sizeExp), time.Now())
 	} else {
 		err = generateFiles("s1", files, sizeExp, "../LICENSE")
 	}
 	if err != nil {
 		t.Fatal(err)
 	}
+
+	benchmarkTransfer(t)
+}
+
+// TestBenchmarkTransferSameFiles doesn't actually transfer anything, but tests
+// how fast two devicees get in sync if they have the same data locally.
+func TestBenchmarkTransferSameFiles(t *testing.T) {
+	cleanBenchmarkTransfer(t)
+
+	t0 := time.Now()
+	rand.Seed(0)
+	log.Println("Generating files in s1...")
+	if err := generateFilesWithTime("s1", 10000, 10, "../LICENSE", t0); err != nil {
+		t.Fatal(err)
+	}
+
+	rand.Seed(0)
+	log.Println("Generating same files in s2...")
+	if err := generateFilesWithTime("s2", 10000, 10, "../LICENSE", t0); err != nil {
+		t.Fatal(err)
+	}
+
+	benchmarkTransfer(t)
+}
+
+func benchmarkTransfer(t *testing.T) {
 	expected, err := directoryContents("s1")
 	if err != nil {
 		t.Fatal(err)
@@ -86,9 +112,9 @@ func benchmarkTransfer(t *testing.T, files, sizeExp int) {
 	sender.ResumeAll()
 	receiver.ResumeAll()
 
-	var t0, t1 time.Time
+	t0 := time.Now()
+	var t1 time.Time
 	lastEvent := 0
-	oneItemFinished := false
 
 loop:
 	for {
@@ -105,35 +131,22 @@ loop:
 
 			switch ev.Type {
 			case "ItemFinished":
-				oneItemFinished = true
-				continue
-
-			case "StateChanged":
-				data := ev.Data.(map[string]interface{})
-				if data["folder"].(string) != "default" {
-					continue
-				}
-
-				switch data["to"].(string) {
-				case "syncing":
-					t0 = ev.Time
-					continue
-
-				case "idle":
-					if !oneItemFinished {
-						continue
-					}
-					if !t0.IsZero() {
-						t1 = ev.Time
-						break loop
-					}
-				}
+				break loop
 			}
 		}
 
 		time.Sleep(250 * time.Millisecond)
 	}
 
+	processes := []*rc.Process{sender, receiver}
+	for {
+		if rc.InSync("default", processes...) {
+			t1 = time.Now()
+			break
+		}
+		time.Sleep(250 * time.Millisecond)
+	}
+
 	sendProc, err := sender.Stop()
 	if err != nil {
 		t.Fatal(err)
@@ -159,4 +172,14 @@ loop:
 
 	printUsage("Receiver", recvProc, total)
 	printUsage("Sender", sendProc, total)
+
+	cleanBenchmarkTransfer(t)
+}
+
+func cleanBenchmarkTransfer(t *testing.T) {
+	log.Println("Cleaning...")
+	err := removeAll("s1", "s2", "h1/index*", "h2/index*")
+	if err != nil {
+		t.Fatal(err)
+	}
 }

test/util.go

@@ -41,6 +41,10 @@ const (
 )
 
 func generateFiles(dir string, files, maxexp int, srcname string) error {
+	return generateFilesWithTime(dir, files, maxexp, srcname, time.Now())
+}
+
+func generateFilesWithTime(dir string, files, maxexp int, srcname string, t0 time.Time) error {
 	fd, err := os.Open(srcname)
 	if err != nil {
 		return err
@@ -69,7 +73,7 @@ func generateFiles(dir string, files, maxexp int, srcname string) error {
 		}
 		s += rand.Int63n(a)
 
-		if err := generateOneFile(fd, p1, s); err != nil {
+		if err := generateOneFile(fd, p1, s, t0); err != nil {
 			return err
 		}
 	}
@@ -77,7 +81,7 @@ func generateFiles(dir string, files, maxexp int, srcname string) error {
 	return nil
 }
 
-func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
+func generateOneFile(fd io.ReadSeeker, p1 string, s int64, t0 time.Time) error {
 	src := io.LimitReader(&inifiteReader{fd}, int64(s))
 	dst, err := os.Create(p1)
 	if err != nil {
@@ -96,7 +100,7 @@ func generateOneFile(fd io.ReadSeeker, p1 string, s int64) error {
 
 	os.Chmod(p1, os.FileMode(rand.Intn(0777)|0400))
 
-	t := time.Now().Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
+	t := t0.Add(-time.Duration(rand.Intn(30*86400)) * time.Second)
 	err = os.Chtimes(p1, t, t)
 	if err != nil {
 		return err