fs/logger: fix flaky tests by generating test data locally

The TestLogger/TestRepoCompare and TestLogger/TestBeforeVsAfter
testscript scenarios filled src and dst by downloading two old rclone
source archives from GitHub with `rclone copyurl`. Whenever GitHub or
the network hiccuped (eg a 502 Bad Gateway) the downloads failed and
the tests failed with it, making them flaky on CI.

Generate two overlapping trees of files in the test Setup instead.
They cover the same comparison categories the scripts exercise
(matching, differing, src-only and dst-only files) so the tests are
just as meaningful but no longer depend on the network.

fs/logger/logger_test.go

@@ -3,6 +3,8 @@
 package logger_test
 
 import (
+	"fmt"
+	"os"
 	"path/filepath"
 	"testing"
 
@@ -26,9 +28,70 @@ func TestLogger(t *testing.T) {
 	testscript.Run(t, testscript.Params{
 		Dir: "testdata/script",
 		Setup: func(env *testscript.Env) error {
-			env.Setenv("SRC", filepath.Join("$WORK", "src"))
-			env.Setenv("DST", filepath.Join("$WORK", "dst"))
+			src := filepath.Join(env.WorkDir, "src")
+			dst := filepath.Join(env.WorkDir, "dst")
+			// Fill src and dst with two overlapping trees of files so the
+			// scripts have a realistic mix of matching, differing and
+			// missing files to compare. This used to download two old
+			// rclone source archives from GitHub which made the tests fail
+			// whenever the network or GitHub was flaky.
+			if err := makeTestTrees(src, dst); err != nil {
+				return err
+			}
+			env.Setenv("SRC", src)
+			env.Setenv("DST", dst)
 			return nil
 		},
 	})
 }
+
+// makeTestTrees populates src and dst with a deterministic set of files
+// covering every comparison category the scripts exercise:
+//
+//   - identical files present in both (match)
+//   - same-named files with different content (differ)
+//   - files only in src (missing on dst)
+//   - files only in dst (missing on src)
+//
+// The files are spread across the root and a shared subdirectory so the
+// listings need sorting and the directory survives a sync (which deletes the
+// dst-only files but keeps the matching ones).
+func makeTestTrees(src, dst string) error {
+	const perCategory = 25
+	dirs := []string{".", "sub"}
+
+	write := func(root, dir, name, content string) error {
+		path := filepath.Join(root, dir, name)
+		if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
+			return err
+		}
+		return os.WriteFile(path, []byte(content), 0666)
+	}
+
+	for _, dir := range dirs {
+		for i := range perCategory {
+			match := fmt.Sprintf("the same content for match%02d\n", i)
+			if err := write(src, dir, fmt.Sprintf("match%02d.txt", i), match); err != nil {
+				return err
+			}
+			if err := write(dst, dir, fmt.Sprintf("match%02d.txt", i), match); err != nil {
+				return err
+			}
+
+			if err := write(src, dir, fmt.Sprintf("differ%02d.txt", i), fmt.Sprintf("src content %02d\n", i)); err != nil {
+				return err
+			}
+			if err := write(dst, dir, fmt.Sprintf("differ%02d.txt", i), fmt.Sprintf("dst content %02d differs\n", i)); err != nil {
+				return err
+			}
+
+			if err := write(src, dir, fmt.Sprintf("srconly%02d.txt", i), fmt.Sprintf("only in src %02d\n", i)); err != nil {
+				return err
+			}
+			if err := write(dst, dir, fmt.Sprintf("dstonly%02d.txt", i), fmt.Sprintf("only in dst %02d\n", i)); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

fs/logger/testdata/script/TestBeforeVsAfter.txtar

@@ -1,11 +1,5 @@
 # tests whether an md5sum file generated post-sync matches our pre-sync prediction
-
-# Filling src and dst with two different versions of rclone source!:
-exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.49.1.zip $SRC/src.zip
-exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.54.1.zip $DST/dst.zip
-exec unzip $SRC/src.zip -d $SRC
-exec unzip $DST/dst.zip -d $DST
-
+# src and dst are filled by the test Setup with two overlapping trees of files.
 
 # generating sumfiles:
 exec rclone md5sum $SRC --output-file $WORK/src-before.txt

fs/logger/testdata/script/TestRepoCompare.txtar

@@ -1,14 +1,7 @@
 # tests whether rclone check and rclone sync output exactly the same file lists.
-# uses two different old versions of rclone source code for the src and dst 
-# to produce a more realistic test than makefiles
-# (has lots of files with same name but different content)
-
-# Filling src and dst with two different versions of rclone source!:
-exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.49.1.zip $SRC/src.zip
-exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.54.1.zip $DST/dst.zip
-exec unzip $SRC/src.zip -d $SRC
-exec unzip $DST/dst.zip -d $DST
-
+# src and dst are filled by the test Setup with two overlapping trees of files
+# (lots of files with the same name but different content) to produce a more
+# realistic test than makefiles.
 
 # running rclone check for baseline test:
 # error is expected here: