add unicode filename test, add env switches for long filenames/unicode filenames, update workflow file to include env variables (#1371)

.github/workflows/make.yml

@@ -23,6 +23,10 @@ env:
   NON_TAG_RELEASE_REPO: ${{ secrets.NON_TAG_RELEASE_REPO }}
   # RPM and APT packages GCS bucket/hostname.
   PACKAGES_HOST: ${{ secrets.PACKAGES_HOST }}
+  # set (to any value other than false) to trigger random unicode filenames testing (logs may be difficult to read)
+  ENABLE_UNICODE_FILENAMES: ${{ secrets.ENABLE_UNICODE_FILENAMES }}
+  # set (to any value other than false) to trigger very long filenames testing
+  ENABLE_LONG_FILENAMES: ${{ secrets.ENABLE_LONG_FILENAMES }}
 jobs:
   build:
     strategy:

internal/testutil/testutil.go

@@ -54,6 +54,30 @@ func ShouldReduceTestComplexity() bool {
 	return strings.Contains(runtime.GOARCH, "arm")
 }
 
+// ShouldSkipUnicodeFilenames returns true if:
+// an environmental variable is unset, set to false, test is running on ARM, or if running race detection.
+func ShouldSkipUnicodeFilenames() bool {
+	val, enable := os.LookupEnv("ENABLE_UNICODE_FILENAMES")
+
+	if !enable || isRaceDetector || strings.EqualFold(val, "false") {
+		return true
+	}
+
+	return strings.Contains(runtime.GOARCH, "arm")
+}
+
+// ShouldSkipLongFilenames returns true if:
+// an environmental variable is unset, set to false, test is running on ARM, or if running race detection.
+func ShouldSkipLongFilenames() bool {
+	val, enable := os.LookupEnv("ENABLE_LONG_FILENAMES")
+
+	if !enable || isRaceDetector || strings.EqualFold(val, "false") {
+		return true
+	}
+
+	return strings.Contains(runtime.GOARCH, "arm")
+}
+
 // MyTestMain runs tests and verifies some post-run invariants.
 func MyTestMain(m *testing.M) {
 	v := m.Run()

tests/end_to_end_test/main_test.go

@@ -28,10 +28,13 @@ func oneTimeSetup() error {
 		return errors.Wrap(err, "unable to create data directory")
 	}
 
-	// make sure the base directory is quite long to trigger very long filenames on Windows.
-	if n, targetLen := len(sharedTestDataDirBase), 270; n < targetLen {
-		sharedTestDataDirBase = filepath.Join(sharedTestDataDirBase, strings.Repeat("f", targetLen-n))
-		os.MkdirAll(sharedTestDataDirBase, 0o700)
+	// if enabled, make sure the base directory is quite long to trigger very long filenames on Windows
+	// skipped during race detection, on ARM, and by default to keep logs cleaner
+	if !testutil.ShouldSkipLongFilenames() {
+		if n, targetLen := len(sharedTestDataDirBase), 270; n < targetLen {
+			sharedTestDataDirBase = filepath.Join(sharedTestDataDirBase, strings.Repeat("f", targetLen-n))
+			os.MkdirAll(sharedTestDataDirBase, 0o700)
+		}
 	}
 
 	var counters1, counters2, counters3 testdirtree.DirectoryTreeCounters

tests/testdirtree/testdirtree.go

@@ -11,8 +11,11 @@
 	"path/filepath"
 	"sync/atomic"
 	"testing"
+	"unicode"
+	"unicode/utf8"
 
 	"github.com/pkg/errors"
+	"golang.org/x/text/unicode/norm"
 
 	"github.com/kopia/kopia/internal/clock"
 	"github.com/kopia/kopia/internal/iocopy"
@@ -29,16 +32,73 @@ func intOrDefault(a, b int) int {
 	return b
 }
 
-func randomName(opt DirectoryTreeOptions) string {
-	maxNameLength := intOrDefault(opt.MaxNameLength, 15)
-	minNameLength := intOrDefault(opt.MinNameLength, 3)
-
-	l := rand.Intn(maxNameLength-minNameLength+1) + minNameLength
+func generateHexString(l int) string {
+	// original hex filename generator
 	b := make([]byte, (l+1)/2)
 
 	cryptorand.Read(b)
 
-	return fmt.Sprintf("%v.%v", hex.EncodeToString(b)[:l], atomic.AddInt32(globalRandomNameCounter, 1))
+	return hex.EncodeToString(b)[:l]
+}
+
+func generateUnicodeString(rangeMin, rangeMax, l int) string {
+	// generate a random unicode string within a defined range
+	s := ""
+
+	for i := 0; i < l; {
+		c := rand.Intn(rangeMax-rangeMin+1) + rangeMin
+		r := rune(c)
+		// IsLetter & IsDigit function as a sanity check to prevent writing punctuation/control characters
+		// ValidRune is a sanity check for macOS since APFS can't handle invalid utf-8 and will error out
+		if (unicode.IsLetter(r) || unicode.IsDigit(r)) && utf8.ValidRune(r) {
+			s += string(r)
+			i++
+		}
+	}
+
+	return s
+}
+
+func randomUnicodeName(l int) string {
+	// random language selection for unicode
+	s := ""
+	n := rand.Intn(4)
+
+	switch n {
+	case 1:
+		// cyrillic runs 0x0400 (1024) to 0x052F (1327)
+		s += generateUnicodeString(1024, 1327, l)
+	case 2:
+		// arabic characters run 0x0600 (1536) to 0x06FF (1791)
+		s += generateUnicodeString(1536, 1791, l)
+	case 3:
+		// cjk characters run 0x4E00 (19968) to 0x9FFF (40959)
+		// however the end of the range has compatibility issues due to infrequent usage, so we trim the range a bit (36864)
+		s += generateUnicodeString(19968, 36864, l)
+	default:
+		// latin characters (including extensions A & B) run 0x0020 (32) to 0x024F (591)
+		s += generateUnicodeString(32, 591, l)
+	}
+
+	return norm.NFKD.String(s)
+}
+
+func randomName(opt DirectoryTreeOptions) string {
+	maxNameLength := intOrDefault(opt.MaxNameLength, 15)
+	minNameLength := intOrDefault(opt.MinNameLength, 3)
+
+	s := ""
+	l := rand.Intn(maxNameLength-minNameLength+1) + minNameLength
+
+	// check if we should skip unicode filename testing
+	// skipped during race detection, on ARM, and by default to keep logs cleaner
+	if testutil.ShouldSkipUnicodeFilenames() {
+		s += generateHexString(l)
+	} else {
+		s += randomUnicodeName(l)
+	}
+
+	return fmt.Sprintf("%v.%v", s, atomic.AddInt32(globalRandomNameCounter, 1))
 }
 
 // DirectoryTreeOptions lists options for CreateDirectoryTree.