From 33c87337501eeed840528a87468330c4c08d7049 Mon Sep 17 00:00:00 2001 From: Z Date: Sat, 9 Oct 2021 14:38:36 -0500 Subject: [PATCH] add unicode filename test, add env switches for long filenames/unicode filenames, update workflow file to include env variables (#1371) --- .github/workflows/make.yml | 4 ++ internal/testutil/testutil.go | 24 ++++++++++ tests/end_to_end_test/main_test.go | 11 +++-- tests/testdirtree/testdirtree.go | 72 +++++++++++++++++++++++++++--- 4 files changed, 101 insertions(+), 10 deletions(-) diff --git a/.github/workflows/make.yml b/.github/workflows/make.yml index 8f90a147e..d5254af7c 100644 --- a/.github/workflows/make.yml +++ b/.github/workflows/make.yml @@ -23,6 +23,10 @@ env: NON_TAG_RELEASE_REPO: ${{ secrets.NON_TAG_RELEASE_REPO }} # RPM and APT packages GCS bucket/hostname. PACKAGES_HOST: ${{ secrets.PACKAGES_HOST }} + # set (to any value other than false) to trigger random unicode filenames testing (logs may be difficult to read) + ENABLE_UNICODE_FILENAMES: ${{ secrets.ENABLE_UNICODE_FILENAMES }} + # set (to any value other than false) to trigger very long filenames testing + ENABLE_LONG_FILENAMES: ${{ secrets.ENABLE_LONG_FILENAMES }} jobs: build: strategy: diff --git a/internal/testutil/testutil.go b/internal/testutil/testutil.go index 509de10c3..48d9b3fa6 100644 --- a/internal/testutil/testutil.go +++ b/internal/testutil/testutil.go @@ -54,6 +54,30 @@ func ShouldReduceTestComplexity() bool { return strings.Contains(runtime.GOARCH, "arm") } +// ShouldSkipUnicodeFilenames returns true if: +// an environmental variable is unset, set to false, test is running on ARM, or if running race detection. +func ShouldSkipUnicodeFilenames() bool { + val, enable := os.LookupEnv("ENABLE_UNICODE_FILENAMES") + + if !enable || isRaceDetector || strings.EqualFold(val, "false") { + return true + } + + return strings.Contains(runtime.GOARCH, "arm") +} + +// ShouldSkipLongFilenames returns true if: +// an environmental variable is unset, set to false, test is running on ARM, or if running race detection. +func ShouldSkipLongFilenames() bool { + val, enable := os.LookupEnv("ENABLE_LONG_FILENAMES") + + if !enable || isRaceDetector || strings.EqualFold(val, "false") { + return true + } + + return strings.Contains(runtime.GOARCH, "arm") +} + // MyTestMain runs tests and verifies some post-run invariants. func MyTestMain(m *testing.M) { v := m.Run() diff --git a/tests/end_to_end_test/main_test.go b/tests/end_to_end_test/main_test.go index 3e620ce51..b50675f34 100644 --- a/tests/end_to_end_test/main_test.go +++ b/tests/end_to_end_test/main_test.go @@ -28,10 +28,13 @@ func oneTimeSetup() error { return errors.Wrap(err, "unable to create data directory") } - // make sure the base directory is quite long to trigger very long filenames on Windows. - if n, targetLen := len(sharedTestDataDirBase), 270; n < targetLen { - sharedTestDataDirBase = filepath.Join(sharedTestDataDirBase, strings.Repeat("f", targetLen-n)) - os.MkdirAll(sharedTestDataDirBase, 0o700) + // if enabled, make sure the base directory is quite long to trigger very long filenames on Windows + // skipped during race detection, on ARM, and by default to keep logs cleaner + if !testutil.ShouldSkipLongFilenames() { + if n, targetLen := len(sharedTestDataDirBase), 270; n < targetLen { + sharedTestDataDirBase = filepath.Join(sharedTestDataDirBase, strings.Repeat("f", targetLen-n)) + os.MkdirAll(sharedTestDataDirBase, 0o700) + } } var counters1, counters2, counters3 testdirtree.DirectoryTreeCounters diff --git a/tests/testdirtree/testdirtree.go b/tests/testdirtree/testdirtree.go index b6a673990..a463c944c 100644 --- a/tests/testdirtree/testdirtree.go +++ b/tests/testdirtree/testdirtree.go @@ -11,8 +11,11 @@ "path/filepath" "sync/atomic" "testing" + "unicode" + "unicode/utf8" "github.com/pkg/errors" + "golang.org/x/text/unicode/norm" "github.com/kopia/kopia/internal/clock" "github.com/kopia/kopia/internal/iocopy" @@ -29,16 +32,73 @@ func intOrDefault(a, b int) int { return b } -func randomName(opt DirectoryTreeOptions) string { - maxNameLength := intOrDefault(opt.MaxNameLength, 15) - minNameLength := intOrDefault(opt.MinNameLength, 3) - - l := rand.Intn(maxNameLength-minNameLength+1) + minNameLength +func generateHexString(l int) string { + // original hex filename generator b := make([]byte, (l+1)/2) cryptorand.Read(b) - return fmt.Sprintf("%v.%v", hex.EncodeToString(b)[:l], atomic.AddInt32(globalRandomNameCounter, 1)) + return hex.EncodeToString(b)[:l] +} + +func generateUnicodeString(rangeMin, rangeMax, l int) string { + // generate a random unicode string within a defined range + s := "" + + for i := 0; i < l; { + c := rand.Intn(rangeMax-rangeMin+1) + rangeMin + r := rune(c) + // IsLetter & IsDigit function as a sanity check to prevent writing punctuation/control characters + // ValidRune is a sanity check for macOS since APFS can't handle invalid utf-8 and will error out + if (unicode.IsLetter(r) || unicode.IsDigit(r)) && utf8.ValidRune(r) { + s += string(r) + i++ + } + } + + return s +} + +func randomUnicodeName(l int) string { + // random language selection for unicode + s := "" + n := rand.Intn(4) + + switch n { + case 1: + // cyrillic runs 0x0400 (1024) to 0x052F (1327) + s += generateUnicodeString(1024, 1327, l) + case 2: + // arabic characters run 0x0600 (1536) to 0x06FF (1791) + s += generateUnicodeString(1536, 1791, l) + case 3: + // cjk characters run 0x4E00 (19968) to 0x9FFF (40959) + // however the end of the range has compatibility issues due to infrequent usage, so we trim the range a bit (36864) + s += generateUnicodeString(19968, 36864, l) + default: + // latin characters (including extensions A & B) run 0x0020 (32) to 0x024F (591) + s += generateUnicodeString(32, 591, l) + } + + return norm.NFKD.String(s) +} + +func randomName(opt DirectoryTreeOptions) string { + maxNameLength := intOrDefault(opt.MaxNameLength, 15) + minNameLength := intOrDefault(opt.MinNameLength, 3) + + s := "" + l := rand.Intn(maxNameLength-minNameLength+1) + minNameLength + + // check if we should skip unicode filename testing + // skipped during race detection, on ARM, and by default to keep logs cleaner + if testutil.ShouldSkipUnicodeFilenames() { + s += generateHexString(l) + } else { + s += randomUnicodeName(l) + } + + return fmt.Sprintf("%v.%v", s, atomic.AddInt32(globalRandomNameCounter, 1)) } // DirectoryTreeOptions lists options for CreateDirectoryTree.