From 64af1396f6e5a576008e5ff53d4bc3cd5d895ce7 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sun, 17 Oct 2021 16:32:54 -0700 Subject: [PATCH] mount: fixed name escaping and added disambiguation (#1403) Fixes #1400 Fixes #1050 --- snapshot/snapshotfs/all_sources.go | 10 + snapshot/snapshotfs/source_directories.go | 88 ++++++++- .../snapshotfs/source_directories_test.go | 173 ++++++++++++++++++ snapshot/snapshotfs/source_snapshots.go | 13 +- 4 files changed, 270 insertions(+), 14 deletions(-) create mode 100644 snapshot/snapshotfs/source_directories_test.go diff --git a/snapshot/snapshotfs/all_sources.go b/snapshot/snapshotfs/all_sources.go index bce4d98fa..46e1e6a56 100644 --- a/snapshot/snapshotfs/all_sources.go +++ b/snapshot/snapshotfs/all_sources.go @@ -69,11 +69,21 @@ func (s *repositoryAllSources) Readdir(ctx context.Context) (fs.Entries, error) users[fmt.Sprintf("%v@%v", src.UserName, src.Host)] = true } + // step 2 - compute safe name for each path + name2safe := map[string]string{} + + for u := range users { + name2safe[u] = safeNameForMount(u) + } + + name2safe = disambiguateSafeNames(name2safe) + var result fs.Entries for u := range users { result = append(result, &sourceDirectories{ rep: s.rep, userHost: u, + name: name2safe[u], }) } diff --git a/snapshot/snapshotfs/source_directories.go b/snapshot/snapshotfs/source_directories.go index ba788df0f..d9b5f168d 100644 --- a/snapshot/snapshotfs/source_directories.go +++ b/snapshot/snapshotfs/source_directories.go @@ -2,7 +2,10 @@ import ( "context" + "fmt" "os" + "sort" + "strings" "time" "github.com/pkg/errors" @@ -15,6 +18,7 @@ type sourceDirectories struct { rep repo.Repository userHost string + name string } func (s *sourceDirectories) IsDir() bool { @@ -22,7 +26,7 @@ func (s *sourceDirectories) IsDir() bool { } func (s *sourceDirectories) Name() string { - return s.userHost + return s.name } func (s *sourceDirectories) Mode() os.FileMode { @@ -59,19 +63,35 @@ func (s *sourceDirectories) Child(ctx context.Context, name string) (fs.Entry, e } func (s *sourceDirectories) Readdir(ctx context.Context) (fs.Entries, error) { - sources, err := snapshot.ListSources(ctx, s.rep) + sources0, err := snapshot.ListSources(ctx, s.rep) if err != nil { return nil, errors.Wrap(err, "unable to list sources") } - var result fs.Entries + // step 1 - filter sources. + var sources []snapshot.SourceInfo - for _, src := range sources { + for _, src := range sources0 { if src.UserName+"@"+src.Host != s.userHost { continue } - result = append(result, &sourceSnapshots{s.rep, src}) + sources = append(sources, src) + } + + // step 2 - compute safe name for each path + name2safe := map[string]string{} + + for _, src := range sources { + name2safe[src.Path] = safeNameForMount(src.Path) + } + + name2safe = disambiguateSafeNames(name2safe) + + var result fs.Entries + + for _, src := range sources { + result = append(result, &sourceSnapshots{s.rep, src, name2safe[src.Path]}) } result.Sort() @@ -79,4 +99,62 @@ func (s *sourceDirectories) Readdir(ctx context.Context) (fs.Entries, error) { return result, nil } +func disambiguateSafeNames(m map[string]string) map[string]string { + safe2original := map[string][]string{} + + for name, safe := range m { + l := strings.ToLower(safe) + + // make sure we disambiguate in the lowercase space, so that both case sensitive and case-insensitive + // filesystems will be covered. + safe2original[l] = append(safe2original[l], name) + } + + result := map[string]string{} + any := false + + for _, originals := range safe2original { + if len(originals) == 1 { + result[originals[0]] = m[originals[0]] + } else { + // more than 1 path map to the same path, append .1, .2, and so on in deterministic order + sort.Strings(originals) + + for i, orig := range originals { + if i > 0 { + result[orig] += fmt.Sprintf("%v (%v)", m[orig], i+1) + } else { + result[orig] += m[orig] + } + } + + any = true + } + } + + if !any { + return result + } + + // we could have just produced some newly ambiguous names, resolve again. + return disambiguateSafeNames(result) +} + +func safeNameForMount(p string) string { + if p == "/" { + return "__root" + } + + // on Windows : is not allowed, c:/ => c_ and c:\ => c_ + p = strings.ReplaceAll(p, ":/", "_") + p = strings.ReplaceAll(p, ":\\", "_") + p = strings.TrimLeft(p, "/") + p = strings.ReplaceAll(p, "/", "_") + p = strings.ReplaceAll(p, "\\", "_") + p = strings.TrimRight(p, "_") + p = strings.TrimRight(p, ":") + + return p +} + var _ fs.Directory = (*sourceDirectories)(nil) diff --git a/snapshot/snapshotfs/source_directories_test.go b/snapshot/snapshotfs/source_directories_test.go new file mode 100644 index 000000000..7686dce71 --- /dev/null +++ b/snapshot/snapshotfs/source_directories_test.go @@ -0,0 +1,173 @@ +package snapshotfs + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/internal/mockfs" + "github.com/kopia/kopia/internal/repotesting" + "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/snapshot" +) + +func TestAllSources(t *testing.T) { + ctx, env := repotesting.NewEnvironment(t, repotesting.FormatNotImportant) + + u := NewUploader(env.RepositoryWriter) + man, err := u.Upload(ctx, mockfs.NewDirectory(), nil, snapshot.SourceInfo{Host: "dummy", UserName: "dummy", Path: "dummy"}) + require.NoError(t, err) + + manifests := []struct { + user, host, path, timestamp string + }{ + {"some-user", "some-host", "c:/some/path", "2020-01-01T12:01:03Z"}, + {"some-user", "some-host", "c:/some/path", "2020-01-01T12:01:04Z"}, + {"some-user", "some-host", "c:/some/path", "2020-01-01T12:01:05Z"}, + // uppercase version of the previous one + {"some-user", "some-host", "c:/some/Path", "2020-01-01T12:01:03Z"}, + // make sure we cleanup names even for usernames and hostnames, even though special characters should not be allowed + {"some/user", "some-host", "c:/some/path", "2020-01-01T13:01:03Z"}, + {"some/user", "some-host", "c:/some/path", "2020-01-01T13:01:04Z"}, + {"some\\user", "some-host", "c:/some/path", "2020-01-01T14:01:03Z"}, + {"some\\user", "some-host", "c:/some/path", "2020-01-01T14:01:04Z"}, + // root + {"another-user", "some-host", "/", "2020-01-01T12:01:03Z"}, + {"another-user", "some-host", "/tmp", "2020-01-01T12:01:03Z"}, + {"another-user", "some-host", "/var", "2020-01-01T12:01:03Z"}, + } + + for _, m := range manifests { + ts, err := time.Parse(time.RFC3339, m.timestamp) + require.NoError(t, err) + + mustWriteSnapshotManifest(ctx, t, env.RepositoryWriter, snapshot.SourceInfo{UserName: m.user, Host: m.host, Path: m.path}, ts, man) + } + + as := AllSourcesEntry(env.RepositoryWriter) + gotNames := iterateAllNames(ctx, t, as, "") + wantNames := []string{ + "another-user@some-host/", + "another-user@some-host/__root/", + "another-user@some-host/__root/20200101-120103/", + "another-user@some-host/tmp/", + "another-user@some-host/tmp/20200101-120103/", + "another-user@some-host/var/", + "another-user@some-host/var/20200101-120103/", + "some-user@some-host/", + "some-user@some-host/c_some_Path/", + "some-user@some-host/c_some_Path/20200101-120103/", + "some-user@some-host/c_some_path (2)/", + "some-user@some-host/c_some_path (2)/20200101-120103/", + "some-user@some-host/c_some_path (2)/20200101-120104/", + "some-user@some-host/c_some_path (2)/20200101-120105/", + "some_user@some-host/", + "some_user@some-host/c_some_path/", + "some_user@some-host/c_some_path/20200101-130103/", + "some_user@some-host/c_some_path/20200101-130104/", + "some_user@some-host (2)/", + "some_user@some-host (2)/c_some_path/", + "some_user@some-host (2)/c_some_path/20200101-140103/", + "some_user@some-host (2)/c_some_path/20200101-140104/", + } + + require.Equal(t, wantNames, gotNames) +} + +func iterateAllNames(ctx context.Context, t *testing.T, dir fs.Directory, prefix string) []string { + t.Helper() + + entries, err := dir.Readdir(ctx) + require.NoError(t, err) + + result := []string{} + + for _, ent := range entries { + if ent.IsDir() { + result = append(result, prefix+ent.Name()+"/") + result = append(result, iterateAllNames(ctx, t, ent.(fs.Directory), prefix+ent.Name()+"/")...) + } else { + result = append(result, prefix+ent.Name()) + } + } + + return result +} + +func mustWriteSnapshotManifest(ctx context.Context, t *testing.T, rep repo.RepositoryWriter, src snapshot.SourceInfo, startTime time.Time, man *snapshot.Manifest) { + t.Helper() + + man.Source = src + man.StartTime = startTime + + _, err := snapshot.SaveSnapshot(ctx, rep, man) + require.NoError(t, err) +} + +func TestSafeNameForMount(t *testing.T) { + cases := map[string]string{ + "/tmp/foo/bar": "tmp_foo_bar", + "/root": "root", + "/root/": "root", + "/": "__root", + "C:": "C", + "C:\\": "C", + "C:\\foo": "C_foo", + "C:\\foo/bar": "C_foo_bar", + "\\\\server\\root": "__server_root", + "\\\\server\\root\\": "__server_root", + "\\\\server\\root\\subdir": "__server_root_subdir", + "\\\\server\\root\\subdir/with/forward/slashes": "__server_root_subdir_with_forward_slashes", + "\\\\server\\root\\subdir/with\\mixed/slashes\\": "__server_root_subdir_with_mixed_slashes", + } + + for input, want := range cases { + assert.Equal(t, want, safeNameForMount(input), input) + } +} + +func TestDisambiguateSafeNames(t *testing.T) { + cases := []struct { + input map[string]string + want map[string]string + }{ + { + input: map[string]string{ + "c:/": "c", + "c:\\": "c", + "c:": "c", + "c": "c", + }, + want: map[string]string{ + "c": "c", + "c:": "c (2)", + "c:/": "c (3)", + "c:\\": "c (4)", + }, + }, + { + input: map[string]string{ + "c:/": "c", + "c:\\": "c", + "c:": "c", + "c": "c", + "c (2)": "c (2)", + }, + want: map[string]string{ + "c": "c", + "c (2)": "c (2)", + "c:": "c (2) (2)", + "c:/": "c (3)", + "c:\\": "c (4)", + }, + }, + } + + for _, tc := range cases { + require.Equal(t, tc.want, disambiguateSafeNames(tc.input)) + } +} diff --git a/snapshot/snapshotfs/source_snapshots.go b/snapshot/snapshotfs/source_snapshots.go index 198d0a87d..ed57e318a 100644 --- a/snapshot/snapshotfs/source_snapshots.go +++ b/snapshot/snapshotfs/source_snapshots.go @@ -4,7 +4,6 @@ "context" "fmt" "os" - "strings" "time" "github.com/pkg/errors" @@ -15,8 +14,9 @@ ) type sourceSnapshots struct { - rep repo.Repository - src snapshot.SourceInfo + rep repo.Repository + src snapshot.SourceInfo + name string } func (s *sourceSnapshots) IsDir() bool { @@ -24,7 +24,7 @@ func (s *sourceSnapshots) IsDir() bool { } func (s *sourceSnapshots) Name() string { - return fmt.Sprintf("%v", safeName(s.src.Path)) + return s.name } func (s *sourceSnapshots) Mode() os.FileMode { @@ -55,11 +55,6 @@ func (s *sourceSnapshots) LocalFilesystemPath() string { return "" } -func safeName(path string) string { - path = strings.TrimLeft(path, "/") - return strings.Replace(path, "/", "_", -1) -} - func (s *sourceSnapshots) Child(ctx context.Context, name string) (fs.Entry, error) { // nolint:wrapcheck return fs.ReadDirAndFindChild(ctx, s, name)