mirror of
https://github.com/kopia/kopia.git
synced 2026-01-24 14:28:06 -05:00
* refactor(repository): added fs.DirectoryIterator This significantly reduces number of small allocations while taking snapshots of lots of files, which leads to faster snapshots. ``` $ runbench --kopia-exe ~/go/bin/kopia \ --compare-to-exe ~/go/bin/kopia-baseline --min-duration 30s \ ./snapshot-linux-parallel-4.sh DIFF duration: current:5.1 baseline:5.8 change:-13.0 % DIFF repo_size: current:1081614127.6 baseline:1081615302.8 change:-0.0 % DIFF num_files: current:60.0 baseline:60.0 change:0% DIFF avg_heap_objects: current:4802666.0 baseline:4905741.8 change:-2.1 % DIFF avg_heap_bytes: current:737397275.2 baseline:715263289.6 change:+3.1 % DIFF avg_ram: current:215.0 baseline:211.5 change:+1.6 % DIFF max_ram: current:294.8 baseline:311.4 change:-5.3 % DIFF avg_cpu: current:167.3 baseline:145.3 change:+15.1 % DIFF max_cpu: current:227.2 baseline:251.0 change:-9.5 % ``` * changed `Next()` API * mechanical move of the iterator to its own file * clarified comment * pr feedback * mechanical move of all localfs dependencies on os.FileInfo to a separate file * Update fs/entry.go Co-authored-by: ashmrtn <3891298+ashmrtn@users.noreply.github.com> * Update fs/entry_dir_iterator.go Co-authored-by: Julio Lopez <1953782+julio-lopez@users.noreply.github.com> * doc: clarified valid results from Next() --------- Co-authored-by: ashmrtn <3891298+ashmrtn@users.noreply.github.com> Co-authored-by: Julio Lopez <1953782+julio-lopez@users.noreply.github.com>
233 lines
7.0 KiB
Go
233 lines
7.0 KiB
Go
package diff_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"io"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/kopia/kopia/fs"
|
|
"github.com/kopia/kopia/internal/diff"
|
|
)
|
|
|
|
var _ fs.Entry = (*testFile)(nil)
|
|
|
|
type testFile struct {
|
|
modtime time.Time
|
|
name string
|
|
content string
|
|
}
|
|
|
|
func (f *testFile) IsDir() bool { return false }
|
|
func (f *testFile) LocalFilesystemPath() string { return f.name }
|
|
func (f *testFile) Close() {}
|
|
func (f *testFile) Name() string { return f.name }
|
|
func (f *testFile) Size() int64 { return int64(len(f.content)) }
|
|
func (f *testFile) Mode() os.FileMode { return 0o644 }
|
|
func (f *testFile) ModTime() time.Time { return f.modtime }
|
|
func (f *testFile) Sys() interface{} { return nil }
|
|
func (f *testFile) Owner() fs.OwnerInfo { return fs.OwnerInfo{UserID: 1000, GroupID: 1000} }
|
|
func (f *testFile) Device() fs.DeviceInfo { return fs.DeviceInfo{Dev: 1} }
|
|
func (f *testFile) Open(ctx context.Context) (io.Reader, error) {
|
|
return strings.NewReader(f.content), nil
|
|
}
|
|
|
|
var _ fs.Directory = (*testDirectory)(nil)
|
|
|
|
type testDirectory struct {
|
|
name string
|
|
files []fs.Entry
|
|
modtime time.Time
|
|
}
|
|
|
|
func (d *testDirectory) Iterate(ctx context.Context) (fs.DirectoryIterator, error) {
|
|
return fs.StaticIterator(d.files, nil), nil
|
|
}
|
|
|
|
func (d *testDirectory) SupportsMultipleIterations() bool { return false }
|
|
func (d *testDirectory) IsDir() bool { return true }
|
|
func (d *testDirectory) LocalFilesystemPath() string { return d.name }
|
|
func (d *testDirectory) Close() {}
|
|
func (d *testDirectory) Name() string { return d.name }
|
|
func (d *testDirectory) Size() int64 { return 0 }
|
|
func (d *testDirectory) Mode() os.FileMode { return 0o755 }
|
|
func (d *testDirectory) ModTime() time.Time { return d.modtime }
|
|
func (d *testDirectory) Sys() interface{} { return nil }
|
|
func (d *testDirectory) Owner() fs.OwnerInfo { return fs.OwnerInfo{UserID: 1000, GroupID: 1000} }
|
|
func (d *testDirectory) Device() fs.DeviceInfo { return fs.DeviceInfo{Dev: 1} }
|
|
func (d *testDirectory) Child(ctx context.Context, name string) (fs.Entry, error) {
|
|
for _, f := range d.files {
|
|
if f.Name() == name {
|
|
return f, nil
|
|
}
|
|
}
|
|
|
|
return nil, fs.ErrEntryNotFound
|
|
}
|
|
func (d *testDirectory) Readdir(ctx context.Context) ([]fs.Entry, error) { return d.files, nil }
|
|
|
|
func TestCompareEmptyDirectories(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
|
|
ctx := context.Background()
|
|
|
|
dmodtime := time.Date(2023, time.April, 12, 10, 30, 0, 0, time.UTC)
|
|
dir1 := createTestDirectory("testDir1", dmodtime)
|
|
dir2 := createTestDirectory("testDir2", dmodtime)
|
|
|
|
c, err := diff.NewComparer(&buf)
|
|
require.NoError(t, err)
|
|
|
|
t.Cleanup(func() {
|
|
_ = c.Close()
|
|
})
|
|
|
|
err = c.Compare(ctx, dir1, dir2)
|
|
require.NoError(t, err)
|
|
require.Empty(t, buf.String())
|
|
}
|
|
|
|
func TestCompareIdenticalDirectories(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
|
|
ctx := context.Background()
|
|
|
|
dmodtime := time.Date(2023, time.April, 12, 10, 30, 0, 0, time.UTC)
|
|
fmodtime := time.Date(2023, time.April, 12, 10, 30, 0, 0, time.UTC)
|
|
dir1 := createTestDirectory(
|
|
"testDir1",
|
|
dmodtime,
|
|
&testFile{name: "file1.txt", content: "abcdefghij", modtime: fmodtime},
|
|
&testFile{name: "file2.txt", content: "klmnopqrstuvwxyz", modtime: fmodtime},
|
|
)
|
|
dir2 := createTestDirectory(
|
|
"testDir2",
|
|
dmodtime,
|
|
&testFile{name: "file1.txt", content: "abcdefghij", modtime: fmodtime},
|
|
&testFile{name: "file2.txt", content: "klmnopqrstuvwxyz", modtime: fmodtime},
|
|
)
|
|
|
|
c, err := diff.NewComparer(&buf)
|
|
require.NoError(t, err)
|
|
|
|
t.Cleanup(func() {
|
|
_ = c.Close()
|
|
})
|
|
|
|
err = c.Compare(ctx, dir1, dir2)
|
|
require.NoError(t, err)
|
|
require.Empty(t, buf.String())
|
|
}
|
|
|
|
func TestCompareDifferentDirectories(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
|
|
ctx := context.Background()
|
|
|
|
dmodtime := time.Date(2023, time.April, 12, 10, 30, 0, 0, time.UTC)
|
|
fmodtime := time.Date(2023, time.April, 12, 10, 30, 0, 0, time.UTC)
|
|
dir1 := createTestDirectory(
|
|
"testDir1",
|
|
dmodtime,
|
|
&testFile{name: "file1.txt", content: "abcdefghij", modtime: fmodtime},
|
|
&testFile{name: "file2.txt", content: "klmnopqrstuvwxyz", modtime: fmodtime},
|
|
)
|
|
dir2 := createTestDirectory(
|
|
"testDir2",
|
|
dmodtime,
|
|
&testFile{name: "file3.txt", content: "abcdefghij1", modtime: fmodtime},
|
|
&testFile{name: "file4.txt", content: "klmnopqrstuvwxyz2", modtime: fmodtime},
|
|
)
|
|
|
|
c, err := diff.NewComparer(&buf)
|
|
require.NoError(t, err)
|
|
|
|
t.Cleanup(func() {
|
|
_ = c.Close()
|
|
})
|
|
|
|
expectedOutput := "added file ./file3.txt (11 bytes)\nadded file ./file4.txt (17 bytes)\n" +
|
|
"removed file ./file1.txt (10 bytes)\n" +
|
|
"removed file ./file2.txt (16 bytes)\n"
|
|
|
|
err = c.Compare(ctx, dir1, dir2)
|
|
require.NoError(t, err)
|
|
require.Equal(t, buf.String(), expectedOutput)
|
|
}
|
|
|
|
func TestCompareDifferentDirectories_DirTimeDiff(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
|
|
ctx := context.Background()
|
|
|
|
dmodtime1 := time.Date(2023, time.April, 12, 10, 30, 0, 0, time.UTC)
|
|
dmodtime2 := time.Date(2022, time.April, 12, 10, 30, 0, 0, time.UTC)
|
|
fmodtime := time.Date(2023, time.April, 12, 10, 30, 0, 0, time.UTC)
|
|
dir1 := createTestDirectory(
|
|
"testDir1",
|
|
dmodtime1,
|
|
&testFile{name: "file1.txt", content: "abcdefghij", modtime: fmodtime},
|
|
&testFile{name: "file2.txt", content: "klmnopqrstuvwxyz", modtime: fmodtime},
|
|
)
|
|
dir2 := createTestDirectory(
|
|
"testDir2",
|
|
dmodtime2,
|
|
&testFile{name: "file1.txt", content: "abcdefghij", modtime: fmodtime},
|
|
&testFile{name: "file2.txt", content: "klmnopqrstuvwxyz", modtime: fmodtime},
|
|
)
|
|
|
|
c, err := diff.NewComparer(&buf)
|
|
require.NoError(t, err)
|
|
|
|
t.Cleanup(func() {
|
|
_ = c.Close()
|
|
})
|
|
|
|
expectedOutput := ". modification times differ: 2023-04-12 10:30:00 +0000 UTC 2022-04-12 10:30:00 +0000 UTC\n"
|
|
err = c.Compare(ctx, dir1, dir2)
|
|
require.NoError(t, err)
|
|
require.Equal(t, buf.String(), expectedOutput)
|
|
}
|
|
|
|
func TestCompareDifferentDirectories_FileTimeDiff(t *testing.T) {
|
|
var buf bytes.Buffer
|
|
|
|
ctx := context.Background()
|
|
|
|
fmodtime1 := time.Date(2023, time.April, 12, 10, 30, 0, 0, time.UTC)
|
|
fmodtime2 := time.Date(2022, time.April, 12, 10, 30, 0, 0, time.UTC)
|
|
dmodtime := time.Date(2023, time.April, 12, 10, 30, 0, 0, time.UTC)
|
|
dir1 := createTestDirectory(
|
|
"testDir1",
|
|
dmodtime,
|
|
&testFile{name: "file1.txt", content: "abcdefghij", modtime: fmodtime1},
|
|
)
|
|
dir2 := createTestDirectory(
|
|
"testDir2",
|
|
dmodtime,
|
|
&testFile{name: "file1.txt", content: "abcdefghij", modtime: fmodtime2},
|
|
)
|
|
|
|
c, err := diff.NewComparer(&buf)
|
|
require.NoError(t, err)
|
|
|
|
t.Cleanup(func() {
|
|
_ = c.Close()
|
|
})
|
|
|
|
expectedOutput := "./file1.txt modification times differ: 2023-04-12 10:30:00 +0000 UTC 2022-04-12 10:30:00 +0000 UTC\n"
|
|
|
|
err = c.Compare(ctx, dir1, dir2)
|
|
require.NoError(t, err)
|
|
require.Equal(t, buf.String(), expectedOutput)
|
|
}
|
|
|
|
func createTestDirectory(name string, modtime time.Time, files ...fs.Entry) *testDirectory {
|
|
return &testDirectory{name: name, files: files, modtime: modtime}
|
|
}
|