From 514df69afa9425964efbe4f3937eb2336166e36f Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Tue, 10 Mar 2020 17:31:56 -0700 Subject: [PATCH] performance: added wrapper around io.Copy() this pools copy buffers so they can be reused instead of throwing away after each io.Copy() --- Makefile | 1 + cli/command_blob_show.go | 4 ++-- cli/command_show.go | 4 ++-- cli/command_snapshot_verify.go | 4 ++-- cli/show_utils.go | 5 +++-- internal/diff/diff.go | 3 ++- internal/fshasher/fshasher.go | 3 ++- internal/iocopy/copy.go | 26 ++++++++++++++++++++++++++ internal/server/htmlui_fallback.go | 8 ++++---- repo/blob/azure/azure_storage.go | 3 ++- repo/blob/gcs/gcs_storage.go | 4 ++-- repo/compression/compressor_gzip.go | 5 +++-- repo/compression/compressor_pgzip.go | 5 +++-- repo/compression/compressor_s2.go | 5 +++-- repo/compression/compressor_zstd.go | 5 +++-- tests/testenv/cli_test_env.go | 4 +++- 16 files changed, 63 insertions(+), 26 deletions(-) create mode 100644 internal/iocopy/copy.go diff --git a/Makefile b/Makefile index 0b975c989..8d68fb66b 100644 --- a/Makefile +++ b/Makefile @@ -185,6 +185,7 @@ ifneq ($(uname),Windows) | grep -v -e github.com/kopia/kopia/repo \ -e github.com/kopia/kopia/internal/retry \ -e github.com/kopia/kopia/internal/throttle \ + -e github.com/kopia/kopia/internal/iocopy \ -e github.com/kopia/kopia/internal/blobtesting \ -e github.com/kopia/kopia/internal/repotesting \ -e github.com/kopia/kopia/internal/testlogging \ diff --git a/cli/command_blob_show.go b/cli/command_blob_show.go index 3db9e4f50..594ef2d6f 100644 --- a/cli/command_blob_show.go +++ b/cli/command_blob_show.go @@ -3,11 +3,11 @@ import ( "bytes" "context" - "io" "os" "github.com/pkg/errors" + "github.com/kopia/kopia/internal/iocopy" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" ) @@ -24,7 +24,7 @@ func runBlobShow(ctx context.Context, rep *repo.Repository) error { return errors.Wrapf(err, "error getting %v", blobID) } - if _, err := io.Copy(os.Stdout, bytes.NewReader(d)); err != nil { + if _, err := iocopy.Copy(os.Stdout, bytes.NewReader(d)); err != nil { return err } } diff --git a/cli/command_show.go b/cli/command_show.go index 409456fc7..2a6ac8caa 100644 --- a/cli/command_show.go +++ b/cli/command_show.go @@ -2,9 +2,9 @@ import ( "context" - "io" "os" + "github.com/kopia/kopia/internal/iocopy" "github.com/kopia/kopia/repo" ) @@ -26,7 +26,7 @@ func runCatCommand(ctx context.Context, rep *repo.Repository) error { defer r.Close() //nolint:errcheck - _, err = io.Copy(os.Stdout, r) + _, err = iocopy.Copy(os.Stdout, r) return err } diff --git a/cli/command_snapshot_verify.go b/cli/command_snapshot_verify.go index 7944c4745..fb0f3f303 100644 --- a/cli/command_snapshot_verify.go +++ b/cli/command_snapshot_verify.go @@ -3,7 +3,6 @@ import ( "context" "fmt" - "io" "io/ioutil" "math/rand" "sync" @@ -11,6 +10,7 @@ "github.com/pkg/errors" + "github.com/kopia/kopia/internal/iocopy" "github.com/kopia/kopia/internal/parallelwork" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/content" @@ -172,7 +172,7 @@ func (v *verifier) readEntireObject(ctx context.Context, oid object.ID, path str } defer r.Close() //nolint:errcheck - _, err = io.Copy(ioutil.Discard, r) + _, err = iocopy.Copy(ioutil.Discard, r) return err } diff --git a/cli/show_utils.go b/cli/show_utils.go index 924ebfcbb..6dcd570e2 100644 --- a/cli/show_utils.go +++ b/cli/show_utils.go @@ -13,6 +13,7 @@ "github.com/pkg/errors" "gopkg.in/alecthomas/kingpin.v2" + "github.com/kopia/kopia/internal/iocopy" "github.com/kopia/kopia/internal/units" ) @@ -45,7 +46,7 @@ func showContentWithFlags(rd io.Reader, unzip, indentJSON bool) error { var buf1, buf2 bytes.Buffer if indentJSON { - if _, err := io.Copy(&buf1, rd); err != nil { + if _, err := iocopy.Copy(&buf1, rd); err != nil { return err } @@ -56,7 +57,7 @@ func showContentWithFlags(rd io.Reader, unzip, indentJSON bool) error { rd = ioutil.NopCloser(&buf2) } - if _, err := io.Copy(os.Stdout, rd); err != nil { + if _, err := iocopy.Copy(os.Stdout, rd); err != nil { return err } diff --git a/internal/diff/diff.go b/internal/diff/diff.go index cdcc55c1b..99092b61b 100644 --- a/internal/diff/diff.go +++ b/internal/diff/diff.go @@ -13,6 +13,7 @@ "github.com/pkg/errors" "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/internal/iocopy" "github.com/kopia/kopia/repo/logging" "github.com/kopia/kopia/repo/object" ) @@ -280,7 +281,7 @@ func downloadFile(ctx context.Context, f fs.File, fname string) error { } defer dst.Close() //nolint:errcheck - _, err = io.Copy(dst, src) + _, err = iocopy.Copy(dst, src) return err } diff --git a/internal/fshasher/fshasher.go b/internal/fshasher/fshasher.go index efe3b2c14..b8057c95b 100644 --- a/internal/fshasher/fshasher.go +++ b/internal/fshasher/fshasher.go @@ -12,6 +12,7 @@ "golang.org/x/crypto/blake2s" "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/internal/iocopy" "github.com/kopia/kopia/repo/logging" ) @@ -116,7 +117,7 @@ func writeFile(ctx context.Context, w io.Writer, f fs.File) error { } defer r.Close() //nolint:errcheck - if _, err = io.Copy(w, r); err != nil { + if _, err = iocopy.Copy(w, r); err != nil { return err } diff --git a/internal/iocopy/copy.go b/internal/iocopy/copy.go new file mode 100644 index 000000000..9c6755a46 --- /dev/null +++ b/internal/iocopy/copy.go @@ -0,0 +1,26 @@ +// Package iocopy is a wrapper around io.Copy() that recycles shared buffers. +package iocopy + +import ( + "io" + "sync" +) + +const bufSize = 65536 + +var bufferPool = sync.Pool{ + New: func() interface{} { + p := make([]byte, bufSize) + + return &p + }, +} + +// Copy is equivalent to io.Copy() +func Copy(dst io.Writer, src io.Reader) (int64, error) { + bufPtr := bufferPool.Get().(*[]byte) + + defer bufferPool.Put(bufPtr) + + return io.CopyBuffer(dst, src, *bufPtr) +} diff --git a/internal/server/htmlui_fallback.go b/internal/server/htmlui_fallback.go index 8bbe02e11..3a05a2888 100644 --- a/internal/server/htmlui_fallback.go +++ b/internal/server/htmlui_fallback.go @@ -9,13 +9,14 @@ "bytes" "compress/gzip" "fmt" - "net/http" - "io" "io/ioutil" + "net/http" "os" "path/filepath" "strings" "time" + + "github.com/kopia/kopia/internal/iocopy" ) func bindataRead(data []byte, name string) ([]byte, error) { @@ -25,7 +26,7 @@ func bindataRead(data []byte, name string) ([]byte, error) { } var buf bytes.Buffer - _, err = io.Copy(&buf, gz) + _, err = iocopy.Copy(&buf, gz) clErr := gz.Close() if err != nil { @@ -80,7 +81,6 @@ func (fi bindataFileInfo) Sys() interface{} { return nil } - type assetFile struct { *bytes.Reader name string diff --git a/repo/blob/azure/azure_storage.go b/repo/blob/azure/azure_storage.go index 32b0f0a9b..e5514eefe 100644 --- a/repo/blob/azure/azure_storage.go +++ b/repo/blob/azure/azure_storage.go @@ -16,6 +16,7 @@ "gocloud.dev/blob/azureblob" "gocloud.dev/gcerrors" + "github.com/kopia/kopia/internal/iocopy" "github.com/kopia/kopia/internal/retry" "github.com/kopia/kopia/repo/blob" ) @@ -119,7 +120,7 @@ func (az *azStorage) PutBlob(ctx context.Context, b blob.ID, data []byte) error return err } - _, err = io.Copy(writer, throttled) + _, err = iocopy.Copy(writer, throttled) if err != nil { // cancel context before closing the writer causes it to abandon the upload. cancel() diff --git a/repo/blob/gcs/gcs_storage.go b/repo/blob/gcs/gcs_storage.go index 7995292eb..2b2a46f57 100644 --- a/repo/blob/gcs/gcs_storage.go +++ b/repo/blob/gcs/gcs_storage.go @@ -6,7 +6,6 @@ "context" "encoding/json" "fmt" - "io" "io/ioutil" "time" @@ -18,6 +17,7 @@ "google.golang.org/api/iterator" "google.golang.org/api/option" + "github.com/kopia/kopia/internal/iocopy" "github.com/kopia/kopia/internal/retry" "github.com/kopia/kopia/internal/throttle" "github.com/kopia/kopia/repo/blob" @@ -121,7 +121,7 @@ func (gcs *gcsStorage) PutBlob(ctx context.Context, b blob.ID, data []byte) erro } } - _, err := io.Copy(writer, bytes.NewReader(data)) + _, err := iocopy.Copy(writer, bytes.NewReader(data)) if err != nil { // cancel context before closing the writer causes it to abandon the upload. cancel() diff --git a/repo/compression/compressor_gzip.go b/repo/compression/compressor_gzip.go index c9dacd500..8e66143e4 100644 --- a/repo/compression/compressor_gzip.go +++ b/repo/compression/compressor_gzip.go @@ -3,9 +3,10 @@ import ( "bytes" "compress/gzip" - "io" "github.com/pkg/errors" + + "github.com/kopia/kopia/internal/iocopy" ) func init() { @@ -67,7 +68,7 @@ func (c *gzipCompressor) Decompress(b []byte) ([]byte, error) { defer r.Close() //nolint:errcheck var buf bytes.Buffer - if _, err := io.Copy(&buf, r); err != nil { + if _, err := iocopy.Copy(&buf, r); err != nil { return nil, errors.Wrap(err, "decompression error") } diff --git a/repo/compression/compressor_pgzip.go b/repo/compression/compressor_pgzip.go index 391863ecb..ff248e9fd 100644 --- a/repo/compression/compressor_pgzip.go +++ b/repo/compression/compressor_pgzip.go @@ -2,10 +2,11 @@ import ( "bytes" - "io" "github.com/klauspost/pgzip" "github.com/pkg/errors" + + "github.com/kopia/kopia/internal/iocopy" ) func init() { @@ -67,7 +68,7 @@ func (c *pgzipCompressor) Decompress(b []byte) ([]byte, error) { defer r.Close() //nolint:errcheck var buf bytes.Buffer - if _, err := io.Copy(&buf, r); err != nil { + if _, err := iocopy.Copy(&buf, r); err != nil { return nil, errors.Wrap(err, "decompression error") } diff --git a/repo/compression/compressor_s2.go b/repo/compression/compressor_s2.go index ca334d5a1..f0b9791ad 100644 --- a/repo/compression/compressor_s2.go +++ b/repo/compression/compressor_s2.go @@ -2,10 +2,11 @@ import ( "bytes" - "io" "github.com/klauspost/compress/s2" "github.com/pkg/errors" + + "github.com/kopia/kopia/internal/iocopy" ) func init() { @@ -61,7 +62,7 @@ func (c *s2Compressor) Decompress(b []byte) ([]byte, error) { r := s2.NewReader(bytes.NewReader(b[compressionHeaderSize:])) var buf bytes.Buffer - if _, err := io.Copy(&buf, r); err != nil { + if _, err := iocopy.Copy(&buf, r); err != nil { return nil, errors.Wrap(err, "decompression error") } diff --git a/repo/compression/compressor_zstd.go b/repo/compression/compressor_zstd.go index 2e49f847c..302edb55e 100644 --- a/repo/compression/compressor_zstd.go +++ b/repo/compression/compressor_zstd.go @@ -2,10 +2,11 @@ import ( "bytes" - "io" "github.com/klauspost/compress/zstd" "github.com/pkg/errors" + + "github.com/kopia/kopia/internal/iocopy" ) func init() { @@ -68,7 +69,7 @@ func (c *zstdCompressor) Decompress(b []byte) ([]byte, error) { defer r.Close() var buf bytes.Buffer - if _, err := io.Copy(&buf, r); err != nil { + if _, err := iocopy.Copy(&buf, r); err != nil { return nil, errors.Wrap(err, "decompression error") } diff --git a/tests/testenv/cli_test_env.go b/tests/testenv/cli_test_env.go index f0f17b61d..01189e9fe 100644 --- a/tests/testenv/cli_test_env.go +++ b/tests/testenv/cli_test_env.go @@ -19,6 +19,8 @@ "time" "github.com/pkg/errors" + + "github.com/kopia/kopia/internal/iocopy" ) const ( @@ -363,7 +365,7 @@ func createRandomFile(filename string, options DirectoryTreeOptions, counters *D length := rand.Int63n(maxFileSize) - _, err = io.Copy(f, io.LimitReader(rand.New(rand.NewSource(time.Now().UnixNano())), length)) + _, err = iocopy.Copy(f, io.LimitReader(rand.New(rand.NewSource(time.Now().UnixNano())), length)) if err != nil { return errors.Wrap(err, "file create error") }