performance: added wrapper around io.Copy()

this pools copy buffers so they can be reused instead of throwing away
after each io.Copy()

Makefile

@@ -185,6 +185,7 @@ ifneq ($(uname),Windows)
 	   | grep -v -e github.com/kopia/kopia/repo \
 	             -e github.com/kopia/kopia/internal/retry \
 	             -e github.com/kopia/kopia/internal/throttle \
+	             -e github.com/kopia/kopia/internal/iocopy \
 	             -e github.com/kopia/kopia/internal/blobtesting \
 	             -e github.com/kopia/kopia/internal/repotesting \
 	             -e github.com/kopia/kopia/internal/testlogging \

cli/command_blob_show.go

@@ -3,11 +3,11 @@
 import (
 	"bytes"
 	"context"
-	"io"
 	"os"
 
 	"github.com/pkg/errors"
 
+	"github.com/kopia/kopia/internal/iocopy"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/blob"
 )
@@ -24,7 +24,7 @@ func runBlobShow(ctx context.Context, rep *repo.Repository) error {
 			return errors.Wrapf(err, "error getting %v", blobID)
 		}
 
-		if _, err := io.Copy(os.Stdout, bytes.NewReader(d)); err != nil {
+		if _, err := iocopy.Copy(os.Stdout, bytes.NewReader(d)); err != nil {
 			return err
 		}
 	}

cli/command_show.go

@@ -2,9 +2,9 @@
 
 import (
 	"context"
-	"io"
 	"os"
 
+	"github.com/kopia/kopia/internal/iocopy"
 	"github.com/kopia/kopia/repo"
 )
 
@@ -26,7 +26,7 @@ func runCatCommand(ctx context.Context, rep *repo.Repository) error {
 
 	defer r.Close() //nolint:errcheck
 
-	_, err = io.Copy(os.Stdout, r)
+	_, err = iocopy.Copy(os.Stdout, r)
 
 	return err
 }

cli/command_snapshot_verify.go

@@ -3,7 +3,6 @@
 import (
 	"context"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"math/rand"
 	"sync"
@@ -11,6 +10,7 @@
 
 	"github.com/pkg/errors"
 
+	"github.com/kopia/kopia/internal/iocopy"
 	"github.com/kopia/kopia/internal/parallelwork"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/content"
@@ -172,7 +172,7 @@ func (v *verifier) readEntireObject(ctx context.Context, oid object.ID, path str
 	}
 	defer r.Close() //nolint:errcheck
 
-	_, err = io.Copy(ioutil.Discard, r)
+	_, err = iocopy.Copy(ioutil.Discard, r)
 
 	return err
 }

cli/show_utils.go

@@ -13,6 +13,7 @@
 	"github.com/pkg/errors"
 	"gopkg.in/alecthomas/kingpin.v2"
 
+	"github.com/kopia/kopia/internal/iocopy"
 	"github.com/kopia/kopia/internal/units"
 )
 
@@ -45,7 +46,7 @@ func showContentWithFlags(rd io.Reader, unzip, indentJSON bool) error {
 	var buf1, buf2 bytes.Buffer
 
 	if indentJSON {
-		if _, err := io.Copy(&buf1, rd); err != nil {
+		if _, err := iocopy.Copy(&buf1, rd); err != nil {
 			return err
 		}
 
@@ -56,7 +57,7 @@ func showContentWithFlags(rd io.Reader, unzip, indentJSON bool) error {
 		rd = ioutil.NopCloser(&buf2)
 	}
 
-	if _, err := io.Copy(os.Stdout, rd); err != nil {
+	if _, err := iocopy.Copy(os.Stdout, rd); err != nil {
 		return err
 	}
 

internal/diff/diff.go

@@ -13,6 +13,7 @@
 	"github.com/pkg/errors"
 
 	"github.com/kopia/kopia/fs"
+	"github.com/kopia/kopia/internal/iocopy"
 	"github.com/kopia/kopia/repo/logging"
 	"github.com/kopia/kopia/repo/object"
 )
@@ -280,7 +281,7 @@ func downloadFile(ctx context.Context, f fs.File, fname string) error {
 	}
 	defer dst.Close() //nolint:errcheck
 
-	_, err = io.Copy(dst, src)
+	_, err = iocopy.Copy(dst, src)
 
 	return err
 }

internal/fshasher/fshasher.go

@@ -12,6 +12,7 @@
 	"golang.org/x/crypto/blake2s"
 
 	"github.com/kopia/kopia/fs"
+	"github.com/kopia/kopia/internal/iocopy"
 	"github.com/kopia/kopia/repo/logging"
 )
 
@@ -116,7 +117,7 @@ func writeFile(ctx context.Context, w io.Writer, f fs.File) error {
 	}
 	defer r.Close() //nolint:errcheck
 
-	if _, err = io.Copy(w, r); err != nil {
+	if _, err = iocopy.Copy(w, r); err != nil {
 		return err
 	}
 

internal/iocopy/copy.go

@@ -0,0 +1,26 @@
+// Package iocopy is a wrapper around io.Copy() that recycles shared buffers.
+package iocopy
+
+import (
+	"io"
+	"sync"
+)
+
+const bufSize = 65536
+
+var bufferPool = sync.Pool{
+	New: func() interface{} {
+		p := make([]byte, bufSize)
+
+		return &p
+	},
+}
+
+// Copy is equivalent to io.Copy()
+func Copy(dst io.Writer, src io.Reader) (int64, error) {
+	bufPtr := bufferPool.Get().(*[]byte)
+
+	defer bufferPool.Put(bufPtr)
+
+	return io.CopyBuffer(dst, src, *bufPtr)
+}

internal/server/htmlui_fallback.go

@@ -9,13 +9,14 @@
 	"bytes"
 	"compress/gzip"
 	"fmt"
-	"net/http"
-	"io"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
+
+	"github.com/kopia/kopia/internal/iocopy"
 )
 
 func bindataRead(data []byte, name string) ([]byte, error) {
@@ -25,7 +26,7 @@ func bindataRead(data []byte, name string) ([]byte, error) {
 	}
 
 	var buf bytes.Buffer
-	_, err = io.Copy(&buf, gz)
+	_, err = iocopy.Copy(&buf, gz)
 	clErr := gz.Close()
 
 	if err != nil {
@@ -80,7 +81,6 @@ func (fi bindataFileInfo) Sys() interface{} {
 	return nil
 }
 
-
 type assetFile struct {
 	*bytes.Reader
 	name            string

repo/blob/azure/azure_storage.go

@@ -16,6 +16,7 @@
 	"gocloud.dev/blob/azureblob"
 	"gocloud.dev/gcerrors"
 
+	"github.com/kopia/kopia/internal/iocopy"
 	"github.com/kopia/kopia/internal/retry"
 	"github.com/kopia/kopia/repo/blob"
 )
@@ -119,7 +120,7 @@ func (az *azStorage) PutBlob(ctx context.Context, b blob.ID, data []byte) error
 		return err
 	}
 
-	_, err = io.Copy(writer, throttled)
+	_, err = iocopy.Copy(writer, throttled)
 	if err != nil {
 		// cancel context before closing the writer causes it to abandon the upload.
 		cancel()

repo/blob/gcs/gcs_storage.go

@@ -6,7 +6,6 @@
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"time"
 
@@ -18,6 +17,7 @@
 	"google.golang.org/api/iterator"
 	"google.golang.org/api/option"
 
+	"github.com/kopia/kopia/internal/iocopy"
 	"github.com/kopia/kopia/internal/retry"
 	"github.com/kopia/kopia/internal/throttle"
 	"github.com/kopia/kopia/repo/blob"
@@ -121,7 +121,7 @@ func (gcs *gcsStorage) PutBlob(ctx context.Context, b blob.ID, data []byte) erro
 		}
 	}
 
-	_, err := io.Copy(writer, bytes.NewReader(data))
+	_, err := iocopy.Copy(writer, bytes.NewReader(data))
 	if err != nil {
 		// cancel context before closing the writer causes it to abandon the upload.
 		cancel()

repo/compression/compressor_gzip.go

@@ -3,9 +3,10 @@
 import (
 	"bytes"
 	"compress/gzip"
-	"io"
 
 	"github.com/pkg/errors"
+
+	"github.com/kopia/kopia/internal/iocopy"
 )
 
 func init() {
@@ -67,7 +68,7 @@ func (c *gzipCompressor) Decompress(b []byte) ([]byte, error) {
 	defer r.Close() //nolint:errcheck
 
 	var buf bytes.Buffer
-	if _, err := io.Copy(&buf, r); err != nil {
+	if _, err := iocopy.Copy(&buf, r); err != nil {
 		return nil, errors.Wrap(err, "decompression error")
 	}
 

repo/compression/compressor_pgzip.go

@@ -2,10 +2,11 @@
 
 import (
 	"bytes"
-	"io"
 
 	"github.com/klauspost/pgzip"
 	"github.com/pkg/errors"
+
+	"github.com/kopia/kopia/internal/iocopy"
 )
 
 func init() {
@@ -67,7 +68,7 @@ func (c *pgzipCompressor) Decompress(b []byte) ([]byte, error) {
 	defer r.Close() //nolint:errcheck
 
 	var buf bytes.Buffer
-	if _, err := io.Copy(&buf, r); err != nil {
+	if _, err := iocopy.Copy(&buf, r); err != nil {
 		return nil, errors.Wrap(err, "decompression error")
 	}
 

repo/compression/compressor_s2.go

@@ -2,10 +2,11 @@
 
 import (
 	"bytes"
-	"io"
 
 	"github.com/klauspost/compress/s2"
 	"github.com/pkg/errors"
+
+	"github.com/kopia/kopia/internal/iocopy"
 )
 
 func init() {
@@ -61,7 +62,7 @@ func (c *s2Compressor) Decompress(b []byte) ([]byte, error) {
 	r := s2.NewReader(bytes.NewReader(b[compressionHeaderSize:]))
 
 	var buf bytes.Buffer
-	if _, err := io.Copy(&buf, r); err != nil {
+	if _, err := iocopy.Copy(&buf, r); err != nil {
 		return nil, errors.Wrap(err, "decompression error")
 	}
 

repo/compression/compressor_zstd.go

@@ -2,10 +2,11 @@
 
 import (
 	"bytes"
-	"io"
 
 	"github.com/klauspost/compress/zstd"
 	"github.com/pkg/errors"
+
+	"github.com/kopia/kopia/internal/iocopy"
 )
 
 func init() {
@@ -68,7 +69,7 @@ func (c *zstdCompressor) Decompress(b []byte) ([]byte, error) {
 	defer r.Close()
 
 	var buf bytes.Buffer
-	if _, err := io.Copy(&buf, r); err != nil {
+	if _, err := iocopy.Copy(&buf, r); err != nil {
 		return nil, errors.Wrap(err, "decompression error")
 	}
 

tests/testenv/cli_test_env.go

@@ -19,6 +19,8 @@
 	"time"
 
 	"github.com/pkg/errors"
+
+	"github.com/kopia/kopia/internal/iocopy"
 )
 
 const (
@@ -363,7 +365,7 @@ func createRandomFile(filename string, options DirectoryTreeOptions, counters *D
 
 	length := rand.Int63n(maxFileSize)
 
-	_, err = io.Copy(f, io.LimitReader(rand.New(rand.NewSource(time.Now().UnixNano())), length))
+	_, err = iocopy.Copy(f, io.LimitReader(rand.New(rand.NewSource(time.Now().UnixNano())), length))
 	if err != nil {
 		return errors.Wrap(err, "file create error")
 	}