mirror/kopia
1
0
Fork 0
mirror of https://github.com/kopia/kopia.git synced 2026-05-19 04:04:56 -04:00
Code Issues Packages Projects Releases Wiki Activity

performance: optimized restore performance for webdav/rclone (#1491)

Browse Source 
This improves the performance of partial data reads, such as the ones
during restore by avoiding reading the full blob only to discard most
of it.

The impact on restore time is dramatic:

Restoring 5.6 GB files:132921 dirs:18980 from rclone based on local
directory:

before: >2h
after: 49.45s
This commit is contained in:
Jarek Kowalski
2021-11-06 11:06:04 -07:00
committed by GitHub
parent dcff6c285d
commit aa40391eed
3 changed files with 21 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@@ -45,7 +45,7 @@ require (
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
github.com/stretchr/testify v1.7.0
github.com/studio-b12/gowebdav v0.0.0-20210917133250-a3a86976a1df
github.com/studio-b12/gowebdav v0.0.0-20211106090535-29e74efa701f
github.com/tg123/go-htpasswd v1.2.0
github.com/zalando/go-keyring v0.1.1
github.com/zeebo/blake3 v0.2.1

2
go.sum
View File

@@ -679,6 +679,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/studio-b12/gowebdav v0.0.0-20210917133250-a3a86976a1df h1:C+J/LwTqP8gRPt1MdSzBNZP0OYuDm5wsmDKgwpLjYzo=
github.com/studio-b12/gowebdav v0.0.0-20210917133250-a3a86976a1df/go.mod h1:gCcfDlA1Y7GqOaeEKw5l9dOGx1VLdc/HuQSlQAaZ30s=
github.com/studio-b12/gowebdav v0.0.0-20211106090535-29e74efa701f h1:SLJx0nHhb2ZLlYNMAbrYsjwmVwXx4yRT48lNIxOp7ts=
github.com/studio-b12/gowebdav v0.0.0-20211106090535-29e74efa701f/go.mod h1:gCcfDlA1Y7GqOaeEKw5l9dOGx1VLdc/HuQSlQAaZ30s=
github.com/tg123/go-htpasswd v1.2.0 h1:UKp34m9H467/xklxUxU15wKRru7fwXoTojtxg25ITF0=
github.com/tg123/go-htpasswd v1.2.0/go.mod h1:h7IzlfpvIWnVJhNZ0nQ9HaFxHb7pn5uFJYLlEUJa2sM=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=

29
repo/blob/webdav/webdav_storage.go
View File

@@ -54,25 +54,32 @@ func (d *davStorageImpl) GetBlobFromPath(ctx context.Context, dirPath, path stri
return blob.ErrInvalidRange
}
s, err := d.cli.ReadStream(path)
var (
s io.ReadCloser
err error
)
switch {
case length < 0:
s, err = d.cli.ReadStream(path)
case length == 0:
s, err = d.cli.ReadStreamRange(path, offset, 1)
default:
s, err = d.cli.ReadStreamRange(path, offset, length)
}
if err != nil {
return d.translateError(err)
}
defer s.Close() // nolint:errcheck
if length < 0 {
// nolint:wrapcheck
return iocopy.JustCopy(output, s)
if length == 0 {
return nil
}
// this is horrible, but gowebdav does not support seeking (yet).
if err := iocopy.JustCopy(io.Discard, io.LimitReader(s, offset)); err != nil {
return errors.Wrap(err, "error discarding data from stream")
}
if err := iocopy.JustCopy(output, io.LimitReader(s, length)); err != nil {
return errors.Wrap(err, "error reading stream")
if err := iocopy.JustCopy(output, s); err != nil {
return errors.Wrap(err, "error populating output")
}
// nolint:wrapcheck