mirror/rclone
1
0
Fork 0
mirror of https://github.com/rclone/rclone.git synced 2026-03-28 20:32:45 -04:00
Code Issues Packages Projects Releases Wiki Activity

webdav: request only required properties in listAll to improve performance

Browse Source 
This PR optimizes the PROPFIND requests in the webdav backend to only ask for
the specific properties rclone actually needs.

Currently, the generic webdav backend sends an empty XML body during directory
listing (listAll), which causes the server to fall back to allprops by default.
This forces the server to return properties we never use, such as
getcontenttype.

Fetching getcontenttype can be a very expensive operation on the server side.
For instance, in the official golang.org/x/net/webdav library, determining the
content type requires the server to open the file and read the first 500 bytes.

For a directory with 1,300 files in my environment, rclone ls time dropped from
~30s to ~4s (as fast as native ls).

This only applies to the other vendor for backwards compatibility which could be
expanded.
This commit is contained in:
ZRHan
2026-03-17 01:29:45 +08:00
committed by GitHub
parent a3e1312d9d
commit 523a29a4e9
1 changed files with 19 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
backend/webdav/webdav.go
View File

@@ -225,6 +225,7 @@ type Fs struct {
hasOCMD5 bool // set if can use owncloud style checksums for MD5
hasOCSHA1 bool // set if can use owncloud style checksums for SHA1
hasMESHA1 bool // set if can use fastmail style checksums for SHA1
useStandardProps bool // set if should use standard props for PROPFIND
ntlmAuthMu sync.Mutex // mutex to serialize NTLM auth roundtrips
chunksUploadURL string // upload URL for nextcloud chunked
canChunk bool // set if nextcloud and nextcloud_chunk_size is set
@@ -352,7 +353,10 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, depth string)
}
if f.hasOCMD5 || f.hasOCSHA1 {
opts.Body = bytes.NewBuffer(owncloudProps)
} else if f.useStandardProps {
opts.Body = bytes.NewBuffer(standardProps)
}
// Note: According to WebDAV RFC 4918, empty PROPFIND body defaults to allprop
var result api.Multistatus
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
@@ -711,6 +715,7 @@ func (f *Fs) setQuirks(ctx context.Context, vendor string) error {
f.precision = time.Second
f.useOCMtime = true
case "other":
f.useStandardProps = true
default:
fs.Debugf(f, "Unknown vendor %q", vendor)
}
@@ -759,13 +764,23 @@ var owncloudProps = []byte(`<?xml version="1.0"?>
<d:getlastmodified />
<d:getcontentlength />
<d:resourcetype />
<d:getcontenttype />
<oc:checksums />
<oc:permissions />
</d:prop>
</d:propfind>
`)
var standardProps = []byte(`<?xml version="1.0"?>
<d:propfind xmlns:d="DAV:">
<d:prop>
<d:displayname/>
<d:getlastmodified/>
<d:getcontentlength/>
<d:resourcetype/>
</d:prop>
</d:propfind>
`)
// list the objects into the function supplied
//
// If directories is set it only sends directories
@@ -787,7 +802,10 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
}
if f.hasOCMD5 || f.hasOCSHA1 {
opts.Body = bytes.NewBuffer(owncloudProps)
} else if f.useStandardProps {
opts.Body = bytes.NewBuffer(standardProps)
}
// Note: According to WebDAV RFC 4918, empty PROPFIND body defaults to `allprop`
var result api.Multistatus
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {