serve webdav: add gzip compression for compressible responses

Enable on-the-fly response compression for WebDAV when the client sends
Accept-Encoding and the response content type is suitable for
compression.

This adds compression for the WebDAV responses that benefit most in
practice, notably PROPFIND XML responses and text file downloads.
I tested this with Cyberduck, which sends
`Accept-Encoding: gzip,deflate` and accepted the compressed responses.

Range requests are explicitly left uncompressed.

Fixes #5777

cmd/serve/webdav/webdav.go

@@ -122,6 +122,13 @@ supported hash on the backend or you can use a named hash such as
 "MD5" or "SHA-1". Use the [hashsum](/commands/rclone_hashsum/) command
 to see the full list.
 
+### Gzip compression
+
+The server will compress certain response bodies (text and XML, including
+WebDAV PROPFIND responses) using gzip when the client advertises gzip
+support via the ` + "`Accept-Encoding: gzip`" + ` request header. This reduces
+bandwidth usage.
+
 ### Access WebDAV on Windows
 
 WebDAV shared folder can be mapped as a drive on Windows, however the default
@@ -236,6 +243,20 @@ type WebDAV struct {
 	etagHashType  hash.Type
 }
 
+func webDAVCompressMiddleware() func(http.Handler) http.Handler {
+	compress := middleware.Compress(5, "text/*", "application/xml")
+	return func(next http.Handler) http.Handler {
+		compressedNext := compress(next)
+		return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+			if r.Header.Get("Range") != "" {
+				next.ServeHTTP(rw, r)
+				return
+			}
+			compressedNext.ServeHTTP(rw, r)
+		})
+	}
+}
+
 // check interface
 var _ webdav.FileSystem = (*WebDAV)(nil)
 
@@ -288,6 +309,7 @@ func newWebDAV(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
 
 	router := w.server.Router()
 	router.Use(
+		webDAVCompressMiddleware(),
 		middleware.SetHeader("Accept-Ranges", "bytes"),
 		middleware.SetHeader("Server", "rclone/"+fs.Version),
 	)

cmd/serve/webdav/webdav_test.go

@@ -8,6 +8,7 @@
 package webdav
 
 import (
+	"compress/gzip"
 	"context"
 	"flag"
 	"io"
@@ -15,7 +16,6 @@ import (
 	"os"
 	"strings"
 	"testing"
-	"time"
 
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/cmd/serve/proxy"
@@ -115,22 +115,6 @@ func TestHTTPFunction(t *testing.T) {
 		assert.NoError(t, w.Shutdown())
 	}()
 	testURL := w.server.URLs()[0]
-	pause := time.Millisecond
-	i := 0
-	for ; i < 10; i++ {
-		resp, err := http.Head(testURL)
-		if err == nil {
-			_ = resp.Body.Close()
-			break
-		}
-		// t.Logf("couldn't connect, sleeping for %v: %v", pause, err)
-		time.Sleep(pause)
-		pause *= 2
-	}
-	if i >= 10 {
-		t.Fatal("couldn't connect to server")
-	}
-
 	HelpTestGET(t, testURL)
 }
 
@@ -265,6 +249,103 @@ func HelpTestGET(t *testing.T, testURL string) {
 	}
 }
 
+// startAuthenticatedServer creates a webdav server with basic auth against
+// the test files directory, starts it, waits for it to be ready, and returns
+// the base URL. It registers cleanup to shut the server down.
+func startAuthenticatedServer(t *testing.T) string {
+	t.Helper()
+
+	f, err := fs.NewFs(context.Background(), "../http/testdata/files")
+	require.NoError(t, err)
+
+	opt := Opt
+	opt.HTTP.ListenAddr = []string{testBindAddress}
+	opt.Template.Path = testTemplate
+	opt.Auth.BasicUser = testUser
+	opt.Auth.BasicPass = testPass
+
+	w, err := newWebDAV(context.Background(), f, &opt, &vfscommon.Opt, &proxy.Opt)
+	require.NoError(t, err)
+	go func() {
+		require.NoError(t, w.Serve())
+	}()
+	t.Cleanup(func() {
+		assert.NoError(t, w.Shutdown())
+	})
+
+	testURL := w.server.URLs()[0]
+	return testURL
+}
+
+func TestCompressedTextFile(t *testing.T) {
+	testURL := startAuthenticatedServer(t)
+
+	req, err := http.NewRequest("GET", testURL+"two.txt", nil)
+	require.NoError(t, err)
+	req.SetBasicAuth(testUser, testPass)
+	req.Header.Set("Accept-Encoding", "gzip")
+
+	resp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	defer func() { _ = resp.Body.Close() }()
+
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.Equal(t, "gzip", resp.Header.Get("Content-Encoding"))
+
+	gr, err := gzip.NewReader(resp.Body)
+	require.NoError(t, err)
+	defer func() { _ = gr.Close() }()
+
+	body, err := io.ReadAll(gr)
+	require.NoError(t, err)
+	assert.Equal(t, "0123456789\n", string(body))
+}
+
+func TestCompressedPROPFIND(t *testing.T) {
+	testURL := startAuthenticatedServer(t)
+
+	req, err := http.NewRequest("PROPFIND", testURL, nil)
+	require.NoError(t, err)
+	req.SetBasicAuth(testUser, testPass)
+	req.Header.Set("Depth", "1")
+	req.Header.Set("Accept-Encoding", "gzip")
+
+	resp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	defer func() { _ = resp.Body.Close() }()
+
+	assert.Equal(t, http.StatusMultiStatus, resp.StatusCode)
+	assert.Equal(t, "gzip", resp.Header.Get("Content-Encoding"))
+
+	gr, err := gzip.NewReader(resp.Body)
+	require.NoError(t, err)
+	defer func() { _ = gr.Close() }()
+
+	body, err := io.ReadAll(gr)
+	require.NoError(t, err)
+	assert.Contains(t, string(body), "multistatus")
+}
+
+func TestRangeRequestNotCompressed(t *testing.T) {
+	testURL := startAuthenticatedServer(t)
+
+	req, err := http.NewRequest("GET", testURL+"two.txt", nil)
+	require.NoError(t, err)
+	req.SetBasicAuth(testUser, testPass)
+	req.Header.Set("Accept-Encoding", "gzip")
+	req.Header.Set("Range", "bytes=2-5")
+
+	resp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	defer func() { _ = resp.Body.Close() }()
+
+	assert.Equal(t, http.StatusPartialContent, resp.StatusCode)
+	assert.Empty(t, resp.Header.Get("Content-Encoding"))
+	body, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	assert.Equal(t, "2345", string(body))
+}
+
 func TestRc(t *testing.T) {
 	servetest.TestRc(t, rc.Params{
 		"type":           "webdav",