serve http: fix downloads failing after 5 minutes with --auth-proxy

With --auth-proxy the backend of each user is shut down once it has
been unused for 5 minutes. Only the start of each request counted as a
use, so a download lasting longer than that had its backend shut down
under it and failed.

Each request now holds the backend it uses until it has been served.

This was introduced in v1.75.1 by

f425f8d46 serve: refactor VFS and proxy handling into Provider

cmd/serve/http/http.go

@@ -199,6 +199,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
 
 	router := s.server.Router()
 	router.Use(
+		s.provider.HoldVFS,
 		middleware.Compress(5),
 		middleware.SetHeader("Accept-Ranges", "bytes"),
 		middleware.SetHeader("Server", "rclone/"+fs.Version),

cmd/serve/http/http_test.go

@@ -22,6 +22,7 @@ import (
 	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/fs/rc"
 	libhttp "github.com/rclone/rclone/lib/http"
+	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/vfs/vfscommon"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -559,3 +560,40 @@ func TestNewServerError(t *testing.T) {
 	require.Error(t, err)
 	assert.Nil(t, s)
 }
+
+// TestAuthProxyDownloadOutlivesCache checks a download in progress
+// carries on working when the auth proxy drops its VFS from its cache,
+// as it does when a download takes longer than the cache expiry time.
+func TestAuthProxyDownloadOutlivesCache(t *testing.T) {
+	root := t.TempDir()
+	contents := random.String(32 * 1024 * 1024)
+	require.NoError(t, os.WriteFile(filepath.Join(root, "download.bin"), []byte(contents), 0666))
+
+	prog, err := filepath.Abs("../servetest/proxy_code.go")
+	require.NoError(t, err)
+	// FIXME this is untidy setting a global variable!
+	proxy.Opt.AuthProxy = "go run " + prog + " " + root
+	defer func() {
+		proxy.Opt.AuthProxy = ""
+	}()
+	s, testURL := start(context.Background(), t, nil)
+	defer func() { assert.NoError(t, s.Shutdown()) }()
+
+	req, err := http.NewRequest("GET", testURL+"download.bin", nil)
+	require.NoError(t, err)
+	req.SetBasicAuth(testUser, testPass)
+	resp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	defer func() { _ = resp.Body.Close() }()
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+	start := make([]byte, 1024)
+	_, err = io.ReadFull(resp.Body, start)
+	require.NoError(t, err)
+
+	// Drop everything from the proxy's cache as if it had expired
+	s.provider.Proxy().Shutdown()
+
+	rest, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	assert.True(t, contents == string(start)+string(rest), "download corrupted")
+}

cmd/serve/proxy/proxy.go

@@ -12,6 +12,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net/http"
 	"net/netip"
 	"os/exec"
 	"strings"
@@ -577,6 +578,26 @@ func (p *Provider) Get(ctx context.Context) (*vfs.VFS, error) {
 	return VFS, nil
 }
 
+// HoldVFS is HTTP middleware which holds the VFS for the request until
+// the request has been served.
+//
+// An auth proxy shuts down a VFS when it expires from its cache, which
+// a long upload or download can outlast as only the start of a request
+// counts as a use of the VFS.
+func (p *Provider) HoldVFS(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Requests which don't need auth, eg CORS preflight, have no VFS
+		if VFS, err := p.Get(r.Context()); err == nil {
+			if !VFS.Hold() {
+				http.Error(w, "VFS has been shut down", http.StatusServiceUnavailable)
+				return
+			}
+			defer VFS.Shutdown()
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
 // VFS returns the fixed VFS, or nil if using an auth proxy.
 func (p *Provider) VFS() *vfs.VFS {
 	if p == nil {