serve docker: fix tests leaving unkillable processes and stale FUSE mounts

Writing to the mount with os.WriteFile made the Go runtime register
the file with its poller so the kernel then polled the file from
epoll_ctl and epoll_wait, sending POLL requests to the FUSE server
running in this same process. A thread waiting inside epoll cannot be
preempted by the runtime, so a garbage collection starting while such
a POLL was outstanding stopped the world for good - the test binary
could not be killed even with SIGKILL and the mount was left behind,
wedging anything that touched it.

Now we write through the mount with a descriptor straight from
open(2), which os.NewFile keeps out of the poller, check that it
really is out of the poller with SetDeadline, and check at the end of
the test that the mountpoint is unmounted.

In this commit we fixed the same problem for mount by running in a
subprocess however changing one write file routine here was much
easier than re-arranging the tests.

4a382c09ec mount: run tests in a subprocess to fix deadlock - #3259

Note that go-fuse (and hence mount2) works around this problem it by
forcing an early POLL it can answer with ENOSYS.

See: https://github.com/golang/go/issues/21014
(cherry picked from commit e855d2ed36)

cmd/serve/docker/docker_test.go

@@ -6,14 +6,17 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
+	"syscall"
 	"testing"
 	"time"
 
@@ -398,6 +401,50 @@ func (a *APIClient) request(path string, in, out any, wantErr bool) {
 	time.Sleep(tempDelay)
 }
 
+// writeMountedFile writes data to a file on a FUSE mount served by this
+// process.
+//
+// It avoids os.WriteFile as files opened with os.OpenFile are handed
+// to the Go runtime poller and the kernel then asks the FUSE server
+// to poll them from epoll_ctl and epoll_wait. The server lives in
+// this process and a thread waiting inside epoll cannot be preempted,
+// so a garbage collection starting while a poll is outstanding stops
+// the world for good. The poll is never answered, the process cannot
+// be killed even with SIGKILL and the mount is left behind.
+// See: https://github.com/golang/go/issues/21014
+//
+// os.NewFile only returns a pollable file for a descriptor which is
+// already in non-blocking mode, so a descriptor straight from open(2)
+// stays out of the poller.
+func writeMountedFile(path string, data []byte, perm uint32) (err error) {
+	fd, err := syscall.Open(path, syscall.O_WRONLY|syscall.O_CREAT|syscall.O_TRUNC, perm)
+	if err != nil {
+		return err
+	}
+	f := os.NewFile(uintptr(fd), path)
+	defer fs.CheckClose(f, &err)
+	// Deadlines work on polled files only
+	if err := f.SetDeadline(time.Time{}); !errors.Is(err, os.ErrNoDeadline) {
+		return fmt.Errorf("%s is being polled by the runtime which risks deadlocking the test: %w", path, err)
+	}
+	_, err = f.Write(data)
+	return err
+}
+
+// checkUnmounted fails the test and tears the mount down if mountpoint
+// is still mounted.
+func checkUnmounted(t *testing.T, mountpoint string) {
+	// CheckMountReady and fusermount are both Linux only
+	if !mountlib.CanCheckMountReady || mountlib.CheckMountReady(mountpoint) != nil {
+		return
+	}
+	assert.Fail(t, fmt.Sprintf("mountpoint %s was left mounted", mountpoint))
+	// Lazy unmount as the mount may still be busy
+	if out, err := exec.Command("fusermount", "-uz", mountpoint).CombinedOutput(); err != nil {
+		t.Logf("Failed to unmount %s: %v: %s", mountpoint, err, out)
+	}
+}
+
 func testMountAPI(t *testing.T, sockAddr string) {
 	// Disable tests under macOS and linux in the CI since they are locking up
 	if runtime.GOOS == "darwin" || runtime.GOOS == "linux" {
@@ -412,6 +459,7 @@ func testMountAPI(t *testing.T, sockAddr string) {
 	testDir, testFs := initialise(ctx, t)
 	err := config.SetCacheDir(testDir)
 	require.NoError(t, err)
+	mount1 := filepath.Join(testDir, "vol1")
 	defer func() {
 		_ = config.SetCacheDir(oldCacheDir)
 		if !t.Failed() {
@@ -419,6 +467,7 @@ func testMountAPI(t *testing.T, sockAddr string) {
 			_ = os.RemoveAll(testDir)
 		}
 	}()
+	defer checkUnmounted(t, mount1)
 
 	// Prepare API client
 	var cli *APIClient
@@ -457,7 +506,6 @@ func testMountAPI(t *testing.T, sockAddr string) {
 	// Run test sequence
 	path1 := filepath.Join(testDir, "path1")
 	require.NoError(t, file.MkdirAll(path1, 0755))
-	mount1 := filepath.Join(testDir, "vol1")
 	res := ""
 
 	cli.request("Activate", "{}", &res, false)
@@ -485,7 +533,7 @@ func testMountAPI(t *testing.T, sockAddr string) {
 	assert.Contains(t, res, "volume is in use")
 
 	text := []byte("banana")
-	err = os.WriteFile(filepath.Join(mount1, "txt"), text, 0644)
+	err = writeMountedFile(filepath.Join(mount1, "txt"), text, 0644)
 	assert.NoError(t, err)
 	time.Sleep(tempDelay)