feat(general): added tempfile package (#2402)

* feat(general): added tempfile package

* use tempfile.Create() in bigmap

internal/bigmap/bigmap_internal.go

@@ -14,13 +14,12 @@
 	"context"
 	"encoding/binary"
 	"os"
-	"path/filepath"
 	"sync"
 
 	"github.com/edsrzf/mmap-go"
-	"github.com/google/uuid"
 	"github.com/pkg/errors"
 
+	"github.com/kopia/kopia/internal/tempfile"
 	"github.com/kopia/kopia/repo/logging"
 )
 
@@ -29,7 +28,6 @@
 	defaultMemorySegmentSize    = 18 * 1e6   // 18MB enough to store >1M 16-17-byte keys
 	defaultFileSegmentSize      = 1024 << 20 // 1 GiB
 	defaultInitialSizeLogarithm = 20
-	mmapFileMode                = 0o600
 
 	// grow hash table above this percentage utilization, higher values (close to 100) will be very slow,
 	// smaller values will waste memory.
@@ -75,9 +73,6 @@ type internalMap struct {
 
 	// +checklocks:mu
 	cleanups []func()
-
-	// +checklocks:mu
-	tempDir string
 }
 
 // The list of prime numbers close to 2^N from https://primes.utm.edu/lists/2small/0bit.html
@@ -347,43 +342,28 @@ func (m *internalMap) newMemoryMappedSegment(ctx context.Context) (mmap.MMap, er
 
 // +checklocks:m.mu
 func (m *internalMap) maybeCreateMappedFile(ctx context.Context) (*os.File, error) {
-	if m.tempDir == "" {
-		tempDir, err := os.MkdirTemp("", "kopia-map")
-		if err != nil {
-			return nil, errors.Wrap(err, "unable to create temp directory")
-		}
-
-		m.tempDir = tempDir
-	}
-
-	fname := filepath.Join(m.tempDir, uuid.NewString())
-
-	f, err := os.OpenFile(fname, os.O_CREATE|os.O_EXCL|os.O_RDWR, mmapFileMode) //nolint:gosec
+	f, err := tempfile.Create("")
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to create memory-mapped file")
 	}
 
 	if err := f.Truncate(int64(m.opts.FileSegmentSize)); err != nil {
-		closeAndRemoveFile(ctx, f, fname)
+		closeFile(ctx, f)
 
 		return nil, errors.Wrap(err, "unable to truncate memory-mapped file")
 	}
 
 	m.cleanups = append(m.cleanups, func() {
-		closeAndRemoveFile(ctx, f, fname)
+		closeFile(ctx, f)
 	})
 
 	return f, nil
 }
 
-func closeAndRemoveFile(ctx context.Context, f *os.File, fname string) {
+func closeFile(ctx context.Context, f *os.File) {
 	if err := f.Close(); err != nil {
 		log(ctx).Warnf("unable to close segment file: %v", err)
 	}
-
-	if err := os.Remove(fname); err != nil {
-		log(ctx).Warnf("unable to remove segment file: %v", err)
-	}
 }
 
 // +checklocks:m.mu
@@ -417,10 +397,6 @@ func (m *internalMap) Close(ctx context.Context) {
 
 	m.cleanups = nil
 	m.segments = nil
-
-	if m.tempDir != "" {
-		os.RemoveAll(m.tempDir) //nolint:errcheck
-	}
 }
 
 // newInternalMap creates new internalMap.

internal/tempfile/tempfile.go

@@ -0,0 +1,13 @@
+// Package tempfile provides a cross-platform abstraction for creating private
+// read-write temporary files which are automatically deleted when closed.
+package tempfile
+
+import "os"
+
+func tempDirOr(dir string) string {
+	if dir != "" {
+		return dir
+	}
+
+	return os.TempDir()
+}

internal/tempfile/tempfile_linux.go

@@ -0,0 +1,42 @@
+package tempfile
+
+import (
+	"errors"
+	"os"
+	"sync/atomic"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+var unsupportedTmpFile = new(int32) //nolint:gochecknoglobals
+
+// Create creates a temporary file that will be automatically deleted on close.
+func Create(dir string) (*os.File, error) {
+	if atomic.LoadInt32(unsupportedTmpFile) == 1 {
+		// already tried O_TMPFILE, was unsupported, fall back to generic
+		// Unix method.
+		return createUnixFallback(dir)
+	}
+
+	// on reasonably modern Linux (3.11 and above) O_TMPFILE is supported,
+	// which creates invisible, unlinked file in a given directory.
+
+	fd, err := unix.Open(dir, unix.O_RDWR|unix.O_TMPFILE|unix.O_CLOEXEC, permissions)
+	if err == nil {
+		return os.NewFile(uintptr(fd), ""), nil
+	}
+
+	if errors.Is(err, syscall.EISDIR) || errors.Is(err, syscall.EOPNOTSUPP) {
+		// O_TMPFILE is unsupported, fall back and prevent future attempts.
+		atomic.StoreInt32(unsupportedTmpFile, 1)
+
+		return createUnixFallback(dir)
+	}
+
+	return nil, &os.PathError{
+		Op:   "open",
+		Path: dir,
+		Err:  err,
+	}
+}

internal/tempfile/tempfile_test.go

@@ -0,0 +1,38 @@
+package tempfile_test
+
+import (
+	"io"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/kopia/kopia/internal/tempfile"
+)
+
+func TestTempFile(t *testing.T) {
+	td := t.TempDir()
+
+	f, err := tempfile.Create(td)
+	require.NoError(t, err)
+
+	n, err := f.WriteString("hello")
+	require.NoError(t, err)
+	require.Equal(t, 5, n)
+
+	off, err := f.Seek(1, io.SeekStart)
+	require.Equal(t, int64(1), off)
+	require.NoError(t, err)
+
+	buf := make([]byte, 4)
+	n2, err := f.Read(buf)
+	require.NoError(t, err)
+	require.Equal(t, 4, n2)
+	require.Equal(t, []byte("ello"), buf)
+
+	f.Close()
+
+	files, err := os.ReadDir(td)
+	require.NoError(t, err)
+	require.Empty(t, files)
+}

internal/tempfile/tempfile_unix_fallback.go

@@ -0,0 +1,32 @@
+//go:build linux || freebsd || darwin || openbsd
+// +build linux freebsd darwin openbsd
+
+package tempfile
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+)
+
+const permissions = 0o600
+
+// createUnixFallback creates a temporary file that does not need to be removed on close.
+func createUnixFallback(dir string) (*os.File, error) {
+	fullPath := filepath.Join(tempDirOr(dir), uuid.NewString())
+
+	f, err := os.OpenFile(fullPath, os.O_CREATE|os.O_EXCL|os.O_RDWR, permissions) //nolint:gosec
+	if err != nil {
+		return nil, err //nolint:wrapcheck
+	}
+
+	// immediately remove/unlink the file while we keep the handle open.
+	if derr := os.Remove(fullPath); derr != nil {
+		f.Close() //nolint:errcheck
+		return nil, errors.Wrap(derr, "unable to unlink temporary file")
+	}
+
+	return f, nil
+}

internal/tempfile/tempfile_unix_nonlinux.go

@@ -0,0 +1,13 @@
+//go:build freebsd || darwin || openbsd
+// +build freebsd darwin openbsd
+
+package tempfile
+
+import (
+	"os"
+)
+
+// Create creates a temporary file that does not need to be removed on close.
+func Create(dir string) (*os.File, error) {
+	return createUnixFallback(dir)
+}

internal/tempfile/tempfile_windows.go

@@ -0,0 +1,35 @@
+package tempfile
+
+import (
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/google/uuid"
+	"golang.org/x/sys/windows"
+)
+
+// Create creates a temporary file that will be automatically deleted on close.
+func Create(dir string) (*os.File, error) {
+	fullpath := filepath.Join(tempDirOr(dir), uuid.NewString())
+
+	fname, err := syscall.UTF16PtrFromString(fullpath)
+	if err != nil {
+		return nil, err //nolint:wrapcheck
+	}
+
+	// This call creates a file that's automatically deleted on close.
+	h, err := syscall.CreateFile(
+		fname,
+		windows.GENERIC_READ|windows.GENERIC_WRITE,
+		0,
+		nil,
+		syscall.OPEN_ALWAYS,
+		uint32(windows.FILE_FLAG_DELETE_ON_CLOSE),
+		0)
+	if err != nil {
+		return nil, err //nolint:wrapcheck
+	}
+
+	return os.NewFile(uintptr(h), fullpath), nil
+}