test(general): log dir paths and size in robustness tests (#3973)

Log the paths and the sizes for the cache directories in
multi-client robustness jobs.

tests/robustness/multiclient_test/framework/harness.go

@@ -29,7 +29,8 @@
 	metadataCacheLimitMB = 500
 )
 
-var repoPathPrefix = flag.String("repo-path-prefix", "", "Point the robustness tests at this path prefix")
+// RepoPathPrefix is used by robustness tests as a base dir for repository under test.
+var RepoPathPrefix = flag.String("repo-path-prefix", "", "Point the robustness tests at this path prefix")
 
 // NewHarness returns a test harness. It requires a context that contains a client.
 func NewHarness(ctx context.Context) *TestHarness {
@@ -54,13 +55,12 @@ type TestHarness struct {
 }
 
 func (th *TestHarness) init(ctx context.Context) {
-	if *repoPathPrefix == "" {
+	if *RepoPathPrefix == "" {
 		log.Printf("Skipping robustness tests because repo-path-prefix is not set")
 		os.Exit(0)
 	}
-
-	dataRepoPath := path.Join(*repoPathPrefix, dataSubPath)
-	metaRepoPath := path.Join(*repoPathPrefix, metadataSubPath)
+	dataRepoPath := path.Join(*RepoPathPrefix, dataSubPath)
+	metaRepoPath := path.Join(*RepoPathPrefix, metadataSubPath)
 
 	th.dataRepoPath = dataRepoPath
 	th.metaRepoPath = metaRepoPath
@@ -294,3 +294,45 @@ func (th *TestHarness) Cleanup(ctx context.Context) (retErr error) {
 
 	return retErr
 }
+
+// GetDirsToLog collects the directory paths to log.
+func (th *TestHarness) GetDirsToLog(ctx context.Context) []string {
+	if th.snapshotter == nil {
+		return nil
+	}
+
+	var dirList []string
+	dirList = append(dirList,
+		th.dataRepoPath, // repo under test base dir
+		th.metaRepoPath, // metadata repository base dir
+		path.Join(th.fileWriter.DataDirectory(ctx), ".."), // LocalFioDataPathEnvKey
+		th.engine.MetaStore.GetPersistDir(),               // kopia-persistence-root-
+		th.baseDirPath,                                    // engine-data dir
+	)
+
+	cacheDir, _, err := th.snapshotter.GetCacheDirInfo()
+	if err == nil {
+		dirList = append(dirList, cacheDir) // cache dir for repo under test
+	}
+	allCacheDirs := getAllCacheDirs(cacheDir)
+	dirList = append(dirList, allCacheDirs...)
+
+	return dirList
+}
+
+func getAllCacheDirs(dir string) []string {
+	if dir == "" {
+		return nil
+	}
+	var dirs []string
+	// Collect all cache dirs
+	// There are six types of caches, and corresponding dirs.
+	// metadata, contents, indexes,
+	// own-writes, blob-list, server-contents
+	cacheDirSubpaths := []string{"metadata", "contents", "indexes", "own-writes", "blob-list", "server-contents"}
+	for _, s := range cacheDirSubpaths {
+		dirs = append(dirs, path.Join(dir, s))
+	}
+
+	return dirs
+}

tests/robustness/multiclient_test/framework/snapshotter.go

@@ -10,6 +10,7 @@
 	"os"
 	"os/exec"
 	"strconv"
+	"strings"
 	"sync"
 
 	"github.com/kopia/kopia/tests/robustness"
@@ -236,3 +237,16 @@ func (mcs *MultiClientSnapshotter) createOrGetSnapshotter(ctx context.Context) (
 
 	return cs, nil
 }
+
+// GetCacheDirInfo runs cache info command to get cache dir path for
+// the repository.
+func (mcs *MultiClientSnapshotter) GetCacheDirInfo() (stdout, stderr string, err error) {
+	stdout, stderr, err = mcs.server.Run("cache", "info", "--path")
+	if err == nil {
+		// The current output of the cache info command contains a new line
+		// at the end of the cache directory path.
+		stdout = strings.Trim(stdout, "\n")
+	}
+
+	return stdout, stderr, err
+}

tests/robustness/multiclient_test/main_test.go

@@ -12,6 +12,7 @@
 
 	"github.com/kopia/kopia/tests/robustness/engine"
 	"github.com/kopia/kopia/tests/robustness/multiclient_test/framework"
+	"github.com/kopia/kopia/tests/robustness/multiclient_test/storagestats"
 )
 
 // Variables for use in the test functions.
@@ -30,10 +31,24 @@ func TestMain(m *testing.M) {
 
 	eng = th.Engine()
 
+	// Perform setup needed to get storage stats.
+	dirs := th.GetDirsToLog(ctx)
+	log.Printf("Logging storage stats for %v", dirs)
+	err := storagestats.LogStorageStats(ctx, dirs)
+	if err != nil {
+		log.Printf("Error collecting the logs: %s", err.Error())
+	}
+
 	// run the tests
 	result := m.Run()
 
-	err := th.Cleanup(ctx)
+	// Log storage stats after the test run.
+	err = storagestats.LogStorageStats(ctx, dirs)
+	if err != nil {
+		log.Printf("Error collecting the logs: %s", err.Error())
+	}
+
+	err = th.Cleanup(ctx)
 	if err != nil {
 		log.Printf("Error cleaning up the engine: %s\n", err.Error())
 		os.Exit(2)

tests/robustness/multiclient_test/storagestats/storage_stats.go

@@ -0,0 +1,104 @@
+//go:build darwin || (linux && amd64)
+// +build darwin linux,amd64
+
+// Package storagestats contains logging mechanism
+// log disk space consumed by directories created by
+// robustness test framework before and after the test run.
+package storagestats
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"time"
+
+	"github.com/kopia/kopia/tests/robustness/multiclient_test/framework"
+)
+
+const (
+	logFileSubpath = "logs"
+)
+
+var logFilePath string
+
+// DirectorySize represents details about a directory,
+// path, and size.
+type DirectorySize struct {
+	Path string `json:"path"`
+	Size int64  `json:"size"`
+}
+
+// LogStorageStats logs disk space usage of provided dir paths.
+func LogStorageStats(ctx context.Context, dirs []string) error {
+	dd := collectDirectorySizes(dirs)
+
+	// write dir details into a JSON file
+	jsonData, err := json.Marshal(dd)
+	if err != nil {
+		return fmt.Errorf("error marshaling to JSON: %w", err)
+	}
+
+	logFilePath = getLogFilePath()
+	log.Printf("log file path %s", logFilePath)
+	err = os.WriteFile(logFilePath, jsonData, 0o644)
+	if err != nil {
+		return fmt.Errorf("error writing log file: %w", err)
+	}
+
+	return nil
+}
+
+func getSize(dirPath string) (int64, error) {
+	var size int64
+
+	err := filepath.WalkDir(dirPath, func(_ string, d os.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+		// skip
+		if !d.IsDir() {
+			info, err := d.Info()
+			if err != nil {
+				return err
+			}
+			size += info.Size()
+		}
+		return nil
+	})
+
+	return size, err
+}
+
+func getLogFilePath() string {
+	logFileName := "multiclient_kopia_cache_dir_usage_" + time.Now().UTC().Format("20060102_150405") + ".json" //nolint:forbidigo
+	filePath := path.Join(*framework.RepoPathPrefix, logFileSubpath, logFileName)
+
+	return filePath
+}
+
+func collectDirectorySizes(dirs []string) []DirectorySize {
+	dd := make([]DirectorySize, 0, len(dirs))
+
+	for _, dir := range dirs {
+		s, err := getSize(dir)
+		if err != nil {
+			s = -1
+
+			log.Printf("error getting dir size for '%s' %v", dir, err)
+		} else {
+			log.Printf("dir: '%s', size: %d", dir, s)
+		}
+
+		d := DirectorySize{
+			Path: dir,
+			Size: s,
+		}
+		dd = append(dd, d)
+	}
+
+	return dd
+}