feat(snapshots): Implement volume shadow copy support on Windows (#3543)

* Implement volume shadow copy support on Windows

* Update go-vss version

* Fix unused variables

* Rename upload_actions*.go files

* Move vss settings to a separate policy section

* Handle existing shadow copy root

* Fix tests

* Fix lint issues

* Add cli policy test

* Add OS snapshot integration test

* Add GitHub Actions VSS test

* Fix "Incorrect function" error for root VSS snapshots

* Rename err to finalErr in createOSSnapshot

* Add OSSnapshotMode test

* Do not modify paths starting with \\?\ on Windows

* Allow warning messages in logfile tests

* Fix ignorefs not wrapping OS snapshot directory

* Retry VSS creation if another op was in progress

---------

Co-authored-by: Jarek Kowalski <jaak@jkowalski.net>

.github/workflows/volume-shadow-copy-test.yml

@@ -0,0 +1,42 @@
+name: Volume Shadow Copy Test
+on:
+  push:
+    branches: [ master ]
+    tags:
+      - v*
+  pull_request:
+    branches: [ master ]
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  vss-test:
+    name: Volume Shadow Copy Test
+    runs-on: windows-latest
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      with:
+        fetch-depth: 0
+    - name: Set up Go
+      uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+      with:
+        go-version-file: 'go.mod'
+        check-latest: true
+      id: go
+    - name: Install gsudo
+      shell: bash
+      run: |
+        choco install -y --no-progress gsudo
+        echo "C:\tools\gsudo\Current" >> $GITHUB_PATH
+    - name: Admin Test
+      run: gsudo make os-snapshot-tests
+    - name: Non-Admin Test
+      run: gsudo -i Medium make os-snapshot-tests
+    - name: Upload Logs
+      uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0
+      with:
+        name: logs
+        path: .logs/**/*.log
+        if-no-files-found: ignore
+      if: ${{ always() }}

Makefile

@@ -213,7 +213,7 @@ download-rclone:
 	go run ./tools/gettool --tool rclone:$(RCLONE_VERSION) --output-dir dist/kopia_linux_arm_6/ --goos=linux --goarch=arm
 
 
-ci-tests: vet test 
+ci-tests: vet test
 
 ci-integration-tests:
 	$(MAKE) robustness-tool-tests socket-activation-tests
@@ -342,6 +342,11 @@ stress-test: $(gotestsum)
 	$(GO_TEST) -count=$(REPEAT_TEST) -timeout 3600s github.com/kopia/kopia/tests/stress_test
 	$(GO_TEST) -count=$(REPEAT_TEST) -timeout 3600s github.com/kopia/kopia/tests/repository_stress_test
 
+os-snapshot-tests: export KOPIA_EXE ?= $(KOPIA_INTEGRATION_EXE)
+os-snapshot-tests: GOTESTSUM_FORMAT=testname
+os-snapshot-tests: build-integration-test-binary $(gotestsum)
+	$(GO_TEST) -count=$(REPEAT_TEST) github.com/kopia/kopia/tests/os_snapshot_test $(TEST_FLAGS)
+
 layering-test:
 ifneq ($(GOOS),windows)
 	# verify that code under repo/ can only import code also under repo/ + some
@@ -484,6 +489,6 @@ perf-benchmark-test-all:
 	$(MAKE) perf-benchmark-test PERF_BENCHMARK_VERSION=0.7.0~rc1
 
 perf-benchmark-results:
-	gcloud compute scp $(PERF_BENCHMARK_INSTANCE):psrecord-* tests/perf_benchmark --zone=$(PERF_BENCHMARK_INSTANCE_ZONE) 
+	gcloud compute scp $(PERF_BENCHMARK_INSTANCE):psrecord-* tests/perf_benchmark --zone=$(PERF_BENCHMARK_INSTANCE_ZONE)
 	gcloud compute scp $(PERF_BENCHMARK_INSTANCE):repo-size-* tests/perf_benchmark --zone=$(PERF_BENCHMARK_INSTANCE_ZONE)
 	(cd tests/perf_benchmark && go run process_results.go)

cli/command_policy_set.go

@@ -24,6 +24,7 @@ type commandPolicySet struct {
 	policyLoggingFlags
 	policyRetentionFlags
 	policySchedulingFlags
+	policyOSSnapshotFlags
 	policyUploadFlags
 }
 
@@ -39,6 +40,7 @@ func (c *commandPolicySet) setup(svc appServices, parent commandParent) {
 	c.policyLoggingFlags.setup(cmd)
 	c.policyRetentionFlags.setup(cmd)
 	c.policySchedulingFlags.setup(cmd)
+	c.policyOSSnapshotFlags.setup(cmd)
 	c.policyUploadFlags.setup(cmd)
 
 	cmd.Action(svc.repositoryWriterAction(c.run))
@@ -112,6 +114,10 @@ func (c *commandPolicySet) setPolicyFromFlags(ctx context.Context, p *policy.Pol
 		return errors.Wrap(err, "actions policy")
 	}
 
+	if err := c.setOSSnapshotPolicyFromFlags(ctx, &p.OSSnapshotPolicy, changeCount); err != nil {
+		return errors.Wrap(err, "OS snapshot policy")
+	}
+
 	if err := c.setLoggingPolicyFromFlags(ctx, &p.LoggingPolicy, changeCount); err != nil {
 		return errors.Wrap(err, "actions policy")
 	}

cli/command_policy_set_os_snapshot.go

@@ -0,0 +1,64 @@
+package cli
+
+import (
+	"context"
+
+	"github.com/alecthomas/kingpin/v2"
+	"github.com/pkg/errors"
+
+	"github.com/kopia/kopia/snapshot/policy"
+)
+
+type policyOSSnapshotFlags struct {
+	policyEnableVolumeShadowCopy string
+}
+
+func (c *policyOSSnapshotFlags) setup(cmd *kingpin.CmdClause) {
+	osSnapshotMode := []string{policy.OSSnapshotNeverString, policy.OSSnapshotAlwaysString, policy.OSSnapshotWhenAvailableString, inheritPolicyString}
+
+	cmd.Flag("enable-volume-shadow-copy", "Enable Volume Shadow Copy snapshots ('never', 'always', 'when-available', 'inherit')").PlaceHolder("MODE").EnumVar(&c.policyEnableVolumeShadowCopy, osSnapshotMode...)
+}
+
+func (c *policyOSSnapshotFlags) setOSSnapshotPolicyFromFlags(ctx context.Context, fp *policy.OSSnapshotPolicy, changeCount *int) error {
+	if err := applyPolicyOSSnapshotMode(ctx, "enable volume shadow copy", &fp.VolumeShadowCopy.Enable, c.policyEnableVolumeShadowCopy, changeCount); err != nil {
+		return errors.Wrap(err, "enable volume shadow copy")
+	}
+
+	return nil
+}
+
+func applyPolicyOSSnapshotMode(ctx context.Context, desc string, val **policy.OSSnapshotMode, str string, changeCount *int) error {
+	if str == "" {
+		// not changed
+		return nil
+	}
+
+	var mode policy.OSSnapshotMode
+
+	switch str {
+	case inheritPolicyString, defaultPolicyString:
+		*changeCount++
+
+		log(ctx).Infof(" - resetting %q to a default value inherited from parent.", desc)
+
+		*val = nil
+
+		return nil
+	case policy.OSSnapshotNeverString:
+		mode = policy.OSSnapshotNever
+	case policy.OSSnapshotAlwaysString:
+		mode = policy.OSSnapshotAlways
+	case policy.OSSnapshotWhenAvailableString:
+		mode = policy.OSSnapshotWhenAvailable
+	default:
+		return errors.Errorf("invalid %q mode %q", desc, str)
+	}
+
+	*changeCount++
+
+	log(ctx).Infof(" - setting %q to %v.", desc, mode)
+
+	*val = &mode
+
+	return nil
+}

cli/command_policy_set_os_snapshot_test.go

@@ -0,0 +1,42 @@
+package cli_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/kopia/kopia/internal/testutil"
+	"github.com/kopia/kopia/tests/testenv"
+)
+
+func TestSetOSSnapshotPolicy(t *testing.T) {
+	e := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, testenv.NewInProcRunner(t))
+	defer e.RunAndExpectSuccess(t, "repo", "disconnect")
+
+	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
+
+	lines := e.RunAndExpectSuccess(t, "policy", "show", "--global")
+	lines = compressSpaces(lines)
+	require.Contains(t, lines, " Volume Shadow Copy: when-available (defined for this target)")
+
+	// make some directory we'll be setting policy on
+	td := testutil.TempDirectory(t)
+
+	lines = e.RunAndExpectSuccess(t, "policy", "show", td)
+	lines = compressSpaces(lines)
+	require.Contains(t, lines, " Volume Shadow Copy: when-available inherited from (global)")
+
+	e.RunAndExpectSuccess(t, "policy", "set", "--global", "--enable-volume-shadow-copy=always")
+
+	lines = e.RunAndExpectSuccess(t, "policy", "show", td)
+	lines = compressSpaces(lines)
+
+	require.Contains(t, lines, " Volume Shadow Copy: always inherited from (global)")
+
+	e.RunAndExpectSuccess(t, "policy", "set", "--enable-volume-shadow-copy=never", td)
+
+	lines = e.RunAndExpectSuccess(t, "policy", "show", td)
+	lines = compressSpaces(lines)
+
+	require.Contains(t, lines, " Volume Shadow Copy: never (defined for this target)")
+}

cli/command_policy_show.go

@@ -128,6 +128,8 @@ func printPolicy(out *textOutput, p *policy.Policy, def *policy.Definition) {
 	rows = append(rows, policyTableRow{})
 	rows = appendActionsPolicyRows(rows, p, def)
 	rows = append(rows, policyTableRow{})
+	rows = appendOSSnapshotPolicyRows(rows, p, def)
+	rows = append(rows, policyTableRow{})
 	rows = appendLoggingPolicyRows(rows, p, def)
 
 	out.printStdout("Policy for %v:\n\n%v\n", p.Target(), alignedPolicyTableRows(rows))
@@ -449,6 +451,15 @@ func appendActionCommandRows(rows []policyTableRow, h *policy.ActionCommand) []p
 	return rows
 }
 
+func appendOSSnapshotPolicyRows(rows []policyTableRow, p *policy.Policy, def *policy.Definition) []policyTableRow {
+	rows = append(rows,
+		policyTableRow{"OS-level snapshot support:", "", ""},
+		policyTableRow{"  Volume Shadow Copy:", p.OSSnapshotPolicy.VolumeShadowCopy.Enable.String(), definitionPointToString(p.Target(), def.OSSnapshotPolicy.VolumeShadowCopy.Enable)},
+	)
+
+	return rows
+}
+
 func valueOrNotSet(p *policy.OptionalInt) string {
 	if p == nil {
 		return "-"

fs/localfs/local_fs_os.go

@@ -5,7 +5,9 @@
 	"io"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
+	"syscall"
 
 	"github.com/pkg/errors"
 
@@ -109,7 +111,20 @@ func NewEntry(path string) (fs.Entry, error) {
 
 	fi, err := os.Lstat(path)
 	if err != nil {
-		return nil, errors.Wrap(err, "unable to determine entry type")
+		// Paths such as `\\?\GLOBALROOT\Device\HarddiskVolumeShadowCopy01`
+		// cause os.Lstat to fail with "Incorrect function" error unless they
+		// end with a separator. Retry the operation with the separator added.
+		var e syscall.Errno
+		//nolint:goconst
+		if runtime.GOOS == "windows" &&
+			!strings.HasSuffix(path, string(filepath.Separator)) &&
+			errors.As(err, &e) && e == 1 {
+			fi, err = os.Lstat(path + string(filepath.Separator))
+		}
+
+		if err != nil {
+			return nil, errors.Wrap(err, "unable to determine entry type")
+		}
 	}
 
 	if path == "/" {

go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/kylelemons/godebug v1.1.0
 	github.com/mattn/go-colorable v0.1.13
 	github.com/minio/minio-go/v7 v7.0.66
+	github.com/mxk/go-vss v1.2.0
 	github.com/natefinch/atomic v1.0.1
 	github.com/pierrec/lz4 v2.6.1+incompatible
 	github.com/pkg/errors v0.9.1
@@ -93,6 +94,7 @@ require (
 	github.com/frankban/quicktest v1.13.1 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/gobwas/httphead v0.1.0 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/gobwas/ws v1.3.0 // indirect

go.sum

@@ -108,6 +108,8 @@ github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
 github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
+github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
 github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
 github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
 github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
@@ -237,6 +239,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/mxk/go-vss v1.2.0 h1:JpdOPc/P6B3XyRoddn0iMiG/ADBi3AuEsv8RlTb+JeE=
+github.com/mxk/go-vss v1.2.0/go.mod h1:ZQ4yFxCG54vqPnCd+p2IxAe5jwZdz56wSjbwzBXiFd8=
 github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A=
 github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM=
 github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=

internal/atomicfile/atomicfile.go

@@ -21,26 +21,19 @@
 // Because long file names have certain limitations:
 // - we must replace forward slashes with backslashes.
 // - dummy path element (\.\) must be removed.
+//
+// Relative paths are always limited to a total of MAX_PATH characters:
+// https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation
 func MaybePrefixLongFilenameOnWindows(fname string) string {
-	if runtime.GOOS != "windows" {
+	if runtime.GOOS != "windows" || len(fname) < maxPathLength ||
+		fname[:4] == `\\?\` || !ospath.IsAbs(fname) {
 		return fname
 	}
 
-	if len(fname) < maxPathLength {
-		return fname
-	}
-
-	fname = strings.TrimPrefix(fname, "\\\\?\\")
-
-	if !ospath.IsAbs(fname) {
-		// only convert absolute paths
-		return fname
-	}
-
-	fixed := strings.ReplaceAll(fname, "/", "\\")
+	fixed := strings.ReplaceAll(fname, "/", `\`)
 
 	for {
-		fixed2 := strings.ReplaceAll(fixed, "\\.\\", "\\")
+		fixed2 := strings.ReplaceAll(fixed, `\.\`, `\`)
 		if fixed2 == fixed {
 			break
 		}
@@ -48,7 +41,7 @@ func MaybePrefixLongFilenameOnWindows(fname string) string {
 		fixed = fixed2
 	}
 
-	return "\\\\?\\" + fixed
+	return `\\?\` + fixed
 }
 
 // Write is a wrapper around atomic.WriteFile that handles long file names on Windows.

internal/logfile/logfile_test.go

@@ -20,9 +20,9 @@
 )
 
 var (
-	cliLogFormat              = regexp.MustCompile(`^\d{4}-\d\d\-\d\dT\d\d:\d\d:\d\d\.\d{6}Z (DEBUG|INFO) [a-z/]+ .*$`)
+	cliLogFormat              = regexp.MustCompile(`^\d{4}-\d\d\-\d\dT\d\d:\d\d:\d\d\.\d{6}Z (DEBUG|INFO|WARN) [a-z/]+ .*$`)
 	contentLogFormat          = regexp.MustCompile(`^\d{4}-\d\d\-\d\dT\d\d:\d\d:\d\d\.\d{6}Z .*$`)
-	cliLogFormatLocalTimezone = regexp.MustCompile(`^\d{4}-\d\d\-\d\dT\d\d:\d\d:\d\d\.\d{6}[^Z][^ ]+ (DEBUG|INFO) [a-z/]+ .*$`)
+	cliLogFormatLocalTimezone = regexp.MustCompile(`^\d{4}-\d\d\-\d\dT\d\d:\d\d:\d\d\.\d{6}[^Z][^ ]+ (DEBUG|INFO|WARN) [a-z/]+ .*$`)
 )
 
 func TestLoggingFlags(t *testing.T) {

snapshot/policy/os_snapshot_policy.go

@@ -0,0 +1,86 @@
+package policy
+
+import "github.com/kopia/kopia/snapshot"
+
+// OSSnapshotPolicy describes settings for OS-level snapshots.
+type OSSnapshotPolicy struct {
+	VolumeShadowCopy VolumeShadowCopyPolicy `json:"volumeShadowCopy,omitempty"`
+}
+
+// OSSnapshotPolicyDefinition specifies which policy definition provided the value of a particular field.
+type OSSnapshotPolicyDefinition struct {
+	VolumeShadowCopy VolumeShadowCopyPolicyDefinition `json:"volumeShadowCopy,omitempty"`
+}
+
+// Merge applies default values from the provided policy.
+func (p *OSSnapshotPolicy) Merge(src OSSnapshotPolicy, def *OSSnapshotPolicyDefinition, si snapshot.SourceInfo) {
+	p.VolumeShadowCopy.Merge(src.VolumeShadowCopy, &def.VolumeShadowCopy, si)
+}
+
+// VolumeShadowCopyPolicy describes settings for Windows Volume Shadow Copy
+// snapshots.
+type VolumeShadowCopyPolicy struct {
+	Enable *OSSnapshotMode `json:"enable,omitempty"`
+}
+
+// VolumeShadowCopyPolicyDefinition specifies which policy definition provided
+// the value of a particular field.
+type VolumeShadowCopyPolicyDefinition struct {
+	Enable snapshot.SourceInfo `json:"enable,omitempty"`
+}
+
+// Merge applies default values from the provided policy.
+func (p *VolumeShadowCopyPolicy) Merge(src VolumeShadowCopyPolicy, def *VolumeShadowCopyPolicyDefinition, si snapshot.SourceInfo) {
+	mergeOSSnapshotMode(&p.Enable, src.Enable, &def.Enable, si)
+}
+
+// OSSnapshotMode specifies whether OS-level snapshots are used for file systems
+// that support them.
+type OSSnapshotMode byte
+
+// OS-level snapshot modes.
+const (
+	OSSnapshotNever         OSSnapshotMode = iota // Disable OS-level snapshots
+	OSSnapshotAlways                              // Fail if an OS-level snapshot cannot be created
+	OSSnapshotWhenAvailable                       // Fall back to regular file access on error
+)
+
+// OS-level snapshot mode strings.
+const (
+	OSSnapshotNeverString         = "never"
+	OSSnapshotAlwaysString        = "always"
+	OSSnapshotWhenAvailableString = "when-available"
+)
+
+// NewOSSnapshotMode provides an OptionalBool pointer.
+func NewOSSnapshotMode(m OSSnapshotMode) *OSSnapshotMode {
+	return &m
+}
+
+// OrDefault returns the OS snapshot mode or the provided default.
+func (m *OSSnapshotMode) OrDefault(def OSSnapshotMode) OSSnapshotMode {
+	if m == nil {
+		return def
+	}
+
+	return *m
+}
+
+func (m OSSnapshotMode) String() string {
+	switch m {
+	case OSSnapshotAlways:
+		return OSSnapshotAlwaysString
+	case OSSnapshotWhenAvailable:
+		return OSSnapshotWhenAvailableString
+	default:
+		return OSSnapshotNeverString
+	}
+}
+
+func mergeOSSnapshotMode(target **OSSnapshotMode, src *OSSnapshotMode, def *snapshot.SourceInfo, si snapshot.SourceInfo) {
+	if *target == nil && src != nil {
+		v := *src
+		*target = &v
+		*def = si
+	}
+}

snapshot/policy/os_snapshot_policy_test.go

@@ -0,0 +1,25 @@
+package policy
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestOSSnapshotMode(t *testing.T) {
+	assert.Equal(t, OSSnapshotNever, (*OSSnapshotMode)(nil).OrDefault(OSSnapshotNever))
+	assert.Equal(t, OSSnapshotAlways, NewOSSnapshotMode(OSSnapshotAlways).OrDefault(OSSnapshotNever))
+
+	cases := []struct {
+		m OSSnapshotMode
+		s string
+	}{
+		{OSSnapshotNever, "never"},
+		{OSSnapshotAlways, "always"},
+		{OSSnapshotWhenAvailable, "when-available"},
+	}
+
+	for _, tc := range cases {
+		assert.Equal(t, tc.s, tc.m.String())
+	}
+}

snapshot/policy/policy.go

@@ -28,6 +28,7 @@ type Policy struct {
 	SchedulingPolicy    SchedulingPolicy    `json:"scheduling,omitempty"`
 	CompressionPolicy   CompressionPolicy   `json:"compression,omitempty"`
 	Actions             ActionsPolicy       `json:"actions,omitempty"`
+	OSSnapshotPolicy    OSSnapshotPolicy    `json:"osSnapshots,omitempty"`
 	LoggingPolicy       LoggingPolicy       `json:"logging,omitempty"`
 	UploadPolicy        UploadPolicy        `json:"upload,omitempty"`
 	NoParent            bool                `json:"noParent,omitempty"`
@@ -42,6 +43,7 @@ type Definition struct {
 	SchedulingPolicy    SchedulingPolicyDefinition    `json:"scheduling,omitempty"`
 	CompressionPolicy   CompressionPolicyDefinition   `json:"compression,omitempty"`
 	Actions             ActionsPolicyDefinition       `json:"actions,omitempty"`
+	OSSnapshotPolicy    OSSnapshotPolicyDefinition    `json:"osSnapshots,omitempty"`
 	LoggingPolicy       LoggingPolicyDefinition       `json:"logging,omitempty"`
 	UploadPolicy        UploadPolicyDefinition        `json:"upload,omitempty"`
 }

snapshot/policy/policy_merge.go

@@ -25,6 +25,7 @@ func MergePolicies(policies []*Policy, si snapshot.SourceInfo) (*Policy, *Defini
 		merged.UploadPolicy.Merge(p.UploadPolicy, &def.UploadPolicy, p.Target())
 		merged.CompressionPolicy.Merge(p.CompressionPolicy, &def.CompressionPolicy, p.Target())
 		merged.Actions.Merge(p.Actions, &def.Actions, p.Target())
+		merged.OSSnapshotPolicy.Merge(p.OSSnapshotPolicy, &def.OSSnapshotPolicy, p.Target())
 		merged.LoggingPolicy.Merge(p.LoggingPolicy, &def.LoggingPolicy, p.Target())
 
 		if p.NoParent {
@@ -40,6 +41,7 @@ func MergePolicies(policies []*Policy, si snapshot.SourceInfo) (*Policy, *Defini
 	merged.UploadPolicy.Merge(defaultUploadPolicy, &def.UploadPolicy, GlobalPolicySourceInfo)
 	merged.CompressionPolicy.Merge(defaultCompressionPolicy, &def.CompressionPolicy, GlobalPolicySourceInfo)
 	merged.Actions.Merge(defaultActionsPolicy, &def.Actions, GlobalPolicySourceInfo)
+	merged.OSSnapshotPolicy.Merge(defaultOSSnapshotPolicy, &def.OSSnapshotPolicy, GlobalPolicySourceInfo)
 	merged.LoggingPolicy.Merge(defaultLoggingPolicy, &def.LoggingPolicy, GlobalPolicySourceInfo)
 
 	if len(policies) > 0 {

snapshot/policy/policy_merge_test.go

@@ -152,6 +152,10 @@ func testPolicyMergeSingleField(t *testing.T, fieldName string, typ reflect.Type
 		v0 = reflect.ValueOf(compression.Name(""))
 		v1 = reflect.ValueOf(compression.Name("foo"))
 		v2 = reflect.ValueOf(compression.Name("bar"))
+	case "*policy.OSSnapshotMode":
+		v0 = reflect.ValueOf((*policy.OSSnapshotMode)(nil))
+		v1 = reflect.ValueOf(policy.NewOSSnapshotMode(policy.OSSnapshotNever))
+		v2 = reflect.ValueOf(policy.NewOSSnapshotMode(policy.OSSnapshotAlways))
 
 	default:
 		t.Fatalf("unhandled case: %v - %v - please update test", fieldName, typ)

snapshot/policy/policy_tree.go

@@ -53,6 +53,12 @@
 		RunMissed: NewOptionalBool(defaultRunMissed),
 	}
 
+	defaultOSSnapshotPolicy = OSSnapshotPolicy{
+		VolumeShadowCopy: VolumeShadowCopyPolicy{
+			Enable: NewOSSnapshotMode(OSSnapshotWhenAvailable),
+		},
+	}
+
 	defaultUploadPolicy = UploadPolicy{
 		MaxParallelSnapshots: newOptionalInt(1),
 		MaxParallelFileReads: nil, // defaults to runtime.NumCPUs()
@@ -70,6 +76,7 @@
 		SchedulingPolicy:    defaultSchedulingPolicy,
 		LoggingPolicy:       defaultLoggingPolicy,
 		Actions:             defaultActionsPolicy,
+		OSSnapshotPolicy:    defaultOSSnapshotPolicy,
 		UploadPolicy:        defaultUploadPolicy,
 	}
 

snapshot/snapshotfs/upload.go

@@ -598,12 +598,34 @@ func (u *Uploader) uploadDirWithCheckpointing(ctx context.Context, rootDir fs.Di
 		return nil, dirReadError{errors.Wrap(err, "error executing before-snapshot-root action")}
 	}
 
+	defer u.executeAfterFolderAction(ctx, "after-snapshot-root", policyTree.EffectivePolicy().Actions.AfterSnapshotRoot, localDirPathOrEmpty, &hc)
+
+	p := &policyTree.EffectivePolicy().OSSnapshotPolicy
+
+	switch mode := osSnapshotMode(p); mode {
+	case policy.OSSnapshotNever:
+	case policy.OSSnapshotAlways, policy.OSSnapshotWhenAvailable:
+		if overrideDir != nil {
+			rootDir = overrideDir
+		}
+
+		switch osSnapshotDir, cleanup, err := createOSSnapshot(ctx, rootDir, p); {
+		case err == nil:
+			defer cleanup()
+
+			overrideDir = osSnapshotDir
+
+		case mode == policy.OSSnapshotWhenAvailable:
+			uploadLog(ctx).Warnf("OS file system snapshot failed (ignoring): %v", err)
+		default:
+			return nil, dirReadError{errors.Wrap(err, "error creating OS file system snapshot")}
+		}
+	}
+
 	if overrideDir != nil {
 		rootDir = u.wrapIgnorefs(uploadLog(ctx), overrideDir, policyTree, true)
 	}
 
-	defer u.executeAfterFolderAction(ctx, "after-snapshot-root", policyTree.EffectivePolicy().Actions.AfterSnapshotRoot, localDirPathOrEmpty, &hc)
-
 	return uploadDirInternal(ctx, u, rootDir, policyTree, previousDirs, localDirPathOrEmpty, ".", &dmb, &cp)
 }
 

snapshot/snapshotfs/upload_os_snapshot_nonwindows.go

@@ -0,0 +1,21 @@
+//go:build !windows
+// +build !windows
+
+package snapshotfs
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+
+	"github.com/kopia/kopia/fs"
+	"github.com/kopia/kopia/snapshot/policy"
+)
+
+func osSnapshotMode(*policy.OSSnapshotPolicy) policy.OSSnapshotMode {
+	return policy.OSSnapshotNever
+}
+
+func createOSSnapshot(context.Context, fs.Directory, *policy.OSSnapshotPolicy) (newRoot fs.Directory, cleanup func(), err error) {
+	return nil, nil, errors.New("not supported on this platform")
+}

snapshot/snapshotfs/upload_os_snapshot_windows.go

@@ -0,0 +1,90 @@
+package snapshotfs
+
+import (
+	"context"
+	"math/rand"
+	"path/filepath"
+	"time"
+
+	"github.com/mxk/go-vss"
+	"github.com/pkg/errors"
+
+	"github.com/kopia/kopia/fs"
+	"github.com/kopia/kopia/fs/localfs"
+	"github.com/kopia/kopia/internal/clock"
+	"github.com/kopia/kopia/snapshot/policy"
+)
+
+func osSnapshotMode(p *policy.OSSnapshotPolicy) policy.OSSnapshotMode {
+	return p.VolumeShadowCopy.Enable.OrDefault(policy.OSSnapshotNever)
+}
+
+//nolint:wrapcheck
+func createOSSnapshot(ctx context.Context, root fs.Directory, _ *policy.OSSnapshotPolicy) (newRoot fs.Directory, cleanup func(), finalErr error) {
+	local := root.LocalFilesystemPath()
+	if local == "" {
+		return nil, nil, errors.New("not a local filesystem")
+	}
+
+	ok, err := vss.IsShadowCopy(local)
+	if err != nil {
+		uploadLog(ctx).Warnf("failed to determine whether path is a volume shadow copy: %s (%v)", local, err)
+	} else if ok {
+		uploadLog(ctx).Warnf("path is already a volume shadow copy (skipping creation): %s", local)
+		return root, func() {}, nil
+	}
+
+	vol, rel, err := vss.SplitVolume(local)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	uploadLog(ctx).Infof("creating volume shadow copy of %v", vol)
+
+	id, err := vss.Create(vol)
+	if err != nil {
+		if e := vss.CreateError(0); !errors.As(err, &e) || e != 9 {
+			return nil, nil, err
+		}
+
+		// Retry "Another shadow copy operation is already in progress" in 5-10s
+		//nolint:gosec,gomnd
+		delay := 5*time.Second + time.Duration(rand.Int63n(int64(5*time.Second)))
+		if !clock.SleepInterruptibly(ctx, delay) {
+			return nil, nil, ctx.Err()
+		} else if id, err = vss.Create(vol); err != nil {
+			return nil, nil, err
+		}
+	}
+
+	defer func() {
+		if err != nil {
+			_ = vss.Remove(id)
+		}
+	}()
+
+	uploadLog(ctx).Infof("new volume shadow copy id %s", id)
+
+	sc, err := vss.Get(id)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	newRoot, err = localfs.Directory(filepath.Join(sc.DeviceObject, rel))
+
+	if err != nil {
+		return nil, nil, err
+	}
+
+	uploadLog(ctx).Debugf("shadow copy root is %s", newRoot.LocalFilesystemPath())
+
+	cleanup = func() {
+		uploadLog(ctx).Infof("removing volume shadow copy id %s", id)
+
+		if err := vss.Remove(id); err != nil {
+			uploadLog(ctx).Errorf("failed to remove volume shadow copy: %v", err)
+		}
+	}
+
+	return newRoot, cleanup, nil
+}

snapshot/snapshotfs/upload_test.go

@@ -1571,6 +1571,7 @@ func TestUploadLogging(t *testing.T) {
 			u.ParallelUploads = 1
 
 			pol := *policy.DefaultPolicy
+			pol.OSSnapshotPolicy.VolumeShadowCopy.Enable = policy.NewOSSnapshotMode(policy.OSSnapshotNever)
 			if p := tc.globalLoggingPolicy; p != nil {
 				pol.LoggingPolicy = *p
 			}

tests/os_snapshot_test/os_snapshot_nonwindows_test.go

@@ -0,0 +1,4 @@
+//go:build !windows
+// +build !windows
+
+package os_snapshot_test

tests/os_snapshot_test/os_snapshot_windows_test.go

@@ -0,0 +1,58 @@
+package os_snapshot_test
+
+import (
+	"os"
+	"testing"
+
+	"github.com/mxk/go-vss"
+	"github.com/pkg/errors"
+	"github.com/stretchr/testify/require"
+
+	"github.com/kopia/kopia/internal/tempfile"
+	"github.com/kopia/kopia/internal/testutil"
+	"github.com/kopia/kopia/tests/clitestutil"
+	"github.com/kopia/kopia/tests/testenv"
+)
+
+func TestShadowCopy(t *testing.T) {
+	kopiaExe := os.Getenv("KOPIA_EXE")
+	if kopiaExe == "" {
+		t.Skip()
+	}
+
+	runner := testenv.NewExeRunnerWithBinary(t, kopiaExe)
+
+	e := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, runner)
+	e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
+
+	root := testutil.TempDirectory(t)
+	f, err := tempfile.Create(root)
+	require.NoError(t, err)
+	_, err = f.WriteString("locked file\n")
+	require.NoError(t, err)
+	require.NoError(t, f.Sync())
+
+	defer f.Close()
+
+	_, err = vss.Get("{00000000-0000-0000-0000-000000000000}")
+	isAdmin := !errors.Is(err, os.ErrPermission)
+
+	if isAdmin {
+		t.Log("Running as admin, expecting snapshot creation to succeed")
+		e.RunAndExpectSuccess(t, "snap", "create", root)
+	} else {
+		t.Log("Not running as admin, expecting snapshot creation to fail")
+		e.RunAndExpectFailure(t, "snap", "create", root)
+	}
+
+	sources := clitestutil.ListSnapshotsAndExpectSuccess(t, e)
+	oid := sources[0].Snapshots[0].ObjectID
+	entries := clitestutil.ListDirectory(t, e, oid)
+
+	if isAdmin {
+		lines := e.RunAndExpectSuccess(t, "show", entries[0].ObjectID)
+		require.Equal(t, []string{"locked file"}, lines)
+	} else {
+		require.Empty(t, entries)
+	}
+}