diff --git a/.github/workflows/volume-shadow-copy-test.yml b/.github/workflows/volume-shadow-copy-test.yml new file mode 100644 index 000000000..d04232819 --- /dev/null +++ b/.github/workflows/volume-shadow-copy-test.yml @@ -0,0 +1,42 @@ +name: Volume Shadow Copy Test +on: + push: + branches: [ master ] + tags: + - v* + pull_request: + branches: [ master ] +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + vss-test: + name: Volume Shadow Copy Test + runs-on: windows-latest + steps: + - name: Check out repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + - name: Set up Go + uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + with: + go-version-file: 'go.mod' + check-latest: true + id: go + - name: Install gsudo + shell: bash + run: | + choco install -y --no-progress gsudo + echo "C:\tools\gsudo\Current" >> $GITHUB_PATH + - name: Admin Test + run: gsudo make os-snapshot-tests + - name: Non-Admin Test + run: gsudo -i Medium make os-snapshot-tests + - name: Upload Logs + uses: actions/upload-artifact@c7d193f32edcb7bfad88892161225aeda64e9392 # v4.0.0 + with: + name: logs + path: .logs/**/*.log + if-no-files-found: ignore + if: ${{ always() }} diff --git a/Makefile b/Makefile index 91f718772..bbc131806 100644 --- a/Makefile +++ b/Makefile @@ -213,7 +213,7 @@ download-rclone: go run ./tools/gettool --tool rclone:$(RCLONE_VERSION) --output-dir dist/kopia_linux_arm_6/ --goos=linux --goarch=arm -ci-tests: vet test +ci-tests: vet test ci-integration-tests: $(MAKE) robustness-tool-tests socket-activation-tests @@ -342,6 +342,11 @@ stress-test: $(gotestsum) $(GO_TEST) -count=$(REPEAT_TEST) -timeout 3600s github.com/kopia/kopia/tests/stress_test $(GO_TEST) -count=$(REPEAT_TEST) -timeout 3600s github.com/kopia/kopia/tests/repository_stress_test +os-snapshot-tests: export KOPIA_EXE ?= $(KOPIA_INTEGRATION_EXE) +os-snapshot-tests: GOTESTSUM_FORMAT=testname +os-snapshot-tests: build-integration-test-binary $(gotestsum) + $(GO_TEST) -count=$(REPEAT_TEST) github.com/kopia/kopia/tests/os_snapshot_test $(TEST_FLAGS) + layering-test: ifneq ($(GOOS),windows) # verify that code under repo/ can only import code also under repo/ + some @@ -484,6 +489,6 @@ perf-benchmark-test-all: $(MAKE) perf-benchmark-test PERF_BENCHMARK_VERSION=0.7.0~rc1 perf-benchmark-results: - gcloud compute scp $(PERF_BENCHMARK_INSTANCE):psrecord-* tests/perf_benchmark --zone=$(PERF_BENCHMARK_INSTANCE_ZONE) + gcloud compute scp $(PERF_BENCHMARK_INSTANCE):psrecord-* tests/perf_benchmark --zone=$(PERF_BENCHMARK_INSTANCE_ZONE) gcloud compute scp $(PERF_BENCHMARK_INSTANCE):repo-size-* tests/perf_benchmark --zone=$(PERF_BENCHMARK_INSTANCE_ZONE) (cd tests/perf_benchmark && go run process_results.go) diff --git a/cli/command_policy_set.go b/cli/command_policy_set.go index 42dccce89..be822798e 100644 --- a/cli/command_policy_set.go +++ b/cli/command_policy_set.go @@ -24,6 +24,7 @@ type commandPolicySet struct { policyLoggingFlags policyRetentionFlags policySchedulingFlags + policyOSSnapshotFlags policyUploadFlags } @@ -39,6 +40,7 @@ func (c *commandPolicySet) setup(svc appServices, parent commandParent) { c.policyLoggingFlags.setup(cmd) c.policyRetentionFlags.setup(cmd) c.policySchedulingFlags.setup(cmd) + c.policyOSSnapshotFlags.setup(cmd) c.policyUploadFlags.setup(cmd) cmd.Action(svc.repositoryWriterAction(c.run)) @@ -112,6 +114,10 @@ func (c *commandPolicySet) setPolicyFromFlags(ctx context.Context, p *policy.Pol return errors.Wrap(err, "actions policy") } + if err := c.setOSSnapshotPolicyFromFlags(ctx, &p.OSSnapshotPolicy, changeCount); err != nil { + return errors.Wrap(err, "OS snapshot policy") + } + if err := c.setLoggingPolicyFromFlags(ctx, &p.LoggingPolicy, changeCount); err != nil { return errors.Wrap(err, "actions policy") } diff --git a/cli/command_policy_set_os_snapshot.go b/cli/command_policy_set_os_snapshot.go new file mode 100644 index 000000000..596973db2 --- /dev/null +++ b/cli/command_policy_set_os_snapshot.go @@ -0,0 +1,64 @@ +package cli + +import ( + "context" + + "github.com/alecthomas/kingpin/v2" + "github.com/pkg/errors" + + "github.com/kopia/kopia/snapshot/policy" +) + +type policyOSSnapshotFlags struct { + policyEnableVolumeShadowCopy string +} + +func (c *policyOSSnapshotFlags) setup(cmd *kingpin.CmdClause) { + osSnapshotMode := []string{policy.OSSnapshotNeverString, policy.OSSnapshotAlwaysString, policy.OSSnapshotWhenAvailableString, inheritPolicyString} + + cmd.Flag("enable-volume-shadow-copy", "Enable Volume Shadow Copy snapshots ('never', 'always', 'when-available', 'inherit')").PlaceHolder("MODE").EnumVar(&c.policyEnableVolumeShadowCopy, osSnapshotMode...) +} + +func (c *policyOSSnapshotFlags) setOSSnapshotPolicyFromFlags(ctx context.Context, fp *policy.OSSnapshotPolicy, changeCount *int) error { + if err := applyPolicyOSSnapshotMode(ctx, "enable volume shadow copy", &fp.VolumeShadowCopy.Enable, c.policyEnableVolumeShadowCopy, changeCount); err != nil { + return errors.Wrap(err, "enable volume shadow copy") + } + + return nil +} + +func applyPolicyOSSnapshotMode(ctx context.Context, desc string, val **policy.OSSnapshotMode, str string, changeCount *int) error { + if str == "" { + // not changed + return nil + } + + var mode policy.OSSnapshotMode + + switch str { + case inheritPolicyString, defaultPolicyString: + *changeCount++ + + log(ctx).Infof(" - resetting %q to a default value inherited from parent.", desc) + + *val = nil + + return nil + case policy.OSSnapshotNeverString: + mode = policy.OSSnapshotNever + case policy.OSSnapshotAlwaysString: + mode = policy.OSSnapshotAlways + case policy.OSSnapshotWhenAvailableString: + mode = policy.OSSnapshotWhenAvailable + default: + return errors.Errorf("invalid %q mode %q", desc, str) + } + + *changeCount++ + + log(ctx).Infof(" - setting %q to %v.", desc, mode) + + *val = &mode + + return nil +} diff --git a/cli/command_policy_set_os_snapshot_test.go b/cli/command_policy_set_os_snapshot_test.go new file mode 100644 index 000000000..5aea57bb2 --- /dev/null +++ b/cli/command_policy_set_os_snapshot_test.go @@ -0,0 +1,42 @@ +package cli_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kopia/kopia/internal/testutil" + "github.com/kopia/kopia/tests/testenv" +) + +func TestSetOSSnapshotPolicy(t *testing.T) { + e := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, testenv.NewInProcRunner(t)) + defer e.RunAndExpectSuccess(t, "repo", "disconnect") + + e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) + + lines := e.RunAndExpectSuccess(t, "policy", "show", "--global") + lines = compressSpaces(lines) + require.Contains(t, lines, " Volume Shadow Copy: when-available (defined for this target)") + + // make some directory we'll be setting policy on + td := testutil.TempDirectory(t) + + lines = e.RunAndExpectSuccess(t, "policy", "show", td) + lines = compressSpaces(lines) + require.Contains(t, lines, " Volume Shadow Copy: when-available inherited from (global)") + + e.RunAndExpectSuccess(t, "policy", "set", "--global", "--enable-volume-shadow-copy=always") + + lines = e.RunAndExpectSuccess(t, "policy", "show", td) + lines = compressSpaces(lines) + + require.Contains(t, lines, " Volume Shadow Copy: always inherited from (global)") + + e.RunAndExpectSuccess(t, "policy", "set", "--enable-volume-shadow-copy=never", td) + + lines = e.RunAndExpectSuccess(t, "policy", "show", td) + lines = compressSpaces(lines) + + require.Contains(t, lines, " Volume Shadow Copy: never (defined for this target)") +} diff --git a/cli/command_policy_show.go b/cli/command_policy_show.go index df7b29655..dcc6fc439 100644 --- a/cli/command_policy_show.go +++ b/cli/command_policy_show.go @@ -128,6 +128,8 @@ func printPolicy(out *textOutput, p *policy.Policy, def *policy.Definition) { rows = append(rows, policyTableRow{}) rows = appendActionsPolicyRows(rows, p, def) rows = append(rows, policyTableRow{}) + rows = appendOSSnapshotPolicyRows(rows, p, def) + rows = append(rows, policyTableRow{}) rows = appendLoggingPolicyRows(rows, p, def) out.printStdout("Policy for %v:\n\n%v\n", p.Target(), alignedPolicyTableRows(rows)) @@ -449,6 +451,15 @@ func appendActionCommandRows(rows []policyTableRow, h *policy.ActionCommand) []p return rows } +func appendOSSnapshotPolicyRows(rows []policyTableRow, p *policy.Policy, def *policy.Definition) []policyTableRow { + rows = append(rows, + policyTableRow{"OS-level snapshot support:", "", ""}, + policyTableRow{" Volume Shadow Copy:", p.OSSnapshotPolicy.VolumeShadowCopy.Enable.String(), definitionPointToString(p.Target(), def.OSSnapshotPolicy.VolumeShadowCopy.Enable)}, + ) + + return rows +} + func valueOrNotSet(p *policy.OptionalInt) string { if p == nil { return "-" diff --git a/fs/localfs/local_fs_os.go b/fs/localfs/local_fs_os.go index 755db1db0..e790b5b38 100644 --- a/fs/localfs/local_fs_os.go +++ b/fs/localfs/local_fs_os.go @@ -5,7 +5,9 @@ "io" "os" "path/filepath" + "runtime" "strings" + "syscall" "github.com/pkg/errors" @@ -109,7 +111,20 @@ func NewEntry(path string) (fs.Entry, error) { fi, err := os.Lstat(path) if err != nil { - return nil, errors.Wrap(err, "unable to determine entry type") + // Paths such as `\\?\GLOBALROOT\Device\HarddiskVolumeShadowCopy01` + // cause os.Lstat to fail with "Incorrect function" error unless they + // end with a separator. Retry the operation with the separator added. + var e syscall.Errno + //nolint:goconst + if runtime.GOOS == "windows" && + !strings.HasSuffix(path, string(filepath.Separator)) && + errors.As(err, &e) && e == 1 { + fi, err = os.Lstat(path + string(filepath.Separator)) + } + + if err != nil { + return nil, errors.Wrap(err, "unable to determine entry type") + } } if path == "/" { diff --git a/go.mod b/go.mod index 0e129f70f..0ee158a4b 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( github.com/kylelemons/godebug v1.1.0 github.com/mattn/go-colorable v0.1.13 github.com/minio/minio-go/v7 v7.0.66 + github.com/mxk/go-vss v1.2.0 github.com/natefinch/atomic v1.0.1 github.com/pierrec/lz4 v2.6.1+incompatible github.com/pkg/errors v0.9.1 @@ -93,6 +94,7 @@ require ( github.com/frankban/quicktest v1.13.1 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect github.com/gobwas/httphead v0.1.0 // indirect github.com/gobwas/pool v0.2.1 // indirect github.com/gobwas/ws v1.3.0 // indirect diff --git a/go.sum b/go.sum index ed58d340c..200e27f28 100644 --- a/go.sum +++ b/go.sum @@ -108,6 +108,8 @@ github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= @@ -237,6 +239,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mxk/go-vss v1.2.0 h1:JpdOPc/P6B3XyRoddn0iMiG/ADBi3AuEsv8RlTb+JeE= +github.com/mxk/go-vss v1.2.0/go.mod h1:ZQ4yFxCG54vqPnCd+p2IxAe5jwZdz56wSjbwzBXiFd8= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= diff --git a/internal/atomicfile/atomicfile.go b/internal/atomicfile/atomicfile.go index a80b4a61d..5769e9093 100644 --- a/internal/atomicfile/atomicfile.go +++ b/internal/atomicfile/atomicfile.go @@ -21,26 +21,19 @@ // Because long file names have certain limitations: // - we must replace forward slashes with backslashes. // - dummy path element (\.\) must be removed. +// +// Relative paths are always limited to a total of MAX_PATH characters: +// https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation func MaybePrefixLongFilenameOnWindows(fname string) string { - if runtime.GOOS != "windows" { + if runtime.GOOS != "windows" || len(fname) < maxPathLength || + fname[:4] == `\\?\` || !ospath.IsAbs(fname) { return fname } - if len(fname) < maxPathLength { - return fname - } - - fname = strings.TrimPrefix(fname, "\\\\?\\") - - if !ospath.IsAbs(fname) { - // only convert absolute paths - return fname - } - - fixed := strings.ReplaceAll(fname, "/", "\\") + fixed := strings.ReplaceAll(fname, "/", `\`) for { - fixed2 := strings.ReplaceAll(fixed, "\\.\\", "\\") + fixed2 := strings.ReplaceAll(fixed, `\.\`, `\`) if fixed2 == fixed { break } @@ -48,7 +41,7 @@ func MaybePrefixLongFilenameOnWindows(fname string) string { fixed = fixed2 } - return "\\\\?\\" + fixed + return `\\?\` + fixed } // Write is a wrapper around atomic.WriteFile that handles long file names on Windows. diff --git a/internal/logfile/logfile_test.go b/internal/logfile/logfile_test.go index 2d867c887..8cf6c371c 100644 --- a/internal/logfile/logfile_test.go +++ b/internal/logfile/logfile_test.go @@ -20,9 +20,9 @@ ) var ( - cliLogFormat = regexp.MustCompile(`^\d{4}-\d\d\-\d\dT\d\d:\d\d:\d\d\.\d{6}Z (DEBUG|INFO) [a-z/]+ .*$`) + cliLogFormat = regexp.MustCompile(`^\d{4}-\d\d\-\d\dT\d\d:\d\d:\d\d\.\d{6}Z (DEBUG|INFO|WARN) [a-z/]+ .*$`) contentLogFormat = regexp.MustCompile(`^\d{4}-\d\d\-\d\dT\d\d:\d\d:\d\d\.\d{6}Z .*$`) - cliLogFormatLocalTimezone = regexp.MustCompile(`^\d{4}-\d\d\-\d\dT\d\d:\d\d:\d\d\.\d{6}[^Z][^ ]+ (DEBUG|INFO) [a-z/]+ .*$`) + cliLogFormatLocalTimezone = regexp.MustCompile(`^\d{4}-\d\d\-\d\dT\d\d:\d\d:\d\d\.\d{6}[^Z][^ ]+ (DEBUG|INFO|WARN) [a-z/]+ .*$`) ) func TestLoggingFlags(t *testing.T) { diff --git a/snapshot/policy/os_snapshot_policy.go b/snapshot/policy/os_snapshot_policy.go new file mode 100644 index 000000000..f1db2258e --- /dev/null +++ b/snapshot/policy/os_snapshot_policy.go @@ -0,0 +1,86 @@ +package policy + +import "github.com/kopia/kopia/snapshot" + +// OSSnapshotPolicy describes settings for OS-level snapshots. +type OSSnapshotPolicy struct { + VolumeShadowCopy VolumeShadowCopyPolicy `json:"volumeShadowCopy,omitempty"` +} + +// OSSnapshotPolicyDefinition specifies which policy definition provided the value of a particular field. +type OSSnapshotPolicyDefinition struct { + VolumeShadowCopy VolumeShadowCopyPolicyDefinition `json:"volumeShadowCopy,omitempty"` +} + +// Merge applies default values from the provided policy. +func (p *OSSnapshotPolicy) Merge(src OSSnapshotPolicy, def *OSSnapshotPolicyDefinition, si snapshot.SourceInfo) { + p.VolumeShadowCopy.Merge(src.VolumeShadowCopy, &def.VolumeShadowCopy, si) +} + +// VolumeShadowCopyPolicy describes settings for Windows Volume Shadow Copy +// snapshots. +type VolumeShadowCopyPolicy struct { + Enable *OSSnapshotMode `json:"enable,omitempty"` +} + +// VolumeShadowCopyPolicyDefinition specifies which policy definition provided +// the value of a particular field. +type VolumeShadowCopyPolicyDefinition struct { + Enable snapshot.SourceInfo `json:"enable,omitempty"` +} + +// Merge applies default values from the provided policy. +func (p *VolumeShadowCopyPolicy) Merge(src VolumeShadowCopyPolicy, def *VolumeShadowCopyPolicyDefinition, si snapshot.SourceInfo) { + mergeOSSnapshotMode(&p.Enable, src.Enable, &def.Enable, si) +} + +// OSSnapshotMode specifies whether OS-level snapshots are used for file systems +// that support them. +type OSSnapshotMode byte + +// OS-level snapshot modes. +const ( + OSSnapshotNever OSSnapshotMode = iota // Disable OS-level snapshots + OSSnapshotAlways // Fail if an OS-level snapshot cannot be created + OSSnapshotWhenAvailable // Fall back to regular file access on error +) + +// OS-level snapshot mode strings. +const ( + OSSnapshotNeverString = "never" + OSSnapshotAlwaysString = "always" + OSSnapshotWhenAvailableString = "when-available" +) + +// NewOSSnapshotMode provides an OptionalBool pointer. +func NewOSSnapshotMode(m OSSnapshotMode) *OSSnapshotMode { + return &m +} + +// OrDefault returns the OS snapshot mode or the provided default. +func (m *OSSnapshotMode) OrDefault(def OSSnapshotMode) OSSnapshotMode { + if m == nil { + return def + } + + return *m +} + +func (m OSSnapshotMode) String() string { + switch m { + case OSSnapshotAlways: + return OSSnapshotAlwaysString + case OSSnapshotWhenAvailable: + return OSSnapshotWhenAvailableString + default: + return OSSnapshotNeverString + } +} + +func mergeOSSnapshotMode(target **OSSnapshotMode, src *OSSnapshotMode, def *snapshot.SourceInfo, si snapshot.SourceInfo) { + if *target == nil && src != nil { + v := *src + *target = &v + *def = si + } +} diff --git a/snapshot/policy/os_snapshot_policy_test.go b/snapshot/policy/os_snapshot_policy_test.go new file mode 100644 index 000000000..e9e2be460 --- /dev/null +++ b/snapshot/policy/os_snapshot_policy_test.go @@ -0,0 +1,25 @@ +package policy + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestOSSnapshotMode(t *testing.T) { + assert.Equal(t, OSSnapshotNever, (*OSSnapshotMode)(nil).OrDefault(OSSnapshotNever)) + assert.Equal(t, OSSnapshotAlways, NewOSSnapshotMode(OSSnapshotAlways).OrDefault(OSSnapshotNever)) + + cases := []struct { + m OSSnapshotMode + s string + }{ + {OSSnapshotNever, "never"}, + {OSSnapshotAlways, "always"}, + {OSSnapshotWhenAvailable, "when-available"}, + } + + for _, tc := range cases { + assert.Equal(t, tc.s, tc.m.String()) + } +} diff --git a/snapshot/policy/policy.go b/snapshot/policy/policy.go index ce4512c09..85beed3cd 100644 --- a/snapshot/policy/policy.go +++ b/snapshot/policy/policy.go @@ -28,6 +28,7 @@ type Policy struct { SchedulingPolicy SchedulingPolicy `json:"scheduling,omitempty"` CompressionPolicy CompressionPolicy `json:"compression,omitempty"` Actions ActionsPolicy `json:"actions,omitempty"` + OSSnapshotPolicy OSSnapshotPolicy `json:"osSnapshots,omitempty"` LoggingPolicy LoggingPolicy `json:"logging,omitempty"` UploadPolicy UploadPolicy `json:"upload,omitempty"` NoParent bool `json:"noParent,omitempty"` @@ -42,6 +43,7 @@ type Definition struct { SchedulingPolicy SchedulingPolicyDefinition `json:"scheduling,omitempty"` CompressionPolicy CompressionPolicyDefinition `json:"compression,omitempty"` Actions ActionsPolicyDefinition `json:"actions,omitempty"` + OSSnapshotPolicy OSSnapshotPolicyDefinition `json:"osSnapshots,omitempty"` LoggingPolicy LoggingPolicyDefinition `json:"logging,omitempty"` UploadPolicy UploadPolicyDefinition `json:"upload,omitempty"` } diff --git a/snapshot/policy/policy_merge.go b/snapshot/policy/policy_merge.go index b2e87f518..5c3188021 100644 --- a/snapshot/policy/policy_merge.go +++ b/snapshot/policy/policy_merge.go @@ -25,6 +25,7 @@ func MergePolicies(policies []*Policy, si snapshot.SourceInfo) (*Policy, *Defini merged.UploadPolicy.Merge(p.UploadPolicy, &def.UploadPolicy, p.Target()) merged.CompressionPolicy.Merge(p.CompressionPolicy, &def.CompressionPolicy, p.Target()) merged.Actions.Merge(p.Actions, &def.Actions, p.Target()) + merged.OSSnapshotPolicy.Merge(p.OSSnapshotPolicy, &def.OSSnapshotPolicy, p.Target()) merged.LoggingPolicy.Merge(p.LoggingPolicy, &def.LoggingPolicy, p.Target()) if p.NoParent { @@ -40,6 +41,7 @@ func MergePolicies(policies []*Policy, si snapshot.SourceInfo) (*Policy, *Defini merged.UploadPolicy.Merge(defaultUploadPolicy, &def.UploadPolicy, GlobalPolicySourceInfo) merged.CompressionPolicy.Merge(defaultCompressionPolicy, &def.CompressionPolicy, GlobalPolicySourceInfo) merged.Actions.Merge(defaultActionsPolicy, &def.Actions, GlobalPolicySourceInfo) + merged.OSSnapshotPolicy.Merge(defaultOSSnapshotPolicy, &def.OSSnapshotPolicy, GlobalPolicySourceInfo) merged.LoggingPolicy.Merge(defaultLoggingPolicy, &def.LoggingPolicy, GlobalPolicySourceInfo) if len(policies) > 0 { diff --git a/snapshot/policy/policy_merge_test.go b/snapshot/policy/policy_merge_test.go index d4c66d0a2..dd4c33751 100644 --- a/snapshot/policy/policy_merge_test.go +++ b/snapshot/policy/policy_merge_test.go @@ -152,6 +152,10 @@ func testPolicyMergeSingleField(t *testing.T, fieldName string, typ reflect.Type v0 = reflect.ValueOf(compression.Name("")) v1 = reflect.ValueOf(compression.Name("foo")) v2 = reflect.ValueOf(compression.Name("bar")) + case "*policy.OSSnapshotMode": + v0 = reflect.ValueOf((*policy.OSSnapshotMode)(nil)) + v1 = reflect.ValueOf(policy.NewOSSnapshotMode(policy.OSSnapshotNever)) + v2 = reflect.ValueOf(policy.NewOSSnapshotMode(policy.OSSnapshotAlways)) default: t.Fatalf("unhandled case: %v - %v - please update test", fieldName, typ) diff --git a/snapshot/policy/policy_tree.go b/snapshot/policy/policy_tree.go index 7ef758a97..ee99a4a0f 100644 --- a/snapshot/policy/policy_tree.go +++ b/snapshot/policy/policy_tree.go @@ -53,6 +53,12 @@ RunMissed: NewOptionalBool(defaultRunMissed), } + defaultOSSnapshotPolicy = OSSnapshotPolicy{ + VolumeShadowCopy: VolumeShadowCopyPolicy{ + Enable: NewOSSnapshotMode(OSSnapshotWhenAvailable), + }, + } + defaultUploadPolicy = UploadPolicy{ MaxParallelSnapshots: newOptionalInt(1), MaxParallelFileReads: nil, // defaults to runtime.NumCPUs() @@ -70,6 +76,7 @@ SchedulingPolicy: defaultSchedulingPolicy, LoggingPolicy: defaultLoggingPolicy, Actions: defaultActionsPolicy, + OSSnapshotPolicy: defaultOSSnapshotPolicy, UploadPolicy: defaultUploadPolicy, } diff --git a/snapshot/snapshotfs/upload.go b/snapshot/snapshotfs/upload.go index 92a4280b6..cc6366779 100644 --- a/snapshot/snapshotfs/upload.go +++ b/snapshot/snapshotfs/upload.go @@ -598,12 +598,34 @@ func (u *Uploader) uploadDirWithCheckpointing(ctx context.Context, rootDir fs.Di return nil, dirReadError{errors.Wrap(err, "error executing before-snapshot-root action")} } + defer u.executeAfterFolderAction(ctx, "after-snapshot-root", policyTree.EffectivePolicy().Actions.AfterSnapshotRoot, localDirPathOrEmpty, &hc) + + p := &policyTree.EffectivePolicy().OSSnapshotPolicy + + switch mode := osSnapshotMode(p); mode { + case policy.OSSnapshotNever: + case policy.OSSnapshotAlways, policy.OSSnapshotWhenAvailable: + if overrideDir != nil { + rootDir = overrideDir + } + + switch osSnapshotDir, cleanup, err := createOSSnapshot(ctx, rootDir, p); { + case err == nil: + defer cleanup() + + overrideDir = osSnapshotDir + + case mode == policy.OSSnapshotWhenAvailable: + uploadLog(ctx).Warnf("OS file system snapshot failed (ignoring): %v", err) + default: + return nil, dirReadError{errors.Wrap(err, "error creating OS file system snapshot")} + } + } + if overrideDir != nil { rootDir = u.wrapIgnorefs(uploadLog(ctx), overrideDir, policyTree, true) } - defer u.executeAfterFolderAction(ctx, "after-snapshot-root", policyTree.EffectivePolicy().Actions.AfterSnapshotRoot, localDirPathOrEmpty, &hc) - return uploadDirInternal(ctx, u, rootDir, policyTree, previousDirs, localDirPathOrEmpty, ".", &dmb, &cp) } diff --git a/snapshot/snapshotfs/upload_os_snapshot_nonwindows.go b/snapshot/snapshotfs/upload_os_snapshot_nonwindows.go new file mode 100644 index 000000000..e0039e12d --- /dev/null +++ b/snapshot/snapshotfs/upload_os_snapshot_nonwindows.go @@ -0,0 +1,21 @@ +//go:build !windows +// +build !windows + +package snapshotfs + +import ( + "context" + + "github.com/pkg/errors" + + "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/snapshot/policy" +) + +func osSnapshotMode(*policy.OSSnapshotPolicy) policy.OSSnapshotMode { + return policy.OSSnapshotNever +} + +func createOSSnapshot(context.Context, fs.Directory, *policy.OSSnapshotPolicy) (newRoot fs.Directory, cleanup func(), err error) { + return nil, nil, errors.New("not supported on this platform") +} diff --git a/snapshot/snapshotfs/upload_os_snapshot_windows.go b/snapshot/snapshotfs/upload_os_snapshot_windows.go new file mode 100644 index 000000000..eebe39af2 --- /dev/null +++ b/snapshot/snapshotfs/upload_os_snapshot_windows.go @@ -0,0 +1,90 @@ +package snapshotfs + +import ( + "context" + "math/rand" + "path/filepath" + "time" + + "github.com/mxk/go-vss" + "github.com/pkg/errors" + + "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/fs/localfs" + "github.com/kopia/kopia/internal/clock" + "github.com/kopia/kopia/snapshot/policy" +) + +func osSnapshotMode(p *policy.OSSnapshotPolicy) policy.OSSnapshotMode { + return p.VolumeShadowCopy.Enable.OrDefault(policy.OSSnapshotNever) +} + +//nolint:wrapcheck +func createOSSnapshot(ctx context.Context, root fs.Directory, _ *policy.OSSnapshotPolicy) (newRoot fs.Directory, cleanup func(), finalErr error) { + local := root.LocalFilesystemPath() + if local == "" { + return nil, nil, errors.New("not a local filesystem") + } + + ok, err := vss.IsShadowCopy(local) + if err != nil { + uploadLog(ctx).Warnf("failed to determine whether path is a volume shadow copy: %s (%v)", local, err) + } else if ok { + uploadLog(ctx).Warnf("path is already a volume shadow copy (skipping creation): %s", local) + return root, func() {}, nil + } + + vol, rel, err := vss.SplitVolume(local) + if err != nil { + return nil, nil, err + } + + uploadLog(ctx).Infof("creating volume shadow copy of %v", vol) + + id, err := vss.Create(vol) + if err != nil { + if e := vss.CreateError(0); !errors.As(err, &e) || e != 9 { + return nil, nil, err + } + + // Retry "Another shadow copy operation is already in progress" in 5-10s + //nolint:gosec,gomnd + delay := 5*time.Second + time.Duration(rand.Int63n(int64(5*time.Second))) + if !clock.SleepInterruptibly(ctx, delay) { + return nil, nil, ctx.Err() + } else if id, err = vss.Create(vol); err != nil { + return nil, nil, err + } + } + + defer func() { + if err != nil { + _ = vss.Remove(id) + } + }() + + uploadLog(ctx).Infof("new volume shadow copy id %s", id) + + sc, err := vss.Get(id) + if err != nil { + return nil, nil, err + } + + newRoot, err = localfs.Directory(filepath.Join(sc.DeviceObject, rel)) + + if err != nil { + return nil, nil, err + } + + uploadLog(ctx).Debugf("shadow copy root is %s", newRoot.LocalFilesystemPath()) + + cleanup = func() { + uploadLog(ctx).Infof("removing volume shadow copy id %s", id) + + if err := vss.Remove(id); err != nil { + uploadLog(ctx).Errorf("failed to remove volume shadow copy: %v", err) + } + } + + return newRoot, cleanup, nil +} diff --git a/snapshot/snapshotfs/upload_test.go b/snapshot/snapshotfs/upload_test.go index b4d4d431a..7ebea0c44 100644 --- a/snapshot/snapshotfs/upload_test.go +++ b/snapshot/snapshotfs/upload_test.go @@ -1571,6 +1571,7 @@ func TestUploadLogging(t *testing.T) { u.ParallelUploads = 1 pol := *policy.DefaultPolicy + pol.OSSnapshotPolicy.VolumeShadowCopy.Enable = policy.NewOSSnapshotMode(policy.OSSnapshotNever) if p := tc.globalLoggingPolicy; p != nil { pol.LoggingPolicy = *p } diff --git a/tests/os_snapshot_test/os_snapshot_nonwindows_test.go b/tests/os_snapshot_test/os_snapshot_nonwindows_test.go new file mode 100644 index 000000000..37cf10ed3 --- /dev/null +++ b/tests/os_snapshot_test/os_snapshot_nonwindows_test.go @@ -0,0 +1,4 @@ +//go:build !windows +// +build !windows + +package os_snapshot_test diff --git a/tests/os_snapshot_test/os_snapshot_windows_test.go b/tests/os_snapshot_test/os_snapshot_windows_test.go new file mode 100644 index 000000000..05adabccc --- /dev/null +++ b/tests/os_snapshot_test/os_snapshot_windows_test.go @@ -0,0 +1,58 @@ +package os_snapshot_test + +import ( + "os" + "testing" + + "github.com/mxk/go-vss" + "github.com/pkg/errors" + "github.com/stretchr/testify/require" + + "github.com/kopia/kopia/internal/tempfile" + "github.com/kopia/kopia/internal/testutil" + "github.com/kopia/kopia/tests/clitestutil" + "github.com/kopia/kopia/tests/testenv" +) + +func TestShadowCopy(t *testing.T) { + kopiaExe := os.Getenv("KOPIA_EXE") + if kopiaExe == "" { + t.Skip() + } + + runner := testenv.NewExeRunnerWithBinary(t, kopiaExe) + + e := testenv.NewCLITest(t, testenv.RepoFormatNotImportant, runner) + e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir) + + root := testutil.TempDirectory(t) + f, err := tempfile.Create(root) + require.NoError(t, err) + _, err = f.WriteString("locked file\n") + require.NoError(t, err) + require.NoError(t, f.Sync()) + + defer f.Close() + + _, err = vss.Get("{00000000-0000-0000-0000-000000000000}") + isAdmin := !errors.Is(err, os.ErrPermission) + + if isAdmin { + t.Log("Running as admin, expecting snapshot creation to succeed") + e.RunAndExpectSuccess(t, "snap", "create", root) + } else { + t.Log("Not running as admin, expecting snapshot creation to fail") + e.RunAndExpectFailure(t, "snap", "create", root) + } + + sources := clitestutil.ListSnapshotsAndExpectSuccess(t, e) + oid := sources[0].Snapshots[0].ObjectID + entries := clitestutil.ListDirectory(t, e, oid) + + if isAdmin { + lines := e.RunAndExpectSuccess(t, "show", entries[0].ObjectID) + require.Equal(t, []string{"locked file"}, lines) + } else { + require.Empty(t, entries) + } +}