mirror of
https://github.com/kopia/kopia.git
synced 2026-04-29 02:15:39 -04:00
Added support for actions that run before&after snapshot roots and before/after specific folders (#722)
* policy: add actions * fs: added LocalFilesystemPath() which can optionally return local filesystem path (if entry is local) * cli: added support for setting policy actions * upload: support for executing actions before/after folder (non-inheritable) and before/after snapshots (inheritable) * testing: end-to-end test for actions * additional tests for actions with embedded scripts
This commit is contained in:
7
Makefile
7
Makefile
@@ -1,6 +1,7 @@
|
||||
COVERAGE_PACKAGES=github.com/kopia/kopia/repo/...,github.com/kopia/kopia/fs/...,github.com/kopia/kopia/snapshot/...
|
||||
TEST_FLAGS?=
|
||||
KOPIA_INTEGRATION_EXE=$(CURDIR)/dist/integration/kopia.exe
|
||||
TESTING_ACTION_EXE=$(CURDIR)/dist/integration/testingaction.exe
|
||||
FIO_DOCKER_TAG=ljishen/fio
|
||||
|
||||
export BOTO_PATH=$(CURDIR)/tools/.boto
|
||||
@@ -219,8 +220,12 @@ vtest: $(gotestsum)
|
||||
build-integration-test-binary:
|
||||
go build -o $(KOPIA_INTEGRATION_EXE) -tags testing github.com/kopia/kopia
|
||||
|
||||
$(TESTING_ACTION_EXE): tests/testingaction/main.go
|
||||
go build -o $(TESTING_ACTION_EXE) -tags testing github.com/kopia/kopia/tests/testingaction
|
||||
|
||||
integration-tests: export KOPIA_EXE ?= $(KOPIA_INTEGRATION_EXE)
|
||||
integration-tests: build-integration-test-binary $(gotestsum)
|
||||
integration-tests: export TESTING_ACTION_EXE ?= $(TESTING_ACTION_EXE)
|
||||
integration-tests: build-integration-test-binary $(gotestsum) $(TESTING_ACTION_EXE)
|
||||
$(GO_TEST) $(TEST_FLAGS) -count=1 -parallel $(PARALLEL) -timeout 3600s github.com/kopia/kopia/tests/end_to_end_test
|
||||
|
||||
endurance-tests: export KOPIA_EXE ?= $(KOPIA_INTEGRATION_EXE)
|
||||
|
||||
@@ -86,6 +86,10 @@ func setPolicyFromFlags(ctx context.Context, p *policy.Policy, changeCount *int)
|
||||
return errors.Wrap(err, "scheduling policy")
|
||||
}
|
||||
|
||||
if err := setActionsFromFlags(ctx, &p.Actions, changeCount); err != nil {
|
||||
return errors.Wrap(err, "actions policy")
|
||||
}
|
||||
|
||||
// It's not really a list, just optional boolean, last one wins.
|
||||
for _, inherit := range *policySetInherit {
|
||||
*changeCount++
|
||||
|
||||
116
cli/command_policy_set_actions.go
Normal file
116
cli/command_policy_set_actions.go
Normal file
@@ -0,0 +1,116 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/kopia/kopia/snapshot/policy"
|
||||
)
|
||||
|
||||
const maxScriptLength = 32000
|
||||
|
||||
var (
|
||||
policySetBeforeFolderActionCommand = policySetCommand.Flag("before-folder-action", "Path to before-folder action command ('none' to remove)").Default("-").PlaceHolder("COMMAND").String()
|
||||
policySetAfterFolderActionCommand = policySetCommand.Flag("after-folder-action", "Path to after-folder action command ('none' to remove)").Default("-").PlaceHolder("COMMAND").String()
|
||||
policySetBeforeSnapshotRootActionCommand = policySetCommand.Flag("before-snapshot-root-action", "Path to before-snapshot-root action command ('none' to remove or 'inherit')").Default("-").PlaceHolder("COMMAND").String()
|
||||
policySetAfterSnapshotRootActionCommand = policySetCommand.Flag("after-snapshot-root-action", "Path to after-snapshot-root action command ('none' to remove or 'inherit')").Default("-").PlaceHolder("COMMAND").String()
|
||||
policySetActionCommandTimeout = policySetCommand.Flag("action-command-timeout", "Max time allowed for a action to run in seconds").Default("5m").Duration()
|
||||
policySetActionCommandMode = policySetCommand.Flag("action-command-mode", "Action command mode").Default("essential").Enum("essential", "optional", "async")
|
||||
policySetPersistActionScript = policySetCommand.Flag("persist-action-script", "Persist action script").Bool()
|
||||
)
|
||||
|
||||
func setActionsFromFlags(ctx context.Context, p *policy.ActionsPolicy, changeCount *int) error {
|
||||
if err := setActionCommandFromFlags(ctx, "before-folder", &p.BeforeFolder, *policySetBeforeFolderActionCommand, changeCount); err != nil {
|
||||
return errors.Wrap(err, "invalid before-folder-action")
|
||||
}
|
||||
|
||||
if err := setActionCommandFromFlags(ctx, "after-folder", &p.AfterFolder, *policySetAfterFolderActionCommand, changeCount); err != nil {
|
||||
return errors.Wrap(err, "invalid after-folder-action")
|
||||
}
|
||||
|
||||
if err := setActionCommandFromFlags(ctx, "before-snapshot-root", &p.BeforeSnapshotRoot, *policySetBeforeSnapshotRootActionCommand, changeCount); err != nil {
|
||||
return errors.Wrap(err, "invalid before-snapshot-root-action")
|
||||
}
|
||||
|
||||
if err := setActionCommandFromFlags(ctx, "after-snapshot-root", &p.AfterSnapshotRoot, *policySetAfterSnapshotRootActionCommand, changeCount); err != nil {
|
||||
return errors.Wrap(err, "invalid after-snapshot-root-action")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func setActionCommandFromFlags(ctx context.Context, actionName string, cmd **policy.ActionCommand, value string, changeCount *int) error {
|
||||
if value == "-" {
|
||||
// not set
|
||||
return nil
|
||||
}
|
||||
|
||||
if value == "" {
|
||||
log(ctx).Infof(" - removing %v action", actionName)
|
||||
|
||||
*changeCount++
|
||||
|
||||
*cmd = nil
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
*cmd = &policy.ActionCommand{
|
||||
TimeoutSeconds: int(policySetActionCommandTimeout.Seconds()),
|
||||
Mode: *policySetActionCommandMode,
|
||||
}
|
||||
|
||||
*changeCount++
|
||||
|
||||
if *policySetPersistActionScript {
|
||||
script, err := ioutil.ReadFile(value) //nolint:gosec
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(script) > maxScriptLength {
|
||||
return errors.Errorf("action script file (%v) too long: %v, max allowed %d", value, len(script), maxScriptLength)
|
||||
}
|
||||
|
||||
log(ctx).Infof(" - setting %v (%v) action script from file %v (%v bytes) with timeout %v", actionName, *policySetActionCommandMode, value, len(script), *policySetActionCommandTimeout)
|
||||
|
||||
(*cmd).Script = string(script)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// parse path as CSV as if space was the separator, this automatically takes care of quotations
|
||||
r := csv.NewReader(strings.NewReader(value))
|
||||
r.Comma = ' ' // space
|
||||
|
||||
fields, err := r.Read()
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "error parsing %v command", actionName)
|
||||
}
|
||||
|
||||
(*cmd).Command = fields[0]
|
||||
(*cmd).Arguments = fields[1:]
|
||||
|
||||
if len((*cmd).Arguments) == 0 {
|
||||
log(ctx).Infof(" - setting %v (%v) action command to %v and timeout %v", actionName, *policySetActionCommandMode, quoteArguments((*cmd).Command), *policySetActionCommandTimeout)
|
||||
} else {
|
||||
log(ctx).Infof(" - setting %v (%v) action command to %v with arguments %v and timeout %v", actionName, *policySetActionCommandMode, quoteArguments((*cmd).Command), quoteArguments((*cmd).Arguments...), *policySetActionCommandTimeout)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func quoteArguments(s ...string) string {
|
||||
var result []string
|
||||
|
||||
for _, v := range s {
|
||||
result = append(result, fmt.Sprintf("\"%v\"", v))
|
||||
}
|
||||
|
||||
return strings.Join(result, " ")
|
||||
}
|
||||
@@ -3,11 +3,13 @@
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/kopia/kopia/internal/units"
|
||||
"github.com/kopia/kopia/repo"
|
||||
"github.com/kopia/kopia/snapshot"
|
||||
"github.com/kopia/kopia/snapshot/policy"
|
||||
)
|
||||
|
||||
@@ -44,10 +46,10 @@ func showPolicy(ctx context.Context, rep repo.Repository) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func getDefinitionPoint(parents []*policy.Policy, match func(p *policy.Policy) bool) string {
|
||||
for i, p := range parents {
|
||||
func getDefinitionPoint(target snapshot.SourceInfo, parents []*policy.Policy, match func(p *policy.Policy) bool) string {
|
||||
for _, p := range parents {
|
||||
if match(p) {
|
||||
if i == 0 {
|
||||
if p.Target() == target {
|
||||
return "(defined for this target)"
|
||||
}
|
||||
|
||||
@@ -84,38 +86,40 @@ func printPolicy(p *policy.Policy, parents []*policy.Policy) {
|
||||
printSchedulingPolicy(p, parents)
|
||||
printStdout("\n")
|
||||
printCompressionPolicy(p, parents)
|
||||
printStdout("\n")
|
||||
printActions(p, parents)
|
||||
}
|
||||
|
||||
func printRetentionPolicy(p *policy.Policy, parents []*policy.Policy) {
|
||||
printStdout("Retention:\n")
|
||||
printStdout(" Annual snapshots: %3v %v\n",
|
||||
valueOrNotSet(p.RetentionPolicy.KeepAnnual),
|
||||
getDefinitionPoint(parents, func(pol *policy.Policy) bool {
|
||||
getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
return pol.RetentionPolicy.KeepAnnual != nil
|
||||
}))
|
||||
printStdout(" Monthly snapshots: %3v %v\n",
|
||||
valueOrNotSet(p.RetentionPolicy.KeepMonthly),
|
||||
getDefinitionPoint(parents, func(pol *policy.Policy) bool {
|
||||
getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
return pol.RetentionPolicy.KeepMonthly != nil
|
||||
}))
|
||||
printStdout(" Weekly snapshots: %3v %v\n",
|
||||
valueOrNotSet(p.RetentionPolicy.KeepWeekly),
|
||||
getDefinitionPoint(parents, func(pol *policy.Policy) bool {
|
||||
getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
return pol.RetentionPolicy.KeepWeekly != nil
|
||||
}))
|
||||
printStdout(" Daily snapshots: %3v %v\n",
|
||||
valueOrNotSet(p.RetentionPolicy.KeepDaily),
|
||||
getDefinitionPoint(parents, func(pol *policy.Policy) bool {
|
||||
getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
return pol.RetentionPolicy.KeepDaily != nil
|
||||
}))
|
||||
printStdout(" Hourly snapshots: %3v %v\n",
|
||||
valueOrNotSet(p.RetentionPolicy.KeepHourly),
|
||||
getDefinitionPoint(parents, func(pol *policy.Policy) bool {
|
||||
getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
return pol.RetentionPolicy.KeepHourly != nil
|
||||
}))
|
||||
printStdout(" Latest snapshots: %3v %v\n",
|
||||
valueOrNotSet(p.RetentionPolicy.KeepLatest),
|
||||
getDefinitionPoint(parents, func(pol *policy.Policy) bool {
|
||||
getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
return pol.RetentionPolicy.KeepLatest != nil
|
||||
}))
|
||||
}
|
||||
@@ -125,7 +129,7 @@ func printFilesPolicy(p *policy.Policy, parents []*policy.Policy) {
|
||||
|
||||
printStdout(" Ignore cache directories: %5v %v\n",
|
||||
p.FilesPolicy.IgnoreCacheDirectoriesOrDefault(true),
|
||||
getDefinitionPoint(parents, func(pol *policy.Policy) bool {
|
||||
getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
return pol.FilesPolicy.IgnoreCacheDirs != nil
|
||||
}))
|
||||
|
||||
@@ -137,7 +141,7 @@ func printFilesPolicy(p *policy.Policy, parents []*policy.Policy) {
|
||||
|
||||
for _, rule := range p.FilesPolicy.IgnoreRules {
|
||||
rule := rule
|
||||
printStdout(" %-30v %v\n", rule, getDefinitionPoint(parents, func(pol *policy.Policy) bool {
|
||||
printStdout(" %-30v %v\n", rule, getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
return containsString(pol.FilesPolicy.IgnoreRules, rule)
|
||||
}))
|
||||
}
|
||||
@@ -148,7 +152,7 @@ func printFilesPolicy(p *policy.Policy, parents []*policy.Policy) {
|
||||
|
||||
for _, dotFile := range p.FilesPolicy.DotIgnoreFiles {
|
||||
dotFile := dotFile
|
||||
printStdout(" %-30v %v\n", dotFile, getDefinitionPoint(parents, func(pol *policy.Policy) bool {
|
||||
printStdout(" %-30v %v\n", dotFile, getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
return containsString(pol.FilesPolicy.DotIgnoreFiles, dotFile)
|
||||
}))
|
||||
}
|
||||
@@ -156,14 +160,14 @@ func printFilesPolicy(p *policy.Policy, parents []*policy.Policy) {
|
||||
if maxSize := p.FilesPolicy.MaxFileSize; maxSize > 0 {
|
||||
printStdout(" Ignore files above: %10v %v\n",
|
||||
units.BytesStringBase2(maxSize),
|
||||
getDefinitionPoint(parents, func(pol *policy.Policy) bool {
|
||||
getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
return pol.FilesPolicy.MaxFileSize != 0
|
||||
}))
|
||||
}
|
||||
|
||||
printStdout(" Scan one filesystem only: %5v %v\n",
|
||||
p.FilesPolicy.OneFileSystemOrDefault(false),
|
||||
getDefinitionPoint(parents, func(pol *policy.Policy) bool {
|
||||
getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
return pol.FilesPolicy.OneFileSystem != nil
|
||||
}))
|
||||
}
|
||||
@@ -173,13 +177,13 @@ func printErrorHandlingPolicy(p *policy.Policy, parents []*policy.Policy) {
|
||||
|
||||
printStdout(" Ignore file read errors: %5v %v\n",
|
||||
p.ErrorHandlingPolicy.IgnoreFileErrorsOrDefault(false),
|
||||
getDefinitionPoint(parents, func(pol *policy.Policy) bool {
|
||||
getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
return pol.ErrorHandlingPolicy.IgnoreFileErrors != nil
|
||||
}))
|
||||
|
||||
printStdout(" Ignore directory read errors: %5v %v\n",
|
||||
p.ErrorHandlingPolicy.IgnoreDirectoryErrorsOrDefault(false),
|
||||
getDefinitionPoint(parents, func(pol *policy.Policy) bool {
|
||||
getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
return pol.ErrorHandlingPolicy.IgnoreDirectoryErrors != nil
|
||||
}))
|
||||
}
|
||||
@@ -190,7 +194,7 @@ func printSchedulingPolicy(p *policy.Policy, parents []*policy.Policy) {
|
||||
any := false
|
||||
|
||||
if p.SchedulingPolicy.Interval() != 0 {
|
||||
printStdout(" Snapshot interval: %10v %v\n", p.SchedulingPolicy.Interval(), getDefinitionPoint(parents, func(pol *policy.Policy) bool {
|
||||
printStdout(" Snapshot interval: %10v %v\n", p.SchedulingPolicy.Interval(), getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
return pol.SchedulingPolicy.Interval() != 0
|
||||
}))
|
||||
|
||||
@@ -202,7 +206,7 @@ func printSchedulingPolicy(p *policy.Policy, parents []*policy.Policy) {
|
||||
|
||||
for _, tod := range p.SchedulingPolicy.TimesOfDay {
|
||||
tod := tod
|
||||
printStdout(" %9v %v\n", tod, getDefinitionPoint(parents, func(pol *policy.Policy) bool {
|
||||
printStdout(" %9v %v\n", tod, getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
for _, t := range pol.SchedulingPolicy.TimesOfDay {
|
||||
if t == tod {
|
||||
return true
|
||||
@@ -224,7 +228,7 @@ func printSchedulingPolicy(p *policy.Policy, parents []*policy.Policy) {
|
||||
func printCompressionPolicy(p *policy.Policy, parents []*policy.Policy) {
|
||||
if p.CompressionPolicy.CompressorName != "" && p.CompressionPolicy.CompressorName != "none" {
|
||||
printStdout("Compression:\n")
|
||||
printStdout(" Compressor: %q %v\n", p.CompressionPolicy.CompressorName, getDefinitionPoint(parents, func(pol *policy.Policy) bool {
|
||||
printStdout(" Compressor: %q %v\n", p.CompressionPolicy.CompressorName, getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
return pol.CompressionPolicy.CompressorName != ""
|
||||
}))
|
||||
} else {
|
||||
@@ -238,7 +242,7 @@ func printCompressionPolicy(p *policy.Policy, parents []*policy.Policy) {
|
||||
|
||||
for _, rule := range p.CompressionPolicy.OnlyCompress {
|
||||
rule := rule
|
||||
printStdout(" %-30v %v\n", rule, getDefinitionPoint(parents, func(pol *policy.Policy) bool {
|
||||
printStdout(" %-30v %v\n", rule, getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
return containsString(pol.CompressionPolicy.OnlyCompress, rule)
|
||||
}))
|
||||
}
|
||||
@@ -248,7 +252,7 @@ func printCompressionPolicy(p *policy.Policy, parents []*policy.Policy) {
|
||||
|
||||
for _, rule := range p.CompressionPolicy.NeverCompress {
|
||||
rule := rule
|
||||
printStdout(" %-30v %v\n", rule, getDefinitionPoint(parents, func(pol *policy.Policy) bool {
|
||||
printStdout(" %-30v %v\n", rule, getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
return containsString(pol.CompressionPolicy.NeverCompress, rule)
|
||||
}))
|
||||
}
|
||||
@@ -269,6 +273,60 @@ func printCompressionPolicy(p *policy.Policy, parents []*policy.Policy) {
|
||||
}
|
||||
}
|
||||
|
||||
func printActions(p *policy.Policy, parents []*policy.Policy) {
|
||||
var anyActions bool
|
||||
|
||||
if h := p.Actions.BeforeSnapshotRoot; h != nil {
|
||||
printStdout("Run command before snapshot root: %v\n", getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
return pol.Actions.BeforeSnapshotRoot == h
|
||||
}))
|
||||
|
||||
printActionCommand(h)
|
||||
|
||||
anyActions = true
|
||||
}
|
||||
|
||||
if h := p.Actions.AfterSnapshotRoot; h != nil {
|
||||
printStdout("Run command after snapshot root: %v\n", getDefinitionPoint(p.Target(), parents, func(pol *policy.Policy) bool {
|
||||
return pol.Actions.AfterSnapshotRoot == h
|
||||
}))
|
||||
printActionCommand(h)
|
||||
|
||||
anyActions = true
|
||||
}
|
||||
|
||||
if h := p.Actions.BeforeFolder; h != nil {
|
||||
printStdout("Run command before this folder: (non-inheritable)\n")
|
||||
|
||||
printActionCommand(h)
|
||||
|
||||
anyActions = true
|
||||
}
|
||||
|
||||
if h := p.Actions.AfterFolder; h != nil {
|
||||
printStdout("Run command after this folder: (non-inheritable)\n")
|
||||
printActionCommand(h)
|
||||
|
||||
anyActions = true
|
||||
}
|
||||
|
||||
if !anyActions {
|
||||
printStdout("No actions defined.\n")
|
||||
}
|
||||
}
|
||||
|
||||
func printActionCommand(h *policy.ActionCommand) {
|
||||
if h.Script != "" {
|
||||
printStdout(" Embedded Script: %q\n", h.Script)
|
||||
} else {
|
||||
printStdout(" Command: %v %v\n", h.Command, strings.Join(h.Arguments, " "))
|
||||
}
|
||||
|
||||
printStdout(" Mode: %v\n", h.Mode)
|
||||
printStdout(" Timeout: %v\n", h.TimeoutSeconds)
|
||||
printStdout("\n")
|
||||
}
|
||||
|
||||
func valueOrNotSet(p *int) string {
|
||||
if p == nil {
|
||||
return "-"
|
||||
|
||||
@@ -14,6 +14,7 @@ type Entry interface {
|
||||
os.FileInfo
|
||||
Owner() OwnerInfo
|
||||
Device() DeviceInfo
|
||||
LocalFilesystemPath() string // returns full local filesystem path or "" if not a local filesystem
|
||||
}
|
||||
|
||||
// OwnerInfo describes owner of a filesystem entry.
|
||||
|
||||
@@ -77,6 +77,10 @@ func (e *filesystemEntry) Device() fs.DeviceInfo {
|
||||
return e.device
|
||||
}
|
||||
|
||||
func (e *filesystemEntry) LocalFilesystemPath() string {
|
||||
return e.fullPath()
|
||||
}
|
||||
|
||||
var _ os.FileInfo = (*filesystemEntry)(nil)
|
||||
|
||||
func newEntry(fi os.FileInfo, parentDir string) filesystemEntry {
|
||||
|
||||
@@ -69,6 +69,10 @@ func (e *entry) Device() fs.DeviceInfo {
|
||||
return e.device
|
||||
}
|
||||
|
||||
func (e *entry) LocalFilesystemPath() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// Directory is mock in-memory implementation of fs.Directory.
|
||||
type Directory struct {
|
||||
entry
|
||||
|
||||
@@ -778,7 +778,7 @@ func setupCaches(ctx context.Context, m *Manager, caching *CachingOptions) error
|
||||
}
|
||||
|
||||
if caching.ownWritesCache == nil {
|
||||
// this is test hook to allow test to specify custom cache
|
||||
// this is test action to allow test to specify custom cache
|
||||
caching.ownWritesCache, err = newOwnWritesCache(ctx, caching, m.timeNow)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "unable to initialize own writes cache")
|
||||
|
||||
46
snapshot/policy/actions_policy.go
Normal file
46
snapshot/policy/actions_policy.go
Normal file
@@ -0,0 +1,46 @@
|
||||
package policy
|
||||
|
||||
// ActionsPolicy describes actions to be invoked when taking snapshots.
|
||||
type ActionsPolicy struct {
|
||||
// command runs once before and after the folder it's attached to (not inherited).
|
||||
BeforeFolder *ActionCommand `json:"beforeFolder,omitempty"`
|
||||
AfterFolder *ActionCommand `json:"afterFolder,omitempty"`
|
||||
|
||||
// commands run once before and after each snapshot root (can be inherited).
|
||||
BeforeSnapshotRoot *ActionCommand `json:"beforeSnapshotRoot,omitempty"`
|
||||
AfterSnapshotRoot *ActionCommand `json:"afterSnapshotRoot,omitempty"`
|
||||
}
|
||||
|
||||
// ActionCommand configures a action command.
|
||||
type ActionCommand struct {
|
||||
// command + args to run
|
||||
Command string `json:"path,omitempty"`
|
||||
Arguments []string `json:"args,omitempty"`
|
||||
|
||||
// alternatively inline script to run using either Unix shell or cmd.exe on Windows.
|
||||
Script string `json:"script,omitempty"`
|
||||
|
||||
TimeoutSeconds int `json:"timeout,omitempty"`
|
||||
Mode string `json:"mode,omitempty"` // essential,optional,async
|
||||
}
|
||||
|
||||
// Merge applies default values from the provided policy.
|
||||
// nolint:gocritic
|
||||
func (p *ActionsPolicy) Merge(src ActionsPolicy) {
|
||||
if p.BeforeSnapshotRoot == nil {
|
||||
p.BeforeSnapshotRoot = src.BeforeSnapshotRoot
|
||||
}
|
||||
|
||||
if p.AfterSnapshotRoot == nil {
|
||||
p.AfterSnapshotRoot = src.AfterSnapshotRoot
|
||||
}
|
||||
}
|
||||
|
||||
// MergeNonInheritable copies non-inheritable properties from the provided actions policy.
|
||||
func (p *ActionsPolicy) MergeNonInheritable(src ActionsPolicy) {
|
||||
p.BeforeFolder = src.BeforeFolder
|
||||
p.AfterFolder = src.AfterFolder
|
||||
}
|
||||
|
||||
// defaultActionsPolicy is the default actions policy.
|
||||
var defaultActionsPolicy = ActionsPolicy{}
|
||||
@@ -19,6 +19,7 @@ type Policy struct {
|
||||
ErrorHandlingPolicy ErrorHandlingPolicy `json:"errorHandling,omitempty"`
|
||||
SchedulingPolicy SchedulingPolicy `json:"scheduling,omitempty"`
|
||||
CompressionPolicy CompressionPolicy `json:"compression,omitempty"`
|
||||
Actions ActionsPolicy `json:"actions"`
|
||||
NoParent bool `json:"noParent,omitempty"`
|
||||
}
|
||||
|
||||
@@ -63,6 +64,7 @@ func MergePolicies(policies []*Policy) *Policy {
|
||||
merged.ErrorHandlingPolicy.Merge(p.ErrorHandlingPolicy)
|
||||
merged.SchedulingPolicy.Merge(p.SchedulingPolicy)
|
||||
merged.CompressionPolicy.Merge(p.CompressionPolicy)
|
||||
merged.Actions.Merge(p.Actions)
|
||||
}
|
||||
|
||||
// Merge default expiration policy.
|
||||
@@ -71,6 +73,11 @@ func MergePolicies(policies []*Policy) *Policy {
|
||||
merged.ErrorHandlingPolicy.Merge(defaultErrorHandlingPolicy)
|
||||
merged.SchedulingPolicy.Merge(defaultSchedulingPolicy)
|
||||
merged.CompressionPolicy.Merge(defaultCompressionPolicy)
|
||||
merged.Actions.Merge(defaultActionsPolicy)
|
||||
|
||||
if len(policies) > 0 {
|
||||
merged.Actions.MergeNonInheritable(policies[0].Actions)
|
||||
}
|
||||
|
||||
return &merged
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package policy
|
||||
|
||||
import "strings"
|
||||
import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
// DefaultPolicy is a default policy returned by policy tree in absence of other policies.
|
||||
var DefaultPolicy = &Policy{
|
||||
@@ -9,6 +11,7 @@
|
||||
CompressionPolicy: defaultCompressionPolicy,
|
||||
ErrorHandlingPolicy: defaultErrorHandlingPolicy,
|
||||
SchedulingPolicy: defaultSchedulingPolicy,
|
||||
Actions: defaultActionsPolicy,
|
||||
}
|
||||
|
||||
// Tree represents a node in the policy tree, where a policy can be
|
||||
|
||||
@@ -47,6 +47,10 @@ func (s *repositoryAllSources) Sys() interface{} {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *repositoryAllSources) LocalFilesystemPath() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *repositoryAllSources) Child(ctx context.Context, name string) (fs.Entry, error) {
|
||||
return fs.ReadDirAndFindChild(ctx, s, name)
|
||||
}
|
||||
|
||||
@@ -79,6 +79,10 @@ func (e *repositoryEntry) DirEntry() *snapshot.DirEntry {
|
||||
return e.metadata
|
||||
}
|
||||
|
||||
func (e *repositoryEntry) LocalFilesystemPath() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
type repositoryDirectory struct {
|
||||
repositoryEntry
|
||||
summary *fs.DirectorySummary
|
||||
|
||||
@@ -47,6 +47,10 @@ func (s *sourceDirectories) Device() fs.DeviceInfo {
|
||||
return fs.DeviceInfo{}
|
||||
}
|
||||
|
||||
func (s *sourceDirectories) LocalFilesystemPath() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *sourceDirectories) Child(ctx context.Context, name string) (fs.Entry, error) {
|
||||
return fs.ReadDirAndFindChild(ctx, s, name)
|
||||
}
|
||||
|
||||
@@ -51,6 +51,10 @@ func (s *sourceSnapshots) Device() fs.DeviceInfo {
|
||||
return fs.DeviceInfo{}
|
||||
}
|
||||
|
||||
func (s *sourceSnapshots) LocalFilesystemPath() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func safeName(path string) string {
|
||||
path = strings.TrimLeft(path, "/")
|
||||
return strings.Replace(path, "/", "_", -1)
|
||||
|
||||
@@ -372,7 +372,7 @@ func (u *Uploader) periodicallyCheckpoint(ctx context.Context, cp *checkpointReg
|
||||
return
|
||||
}
|
||||
|
||||
// test hook
|
||||
// test action
|
||||
if u.checkpointFinished != nil {
|
||||
u.checkpointFinished <- struct{}{}
|
||||
}
|
||||
@@ -395,7 +395,22 @@ func (u *Uploader) uploadDirWithCheckpointing(ctx context.Context, rootDir fs.Di
|
||||
cancelCheckpointer := u.periodicallyCheckpoint(ctx, &cp, &snapshot.Manifest{Source: sourceInfo})
|
||||
defer cancelCheckpointer()
|
||||
|
||||
return uploadDirInternal(ctx, u, rootDir, policyTree, previousDirs, ".", &dmb, &cp)
|
||||
var hc actionContext
|
||||
|
||||
localDirPathOrEmpty := rootDir.LocalFilesystemPath()
|
||||
|
||||
overrideDir, err := executeBeforeFolderAction(ctx, "before-snapshot-root", policyTree.EffectivePolicy().Actions.BeforeSnapshotRoot, localDirPathOrEmpty, &hc)
|
||||
if err != nil {
|
||||
return nil, dirReadError{errors.Wrap(err, "error executing before-snapshot-root action")}
|
||||
}
|
||||
|
||||
if overrideDir != nil {
|
||||
rootDir = overrideDir
|
||||
}
|
||||
|
||||
defer executeAfterFolderAction(ctx, "after-snapshot-root", policyTree.EffectivePolicy().Actions.AfterSnapshotRoot, localDirPathOrEmpty, &hc)
|
||||
|
||||
return uploadDirInternal(ctx, u, rootDir, policyTree, previousDirs, localDirPathOrEmpty, ".", &dmb, &cp)
|
||||
}
|
||||
|
||||
func (u *Uploader) foreachEntryUnlessCanceled(ctx context.Context, parallel int, relativePath string, entries fs.Entries, cb func(ctx context.Context, entry fs.Entry, entryRelativePath string) error) error {
|
||||
@@ -563,8 +578,16 @@ func isDir(e *snapshot.DirEntry) bool {
|
||||
return e.Type == snapshot.EntryTypeDirectory
|
||||
}
|
||||
|
||||
func (u *Uploader) processChildren(ctx context.Context, parentDirCheckpointRegistry *checkpointRegistry, parentDirBuilder *dirManifestBuilder, relativePath string, entries fs.Entries, policyTree *policy.Tree, previousEntries []fs.Entries) error {
|
||||
if err := u.processSubdirectories(ctx, parentDirCheckpointRegistry, parentDirBuilder, relativePath, entries, policyTree, previousEntries); err != nil {
|
||||
func (u *Uploader) processChildren(
|
||||
ctx context.Context,
|
||||
parentDirCheckpointRegistry *checkpointRegistry,
|
||||
parentDirBuilder *dirManifestBuilder,
|
||||
localDirPathOrEmpty, relativePath string,
|
||||
entries fs.Entries,
|
||||
policyTree *policy.Tree,
|
||||
previousEntries []fs.Entries,
|
||||
) error {
|
||||
if err := u.processSubdirectories(ctx, parentDirCheckpointRegistry, parentDirBuilder, localDirPathOrEmpty, relativePath, entries, policyTree, previousEntries); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -575,7 +598,15 @@ func (u *Uploader) processChildren(ctx context.Context, parentDirCheckpointRegis
|
||||
return nil
|
||||
}
|
||||
|
||||
func (u *Uploader) processSubdirectories(ctx context.Context, parentDirCheckpointRegistry *checkpointRegistry, parentDirBuilder *dirManifestBuilder, relativePath string, entries fs.Entries, policyTree *policy.Tree, previousEntries []fs.Entries) error {
|
||||
func (u *Uploader) processSubdirectories(
|
||||
ctx context.Context,
|
||||
parentDirCheckpointRegistry *checkpointRegistry,
|
||||
parentDirBuilder *dirManifestBuilder,
|
||||
localDirPathOrEmpty, relativePath string,
|
||||
entries fs.Entries,
|
||||
policyTree *policy.Tree,
|
||||
previousEntries []fs.Entries,
|
||||
) error {
|
||||
// for now don't process subdirectories in parallel, we need a mechanism to
|
||||
// prevent explosion of parallelism
|
||||
const parallelism = 1
|
||||
@@ -598,7 +629,12 @@ func (u *Uploader) processSubdirectories(ctx context.Context, parentDirCheckpoin
|
||||
|
||||
childDirBuilder := &dirManifestBuilder{}
|
||||
|
||||
de, err := uploadDirInternal(ctx, u, dir, policyTree.Child(entry.Name()), previousDirs, entryRelativePath, childDirBuilder, parentDirCheckpointRegistry)
|
||||
childLocalDirPathOrEmpty := ""
|
||||
if localDirPathOrEmpty != "" {
|
||||
childLocalDirPathOrEmpty = filepath.Join(localDirPathOrEmpty, entry.Name())
|
||||
}
|
||||
|
||||
de, err := uploadDirInternal(ctx, u, dir, policyTree.Child(entry.Name()), previousDirs, childLocalDirPathOrEmpty, entryRelativePath, childDirBuilder, parentDirCheckpointRegistry)
|
||||
if errors.Is(err, errCanceled) {
|
||||
return err
|
||||
}
|
||||
@@ -795,7 +831,7 @@ func uploadDirInternal(
|
||||
directory fs.Directory,
|
||||
policyTree *policy.Tree,
|
||||
previousDirs []fs.Directory,
|
||||
dirRelativePath string,
|
||||
localDirPathOrEmpty, dirRelativePath string,
|
||||
thisDirBuilder *dirManifestBuilder,
|
||||
thisCheckpointRegistry *checkpointRegistry,
|
||||
) (*snapshot.DirEntry, error) {
|
||||
@@ -804,6 +840,26 @@ func uploadDirInternal(
|
||||
u.Progress.StartedDirectory(dirRelativePath)
|
||||
defer u.Progress.FinishedDirectory(dirRelativePath)
|
||||
|
||||
var definedActions policy.ActionsPolicy
|
||||
|
||||
if p := policyTree.DefinedPolicy(); p != nil {
|
||||
definedActions = p.Actions
|
||||
}
|
||||
|
||||
var hc actionContext
|
||||
defer cleanupActionContext(ctx, &hc)
|
||||
|
||||
overrideDir, herr := executeBeforeFolderAction(ctx, "before-folder", definedActions.BeforeFolder, localDirPathOrEmpty, &hc)
|
||||
if herr != nil {
|
||||
return nil, dirReadError{errors.Wrap(herr, "error executing before-folder action")}
|
||||
}
|
||||
|
||||
defer executeAfterFolderAction(ctx, "after-folder", definedActions.AfterFolder, localDirPathOrEmpty, &hc)
|
||||
|
||||
if overrideDir != nil {
|
||||
directory = overrideDir
|
||||
}
|
||||
|
||||
t0 := u.repo.Time()
|
||||
entries, direrr := directory.Readdir(ctx)
|
||||
log(ctx).Debugf("finished reading directory %v in %v", dirRelativePath, u.repo.Time().Sub(t0))
|
||||
@@ -842,7 +898,7 @@ func uploadDirInternal(
|
||||
})
|
||||
defer thisCheckpointRegistry.removeCheckpointCallback(directory)
|
||||
|
||||
if err := u.processChildren(ctx, childCheckpointRegistry, thisDirBuilder, dirRelativePath, entries, policyTree, prevEntries); err != nil && !errors.Is(err, errCanceled) {
|
||||
if err := u.processChildren(ctx, childCheckpointRegistry, thisDirBuilder, localDirPathOrEmpty, dirRelativePath, entries, policyTree, prevEntries); err != nil && !errors.Is(err, errCanceled) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
||||
242
snapshot/snapshotfs/upload_actions.go
Normal file
242
snapshot/snapshotfs/upload_actions.go
Normal file
@@ -0,0 +1,242 @@
|
||||
package snapshotfs
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/kopia/kopia/fs"
|
||||
"github.com/kopia/kopia/fs/localfs"
|
||||
"github.com/kopia/kopia/snapshot/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
actionCommandTimeout = 3 * time.Minute
|
||||
actionScriptPermissions = 0o700
|
||||
)
|
||||
|
||||
// actionContext carries state between before/after actions.
|
||||
type actionContext struct {
|
||||
ActionsEnabled bool
|
||||
SnapshotID string
|
||||
SourcePath string
|
||||
SnapshotPath string
|
||||
WorkDir string
|
||||
}
|
||||
|
||||
func (hc *actionContext) envars() []string {
|
||||
return []string{
|
||||
fmt.Sprintf("KOPIA_SNAPSHOT_ID=%v", hc.SnapshotID),
|
||||
fmt.Sprintf("KOPIA_SOURCE_PATH=%v", hc.SourcePath),
|
||||
fmt.Sprintf("KOPIA_SNAPSHOT_PATH=%v", hc.SnapshotPath),
|
||||
}
|
||||
}
|
||||
|
||||
func (hc *actionContext) ensureInitialized(dirPathOrEmpty string) error {
|
||||
if dirPathOrEmpty == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
if hc.ActionsEnabled {
|
||||
// already initialized
|
||||
return nil
|
||||
}
|
||||
|
||||
var randBytes [8]byte
|
||||
|
||||
if _, err := rand.Read(randBytes[:]); err != nil {
|
||||
return errors.Wrap(err, "error reading random bytes")
|
||||
}
|
||||
|
||||
hc.SnapshotID = fmt.Sprintf("%x", randBytes[:])
|
||||
hc.SourcePath = dirPathOrEmpty
|
||||
hc.SnapshotPath = hc.SourcePath
|
||||
|
||||
wd, err := ioutil.TempDir("", "kopia-action")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hc.WorkDir = wd
|
||||
hc.ActionsEnabled = true
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func actionScriptExtension() string {
|
||||
if runtime.GOOS == "windows" {
|
||||
return ".cmd"
|
||||
}
|
||||
|
||||
return ".sh"
|
||||
}
|
||||
|
||||
// prepareCommandForAction prepares *exec.Cmd that will run the provided action command in the provided
|
||||
// working directory.
|
||||
func prepareCommandForAction(ctx context.Context, actionType string, h *policy.ActionCommand, workDir string) (*exec.Cmd, context.CancelFunc, error) {
|
||||
timeout := actionCommandTimeout
|
||||
if h.TimeoutSeconds != 0 {
|
||||
timeout = time.Duration(h.TimeoutSeconds) * time.Second
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
|
||||
var c *exec.Cmd
|
||||
|
||||
switch {
|
||||
case h.Script != "":
|
||||
scriptFile := filepath.Join(workDir, actionType+actionScriptExtension())
|
||||
if err := ioutil.WriteFile(scriptFile, []byte(h.Script), actionScriptPermissions); err != nil {
|
||||
cancel()
|
||||
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
switch {
|
||||
case runtime.GOOS == "windows":
|
||||
c = exec.CommandContext(ctx, "cmd.exe", "/c", scriptFile) // nolint:gosec
|
||||
case strings.HasPrefix(h.Script, "#!"):
|
||||
// on unix if a script starts with #!, it will run under designated interpreter
|
||||
c = exec.CommandContext(ctx, scriptFile) // nolint:gosec
|
||||
default:
|
||||
c = exec.CommandContext(ctx, "sh", "-e", scriptFile) // nolint:gosec
|
||||
}
|
||||
|
||||
case h.Command != "":
|
||||
c = exec.CommandContext(ctx, h.Command, h.Arguments...) // nolint:gosec
|
||||
|
||||
default:
|
||||
cancel()
|
||||
|
||||
return nil, nil, errors.Errorf("action did not provide either script nor command to run")
|
||||
}
|
||||
|
||||
// all actions run inside temporary working directory
|
||||
c.Dir = workDir
|
||||
|
||||
return c, cancel, nil
|
||||
}
|
||||
|
||||
// runActionCommand executes the action command passing the provided inputs as environment
|
||||
// variables. It analyzes the standard output of the command looking for 'key=value'
|
||||
// where the key is present in the provided outputs map and sets the corresponding map value.
|
||||
func runActionCommand(
|
||||
ctx context.Context,
|
||||
actionType string,
|
||||
h *policy.ActionCommand,
|
||||
inputs []string,
|
||||
captures map[string]string,
|
||||
workDir string,
|
||||
) error {
|
||||
cmd, cancel, err := prepareCommandForAction(ctx, actionType, h, workDir)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error preparing command")
|
||||
}
|
||||
|
||||
defer cancel()
|
||||
|
||||
cmd.Env = append(os.Environ(), inputs...)
|
||||
cmd.Stderr = os.Stderr
|
||||
|
||||
if h.Mode == "async" {
|
||||
return cmd.Start()
|
||||
}
|
||||
|
||||
v, err := cmd.Output()
|
||||
if err != nil {
|
||||
if h.Mode == "essential" {
|
||||
return err
|
||||
}
|
||||
|
||||
log(ctx).Warningf("error running non-essential action command: %v", err)
|
||||
}
|
||||
|
||||
return parseCaptures(v, captures)
|
||||
}
|
||||
|
||||
// parseCaptures analyzes given byte array and updated the provided map values whenever
|
||||
// map keys match lines inside the byte array. The lines must be formatted as k=v.
|
||||
func parseCaptures(v []byte, captures map[string]string) error {
|
||||
s := bufio.NewScanner(bytes.NewReader(v))
|
||||
for s.Scan() {
|
||||
l := strings.SplitN(s.Text(), "=", 2)
|
||||
if len(l) <= 1 {
|
||||
continue
|
||||
}
|
||||
|
||||
key, value := l[0], l[1]
|
||||
if _, ok := captures[key]; ok {
|
||||
captures[key] = value
|
||||
}
|
||||
}
|
||||
|
||||
return s.Err()
|
||||
}
|
||||
|
||||
func executeBeforeFolderAction(ctx context.Context, actionType string, h *policy.ActionCommand, dirPathOrEmpty string, hc *actionContext) (fs.Directory, error) {
|
||||
if h == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if err := hc.ensureInitialized(dirPathOrEmpty); err != nil {
|
||||
return nil, errors.Wrap(err, "error initializing action context")
|
||||
}
|
||||
|
||||
if !hc.ActionsEnabled {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
log(ctx).Debugf("running action %v on %v %#v", actionType, hc.SourcePath, *h)
|
||||
|
||||
captures := map[string]string{
|
||||
"KOPIA_SNAPSHOT_PATH": "",
|
||||
}
|
||||
|
||||
if err := runActionCommand(ctx, actionType, h, hc.envars(), captures, hc.WorkDir); err != nil {
|
||||
return nil, errors.Wrapf(err, "error running '%v' action", actionType)
|
||||
}
|
||||
|
||||
if p := captures["KOPIA_SNAPSHOT_PATH"]; p != "" {
|
||||
hc.SnapshotPath = p
|
||||
return localfs.Directory(hc.SnapshotPath)
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func executeAfterFolderAction(ctx context.Context, actionType string, h *policy.ActionCommand, dirPathOrEmpty string, hc *actionContext) {
|
||||
if h == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err := hc.ensureInitialized(dirPathOrEmpty); err != nil {
|
||||
log(ctx).Warningf("error initializing action context: %v", err)
|
||||
}
|
||||
|
||||
if !hc.ActionsEnabled {
|
||||
return
|
||||
}
|
||||
|
||||
if err := runActionCommand(ctx, actionType, h, hc.envars(), nil, hc.WorkDir); err != nil {
|
||||
log(ctx).Warningf("error running '%v' action: %v", actionType, err)
|
||||
}
|
||||
}
|
||||
|
||||
func cleanupActionContext(ctx context.Context, hc *actionContext) {
|
||||
if hc.WorkDir != "" {
|
||||
if err := os.RemoveAll(hc.WorkDir); err != nil {
|
||||
log(ctx).Debugf("unable to remove action working directory: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -309,7 +309,7 @@ func TestUploadWithCheckpointing(t *testing.T) {
|
||||
Path: "path",
|
||||
}
|
||||
|
||||
// inject a hook into mock filesystem to trigger and wait for checkpoints at few places.
|
||||
// inject a action into mock filesystem to trigger and wait for checkpoints at few places.
|
||||
// the places are not important, what's important that those are 3 separate points in time.
|
||||
dirsToCheckpointAt := []*mockfs.Directory{
|
||||
th.sourceDir.Subdir("d1"),
|
||||
|
||||
317
tests/end_to_end_test/snapshot_actions_test.go
Normal file
317
tests/end_to_end_test/snapshot_actions_test.go
Normal file
@@ -0,0 +1,317 @@
|
||||
package endtoend_test
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/kopia/kopia/tests/testenv"
|
||||
)
|
||||
|
||||
func TestSnapshotActionsBeforeSnapshotRoot(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
th := os.Getenv("TESTING_ACTION_EXE")
|
||||
if th == "" {
|
||||
t.Skip("TESTING_ACTION_EXE verifyNoError be set")
|
||||
}
|
||||
|
||||
e := testenv.NewCLITest(t)
|
||||
|
||||
defer e.RunAndExpectSuccess(t, "repo", "disconnect")
|
||||
|
||||
e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir, "--override-hostname=foo", "--override-username=foo")
|
||||
e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir2)
|
||||
|
||||
envFile1 := filepath.Join(e.LogsDir, "env1.txt")
|
||||
|
||||
// set a action before-snapshot-root that fails and which saves the environment to a file.
|
||||
e.RunAndExpectSuccess(t,
|
||||
"policy", "set", sharedTestDataDir1,
|
||||
"--before-snapshot-root-action",
|
||||
th+" --exit-code=3 --save-env="+envFile1)
|
||||
|
||||
// this prevents the snapshot from being created
|
||||
e.RunAndExpectFailure(t, "snapshot", "create", sharedTestDataDir1)
|
||||
|
||||
envFile2 := filepath.Join(e.LogsDir, "env2.txt")
|
||||
|
||||
// now set a action before-snapshot-root that succeeds and saves environment to a different file
|
||||
e.RunAndExpectSuccess(t,
|
||||
"policy", "set", sharedTestDataDir1,
|
||||
"--before-snapshot-root-action",
|
||||
th+" --save-env="+envFile2)
|
||||
|
||||
// snapshot now succeeds.
|
||||
e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1)
|
||||
|
||||
env1 := mustReadEnvFile(t, envFile1)
|
||||
env2 := mustReadEnvFile(t, envFile2)
|
||||
|
||||
// make sure snapshot IDs are different between two attempts
|
||||
if id1, id2 := env1["KOPIA_SNAPSHOT_ID"], env2["KOPIA_SNAPSHOT_ID"]; id1 == id2 {
|
||||
t.Errorf("KOPIA_SNAPSHOT_ID passed to action was not different between runs %v", id1)
|
||||
}
|
||||
|
||||
// Now set up the action again, in optional mode,
|
||||
e.RunAndExpectSuccess(t,
|
||||
"policy", "set", sharedTestDataDir1,
|
||||
"--before-snapshot-root-action",
|
||||
th+" --exit-code=3",
|
||||
"--action-command-mode=optional")
|
||||
|
||||
// this will not prevent snapshot creation.
|
||||
e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1)
|
||||
|
||||
// Now set up the action again, in async mode and pass --sleep so that the command takes some time.
|
||||
// because the action is async it will not wait for the command.
|
||||
e.RunAndExpectSuccess(t,
|
||||
"policy", "set", sharedTestDataDir1,
|
||||
"--before-snapshot-root-action",
|
||||
th+" --exit-code=3 --sleep=30s",
|
||||
"--action-command-mode=async")
|
||||
|
||||
t0 := time.Now()
|
||||
|
||||
// at this point the data is all cached so this will be quick, definitely less than 30s,
|
||||
// async action failure will not prevent snapshot success.
|
||||
e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1)
|
||||
|
||||
if dur := time.Since(t0); dur > 30*time.Second {
|
||||
t.Errorf("command did not execute asynchronously (took %v)", dur)
|
||||
}
|
||||
|
||||
// Now set up essential action with a timeout of 3s and have the action sleep for 30s
|
||||
e.RunAndExpectSuccess(t,
|
||||
"policy", "set", sharedTestDataDir1,
|
||||
"--before-snapshot-root-action",
|
||||
th+" --sleep=30s",
|
||||
"--action-command-timeout=3s")
|
||||
|
||||
t0 = time.Now()
|
||||
|
||||
// the action will be killed after 3s and cause a failure.
|
||||
e.RunAndExpectFailure(t, "snapshot", "create", sharedTestDataDir1)
|
||||
|
||||
if dur := time.Since(t0); dur > 30*time.Second {
|
||||
t.Errorf("command did not apply timeout (took %v)", dur)
|
||||
}
|
||||
|
||||
// Now set up essential action that will cause redirection to an alternative folder which does not exist.
|
||||
e.RunAndExpectSuccess(t,
|
||||
"policy", "set", sharedTestDataDir1,
|
||||
"--before-snapshot-root-action",
|
||||
th+" --stdout-file="+tmpfileWithContents(t, "KOPIA_SNAPSHOT_PATH=/no/such/directory\n"))
|
||||
|
||||
e.RunAndExpectFailure(t, "snapshot", "create", sharedTestDataDir1)
|
||||
|
||||
// Now set up essential action that will cause redirection to an alternative folder which does exist.
|
||||
e.RunAndExpectSuccess(t,
|
||||
"policy", "set", sharedTestDataDir1,
|
||||
"--before-snapshot-root-action",
|
||||
th+" --stdout-file="+tmpfileWithContents(t, "KOPIA_SNAPSHOT_PATH="+sharedTestDataDir2+"\n"))
|
||||
|
||||
e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1)
|
||||
|
||||
// since we redirected to sharedTestDataDir2 the object ID of last snapshot of sharedTestDataDir1
|
||||
// will be the same as snapshots of sharedTestDataDir2
|
||||
snaps1 := e.ListSnapshotsAndExpectSuccess(t, sharedTestDataDir1)[0].Snapshots
|
||||
snaps2 := e.ListSnapshotsAndExpectSuccess(t, sharedTestDataDir2)[0].Snapshots
|
||||
|
||||
if snaps1[0].ObjectID == snaps2[0].ObjectID {
|
||||
t.Fatal("failed sanity check - snapshots are the same")
|
||||
}
|
||||
|
||||
if got, want := snaps1[len(snaps1)-1].ObjectID, snaps2[0].ObjectID; got != want {
|
||||
t.Fatalf("invalid snapshot ID after redirection %v, wanted %v", got, want)
|
||||
}
|
||||
|
||||
// not setup the same redirection but in async mode - will be ignored because Kopia does not wait for asynchronous
|
||||
// actions at all or parse their output.
|
||||
e.RunAndExpectSuccess(t,
|
||||
"policy", "set", sharedTestDataDir1,
|
||||
"--before-snapshot-root-action",
|
||||
th+" --stdout-file="+tmpfileWithContents(t, "KOPIA_SNAPSHOT_PATH="+sharedTestDataDir2+"\n"),
|
||||
"--action-command-mode=async")
|
||||
e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1)
|
||||
|
||||
// verify redirection had no effect - last snapshot will be the same as the first one
|
||||
snaps1 = e.ListSnapshotsAndExpectSuccess(t, sharedTestDataDir1)[0].Snapshots
|
||||
if got, want := snaps1[len(snaps1)-1].ObjectID, snaps1[0].ObjectID; got != want {
|
||||
t.Fatalf("invalid snapshot ID after async action %v, wanted %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshotActionsBeforeAfterFolder(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
th := os.Getenv("TESTING_ACTION_EXE")
|
||||
if th == "" {
|
||||
t.Skip("TESTING_ACTION_EXE verifyNoError be set")
|
||||
}
|
||||
|
||||
e := testenv.NewCLITest(t)
|
||||
|
||||
e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
|
||||
defer e.RunAndExpectSuccess(t, "repo", "disconnect")
|
||||
|
||||
// create directory structure
|
||||
rootDir := t.TempDir()
|
||||
sd1 := filepath.Join(rootDir, "subdir1")
|
||||
sd2 := filepath.Join(rootDir, "subdir2")
|
||||
sd11 := filepath.Join(rootDir, "subdir1", "subdir1")
|
||||
sd12 := filepath.Join(rootDir, "subdir1", "subdir2")
|
||||
|
||||
verifyNoError(t, os.Mkdir(sd1, 0700))
|
||||
verifyNoError(t, os.Mkdir(sd2, 0700))
|
||||
verifyNoError(t, os.Mkdir(sd11, 0700))
|
||||
verifyNoError(t, os.Mkdir(sd12, 0700))
|
||||
|
||||
actionRanDir := t.TempDir()
|
||||
|
||||
actionRanFileBeforeRoot := filepath.Join(actionRanDir, "before-root")
|
||||
actionRanFileAfterRoot := filepath.Join(actionRanDir, "before-root")
|
||||
actionRanFileBeforeSD1 := filepath.Join(actionRanDir, "before-sd1")
|
||||
actionRanFileAfterSD1 := filepath.Join(actionRanDir, "before-sd1")
|
||||
actionRanFileBeforeSD11 := filepath.Join(actionRanDir, "before-sd11")
|
||||
actionRanFileAfterSD11 := filepath.Join(actionRanDir, "before-sd11")
|
||||
actionRanFileBeforeSD2 := filepath.Join(actionRanDir, "before-sd2")
|
||||
actionRanFileAfterSD2 := filepath.Join(actionRanDir, "before-sd2")
|
||||
|
||||
// setup actions that will write a marker file when the action is executed.
|
||||
//
|
||||
// We are not setting a policy on 'sd12' to ensure it's not inherited
|
||||
// from sd1. If it was inherited, the action would fail since it refuses to create the
|
||||
// file if one already exists.
|
||||
e.RunAndExpectSuccess(t, "policy", "set", rootDir,
|
||||
"--before-folder-action", th+" --create-file="+actionRanFileBeforeRoot)
|
||||
e.RunAndExpectSuccess(t, "policy", "set", rootDir,
|
||||
"--after-folder-action", th+" --create-file="+actionRanFileAfterRoot)
|
||||
e.RunAndExpectSuccess(t, "policy", "set", sd1,
|
||||
"--before-folder-action", th+" --create-file="+actionRanFileBeforeSD1)
|
||||
e.RunAndExpectSuccess(t, "policy", "set", sd1,
|
||||
"--after-folder-action", th+" --create-file="+actionRanFileAfterSD1)
|
||||
e.RunAndExpectSuccess(t, "policy", "set", sd2,
|
||||
"--before-folder-action", th+" --create-file="+actionRanFileBeforeSD2)
|
||||
e.RunAndExpectSuccess(t, "policy", "set", sd2,
|
||||
"--after-folder-action", th+" --create-file="+actionRanFileAfterSD2)
|
||||
e.RunAndExpectSuccess(t, "policy", "set", sd11,
|
||||
"--before-folder-action", th+" --create-file="+actionRanFileBeforeSD11)
|
||||
e.RunAndExpectSuccess(t, "policy", "set", sd11,
|
||||
"--after-folder-action", th+" --create-file="+actionRanFileAfterSD11)
|
||||
|
||||
e.RunAndExpectSuccess(t, "snapshot", "create", rootDir)
|
||||
|
||||
verifyFileExists(t, actionRanFileBeforeRoot)
|
||||
verifyFileExists(t, actionRanFileAfterRoot)
|
||||
verifyFileExists(t, actionRanFileBeforeSD1)
|
||||
verifyFileExists(t, actionRanFileBeforeSD11)
|
||||
verifyFileExists(t, actionRanFileAfterSD11)
|
||||
verifyFileExists(t, actionRanFileAfterSD1)
|
||||
verifyFileExists(t, actionRanFileBeforeSD2)
|
||||
verifyFileExists(t, actionRanFileAfterSD2)
|
||||
|
||||
// the action will fail to run the next time since all 'actionRan*' files already exist.
|
||||
e.RunAndExpectFailure(t, "snapshot", "create", rootDir)
|
||||
}
|
||||
|
||||
func TestSnapshotActionsEmbeddedScript(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
e := testenv.NewCLITest(t)
|
||||
|
||||
e.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", e.RepoDir)
|
||||
defer e.RunAndExpectSuccess(t, "repo", "disconnect")
|
||||
|
||||
var (
|
||||
successScript = tmpfileWithContents(t, "echo Hello world!")
|
||||
successScript2 string
|
||||
failingScript string
|
||||
goodRedirectScript = tmpfileWithContents(t, "echo KOPIA_SNAPSHOT_PATH="+sharedTestDataDir2)
|
||||
badRedirectScript = tmpfileWithContents(t, "echo KOPIA_SNAPSHOT_PATH=/no/such/directory")
|
||||
)
|
||||
|
||||
if runtime.GOOS == "windows" {
|
||||
failingScript = tmpfileWithContents(t, "exit /b 1")
|
||||
successScript2 = tmpfileWithContents(t, "echo Hello world!")
|
||||
} else {
|
||||
failingScript = tmpfileWithContents(t, "#!/bin/sh\nexit 1")
|
||||
successScript2 = tmpfileWithContents(t, "#!/bin/sh\necho Hello world!")
|
||||
}
|
||||
|
||||
e.RunAndExpectSuccess(t, "policy", "set", sharedTestDataDir1, "--before-folder-action", successScript, "--persist-action-script")
|
||||
e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1)
|
||||
|
||||
e.RunAndExpectSuccess(t, "policy", "set", sharedTestDataDir1, "--before-folder-action", goodRedirectScript, "--persist-action-script")
|
||||
e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1)
|
||||
|
||||
e.RunAndExpectSuccess(t, "policy", "set", sharedTestDataDir1, "--before-folder-action", successScript2, "--persist-action-script")
|
||||
e.RunAndExpectSuccess(t, "snapshot", "create", sharedTestDataDir1)
|
||||
|
||||
snaps1 := e.ListSnapshotsAndExpectSuccess(t, sharedTestDataDir1)[0].Snapshots
|
||||
if snaps1[0].ObjectID == snaps1[1].ObjectID {
|
||||
t.Fatalf("redirection did not happen!")
|
||||
}
|
||||
|
||||
e.RunAndExpectSuccess(t, "policy", "set", sharedTestDataDir1, "--before-folder-action", badRedirectScript, "--persist-action-script")
|
||||
e.RunAndExpectFailure(t, "snapshot", "create", sharedTestDataDir1)
|
||||
|
||||
e.RunAndExpectSuccess(t, "policy", "set", sharedTestDataDir1, "--before-folder-action", failingScript, "--persist-action-script")
|
||||
e.RunAndExpectFailure(t, "snapshot", "create", sharedTestDataDir1)
|
||||
}
|
||||
|
||||
func tmpfileWithContents(t *testing.T, contents string) string {
|
||||
f, err := ioutil.TempFile("", "kopia-test")
|
||||
verifyNoError(t, err)
|
||||
|
||||
f.WriteString(contents)
|
||||
f.Close()
|
||||
|
||||
t.Cleanup(func() { os.Remove(f.Name()) })
|
||||
|
||||
return f.Name()
|
||||
}
|
||||
|
||||
func verifyFileExists(t *testing.T, fname string) {
|
||||
t.Helper()
|
||||
|
||||
_, err := os.Stat(fname)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func verifyNoError(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func mustReadEnvFile(t *testing.T, fname string) map[string]string {
|
||||
f, err := os.Open(fname)
|
||||
|
||||
verifyNoError(t, err)
|
||||
|
||||
defer f.Close()
|
||||
s := bufio.NewScanner(f)
|
||||
|
||||
m := map[string]string{}
|
||||
|
||||
for s.Scan() {
|
||||
parts := strings.SplitN(s.Text(), "=", 2)
|
||||
if len(parts) == 2 {
|
||||
m[parts[0]] = parts[1]
|
||||
}
|
||||
}
|
||||
|
||||
verifyNoError(t, s.Err())
|
||||
|
||||
return m
|
||||
}
|
||||
115
tests/testingaction/main.go
Normal file
115
tests/testingaction/main.go
Normal file
@@ -0,0 +1,115 @@
|
||||
// Command testingaction implements a action that is used in various tests.
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"flag"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
var (
|
||||
exitCode = flag.Int("exit-code", 0, "Exit code")
|
||||
sleepDuration = flag.Duration("sleep", 0, "Sleep duration")
|
||||
saveEnvironmentToFile = flag.String("save-env", "", "Save environment to file (key=value).")
|
||||
copyFilesSpec = flag.String("copy-files", "", "Copy files based on spec in the provided file (each line containing 'source => destination')")
|
||||
createFile = flag.String("create-file", "", "Create empty file with a given name")
|
||||
writeToStdout = flag.String("stdout-file", "", "Copy contents of the provided file to stdout.")
|
||||
writeToStderr = flag.String("stderr-file", "", "Copy contents of the provided file to stderr.")
|
||||
)
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
|
||||
if fn := *saveEnvironmentToFile; fn != "" {
|
||||
if err := ioutil.WriteFile(fn, []byte(strings.Join(os.Environ(), "\n")), 0600); err != nil {
|
||||
log.Fatalf("error writing environment file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if fn := *writeToStdout; fn != "" {
|
||||
if err := writeFileTo(os.Stdout, fn); err != nil {
|
||||
log.Fatalf("error writing to stdout: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if fn := *writeToStderr; fn != "" {
|
||||
if err := writeFileTo(os.Stderr, fn); err != nil {
|
||||
log.Fatalf("error writing to stderr: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if fn := *copyFilesSpec; fn != "" {
|
||||
if err := copyFiles(fn); err != nil {
|
||||
log.Fatalf("unable to copy files: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if fn := *createFile; fn != "" {
|
||||
if _, err := os.Stat(fn); !os.IsNotExist(err) {
|
||||
log.Fatalf("unexpected file found: %v", fn)
|
||||
}
|
||||
|
||||
if err := ioutil.WriteFile(fn, nil, 0600); err != nil {
|
||||
log.Fatalf("unable to create file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(*sleepDuration)
|
||||
os.Exit(*exitCode)
|
||||
}
|
||||
|
||||
func writeFileTo(dst io.Writer, fn string) error {
|
||||
f, err := os.Open(fn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer f.Close()
|
||||
|
||||
io.Copy(dst, f)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyFiles(specFile string) error {
|
||||
f, err := os.Open(specFile)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "unable to open spec file")
|
||||
}
|
||||
|
||||
defer f.Close()
|
||||
|
||||
s := bufio.NewScanner(f)
|
||||
for s.Scan() {
|
||||
parts := strings.Split(s.Text(), " => ")
|
||||
if len(parts) != 2 {
|
||||
continue
|
||||
}
|
||||
|
||||
src := os.ExpandEnv(parts[0])
|
||||
dst := os.ExpandEnv(parts[1])
|
||||
|
||||
if err := copyFile(src, dst); err != nil {
|
||||
return errors.Wrap(err, "copy file error")
|
||||
}
|
||||
}
|
||||
|
||||
return s.Err()
|
||||
}
|
||||
|
||||
func copyFile(src, dst string) error {
|
||||
df, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer df.Close()
|
||||
|
||||
return writeFileTo(df, src)
|
||||
}
|
||||
Reference in New Issue
Block a user