diff --git a/cli/command_policy_set.go b/cli/command_policy_set.go index 9cdf1a5ed..161a1166d 100644 --- a/cli/command_policy_set.go +++ b/cli/command_policy_set.go @@ -37,6 +37,8 @@ policySetRemoveIgnore = policySetCommand.Flag("remove-ignore", "List of paths to remove from the ignore list").PlaceHolder("PATTERN").Strings() policySetClearIgnore = policySetCommand.Flag("clear-ignore", "Clear list of paths in the ignore list").Bool() + policyIgnoreCacheDirs = policySetCommand.Flag("ignore-cache-dirs", "Ignore cache directories ('true', 'false', 'inherit')").Enum(booleanEnumValues...) + // Name of compression algorithm. policySetCompressionAlgorithm = policySetCommand.Flag("compression", "Compression algorithm").Enum(supportedCompressionAlgorithms()...) policySetCompressionMinSize = policySetCommand.Flag("compression-min-size", "Min size of file to attempt compression for").String() @@ -114,7 +116,9 @@ func setPolicyFromFlags(p *policy.Policy, changeCount *int) error { return errors.Wrap(err, "retention policy") } - setFilesPolicyFromFlags(&p.FilesPolicy, changeCount) + if err := setFilesPolicyFromFlags(&p.FilesPolicy, changeCount); err != nil { + return errors.Wrap(err, "files policy") + } if err := setErrorHandlingPolicyFromFlags(&p.ErrorHandlingPolicy, changeCount); err != nil { return errors.Wrap(err, "error handling policy") @@ -142,7 +146,7 @@ func setPolicyFromFlags(p *policy.Policy, changeCount *int) error { return nil } -func setFilesPolicyFromFlags(fp *policy.FilesPolicy, changeCount *int) { +func setFilesPolicyFromFlags(fp *policy.FilesPolicy, changeCount *int) error { if *policySetClearDotIgnore { *changeCount++ @@ -162,6 +166,30 @@ func setFilesPolicyFromFlags(fp *policy.FilesPolicy, changeCount *int) { } else { fp.IgnoreRules = addRemoveDedupeAndSort("ignored files", fp.IgnoreRules, *policySetAddIgnore, *policySetRemoveIgnore, changeCount) } + + switch { + case *policyIgnoreCacheDirs == "": + case *policyIgnoreCacheDirs == inheritPolicyString: + *changeCount++ + + fp.IgnoreCacheDirs = nil + + printStderr(" - inherit ignoring cache dirs from parent\n") + + default: + val, err := strconv.ParseBool(*policyIgnoreCacheDirs) + if err != nil { + return err + } + + *changeCount++ + + fp.IgnoreCacheDirs = &val + + printStderr(" - setting ignore cache dirs to %v\n", val) + } + + return nil } func setErrorHandlingPolicyFromFlags(fp *policy.ErrorHandlingPolicy, changeCount *int) error { diff --git a/cli/command_policy_show.go b/cli/command_policy_show.go index 3ca78eaf8..debc637a2 100644 --- a/cli/command_policy_show.go +++ b/cli/command_policy_show.go @@ -123,6 +123,12 @@ func printRetentionPolicy(p *policy.Policy, parents []*policy.Policy) { func printFilesPolicy(p *policy.Policy, parents []*policy.Policy) { printStdout("Files policy:\n") + printStdout(" Ignore cache directories: %5v %v\n", + p.FilesPolicy.IgnoreCacheDirectoriesOrDefault(true), + getDefinitionPoint(parents, func(pol *policy.Policy) bool { + return pol.FilesPolicy.IgnoreCacheDirs != nil + })) + if len(p.FilesPolicy.IgnoreRules) > 0 { printStdout(" Ignore rules:\n") } else { diff --git a/cli/command_snapshot_estimate.go b/cli/command_snapshot_estimate.go index 44752894e..90b76e153 100644 --- a/cli/command_snapshot_estimate.go +++ b/cli/command_snapshot_estimate.go @@ -82,12 +82,14 @@ func runSnapshotEstimateCommand(ctx context.Context, rep repo.Repository) error eb := makeBuckets() onIgnoredFile := func(relativePath string, e fs.Entry) { - log(ctx).Infof("ignoring %v", relativePath) eb.add(relativePath, e.Size()) if e.IsDir() { stats.ExcludedDirCount++ + + log(ctx).Infof("excluded dir %v", relativePath) } else { + log(ctx).Infof("excluded file %v (%v)", relativePath, units.BytesStringBase10(e.Size())) stats.ExcludedFileCount++ stats.ExcludedTotalFileSize += e.Size() } diff --git a/fs/ignorefs/ignorefs.go b/fs/ignorefs/ignorefs.go index 4e323aed4..748d8c2e3 100644 --- a/fs/ignorefs/ignorefs.go +++ b/fs/ignorefs/ignorefs.go @@ -10,9 +10,13 @@ "github.com/kopia/kopia/fs" "github.com/kopia/kopia/internal/ignore" + "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/repo/logging" "github.com/kopia/kopia/snapshot/policy" ) +var log = logging.GetContextLoggerFunc("ignorefs") + // IgnoreCallback is a function called by ignorefs to report whenever a file or directory is being ignored while listing its parent. type IgnoreCallback func(path string, metadata fs.Entry) @@ -52,12 +56,66 @@ type ignoreDirectory struct { fs.Directory } +func isCorrectCacheDirSignature(ctx context.Context, f fs.File) (bool, error) { + const ( + validSignature = repo.CacheDirMarkerHeader + validSignatureLen = len(validSignature) + ) + + if f.Size() < int64(validSignatureLen) { + return false, nil + } + + r, err := f.Open(ctx) + if err != nil { + return false, err + } + + defer r.Close() //nolint:errcheck + + sig := make([]byte, validSignatureLen) + + if _, err := r.Read(sig); err != nil { + return false, err + } + + return string(sig) == validSignature, nil +} + +func (d *ignoreDirectory) skipCacheDirectory(ctx context.Context, entries fs.Entries, relativePath string, policyTree *policy.Tree) fs.Entries { + if !policyTree.EffectivePolicy().FilesPolicy.IgnoreCacheDirectoriesOrDefault(true) { + return entries + } + + f, ok := entries.FindByName(repo.CacheDirMarkerFile).(fs.File) + if ok { + correct, err := isCorrectCacheDirSignature(ctx, f) + if err != nil { + log(ctx).Debugf("unable to check cache dir signature, assuming not a cache directory: %v", err) + return entries + } + + if correct { + // if the given directory contains a marker file used for kopia cache, pretend the directory was empty. + for _, oi := range d.parentContext.onIgnore { + oi(relativePath, d) + } + + return nil + } + } + + return entries +} + func (d *ignoreDirectory) Readdir(ctx context.Context) (fs.Entries, error) { entries, err := d.Directory.Readdir(ctx) if err != nil { return nil, err } + entries = d.skipCacheDirectory(ctx, entries, d.relativePath, d.policyTree) + thisContext, err := d.buildContext(ctx, entries) if err != nil { return nil, err @@ -125,6 +183,7 @@ func (d *ignoreDirectory) buildContext(ctx context.Context, entries fs.Entries) return newic, nil } +// nolint:gocritic func (c *ignoreContext) overrideFromPolicy(fp policy.FilesPolicy, dirPath string) error { if fp.NoParentDotIgnoreFiles { c.dotIgnoreFiles = nil diff --git a/htmlui/src/PolicyEditor.js b/htmlui/src/PolicyEditor.js index 5ca5b8be8..7645483b7 100644 --- a/htmlui/src/PolicyEditor.js +++ b/htmlui/src/PolicyEditor.js @@ -179,6 +179,9 @@ export class PolicyEditor extends Component { {RequiredBoolean(this, "Ignore Parent Rules", "policy.files.noParentIgnore")} {RequiredBoolean(this, "Ignore Parent Rule Files", "policy.files.noParentDotFiles")} + + {OptionalBoolean(this, "Ignore Well-Known Cache Directories", "policy.files.ignoreCacheDirs", "inherit from parent")} + diff --git a/repo/open.go b/repo/open.go index 093687e43..3115412c4 100644 --- a/repo/open.go +++ b/repo/open.go @@ -22,7 +22,22 @@ ) // CacheDirMarkerFile is the name of the marker file indicating a directory contains Kopia caches. -const CacheDirMarkerFile = ".kopia-cache" +// See https://bford.info/cachedir/ +const CacheDirMarkerFile = "CACHEDIR.TAG" + +// CacheDirMarkerHeader is the header signature for cache dir marker files. +const CacheDirMarkerHeader = "Signature: 8a477f597d28d172789f06886806bc55" + +const cacheDirMarkerContents = CacheDirMarkerHeader + ` +# +# This file is a cache directory tag created by Kopia - Fast And Secure Open-Source Backup. +# +# For information about Kopia, see: +# https://kopia.io +# +# For information about cache directory tags, see: +# http://www.brynosaurus.com/cachedir/ +` var log = logging.GetContextLoggerFunc("kopia/repo") @@ -193,7 +208,14 @@ func writeCacheMarker(cacheDir string) error { } markerFile := filepath.Join(cacheDir, CacheDirMarkerFile) - if _, err := os.Stat(markerFile); !os.IsNotExist(err) { + + st, err := os.Stat(markerFile) + if err == nil && st.Size() >= int64(len(cacheDirMarkerContents)) { + // ok + return nil + } + + if !os.IsNotExist(err) { return err } @@ -202,6 +224,10 @@ func writeCacheMarker(cacheDir string) error { return err } + if _, err := f.WriteString(cacheDirMarkerContents); err != nil { + return errors.Wrap(err, "unable to write cachedir marker contents") + } + return f.Close() } diff --git a/snapshot/policy/files_policy.go b/snapshot/policy/files_policy.go index b2272cafd..fcee9958f 100644 --- a/snapshot/policy/files_policy.go +++ b/snapshot/policy/files_policy.go @@ -8,10 +8,13 @@ type FilesPolicy struct { DotIgnoreFiles []string `json:"ignoreDotFiles,omitempty"` NoParentDotIgnoreFiles bool `json:"noParentDotFiles,omitempty"` + IgnoreCacheDirs *bool `json:"ignoreCacheDirs,omitempty"` + MaxFileSize int64 `json:"maxFileSize,omitempty"` } // Merge applies default values from the provided policy. +// nolint:gocritic func (p *FilesPolicy) Merge(src FilesPolicy) { if p.MaxFileSize == 0 { p.MaxFileSize = src.MaxFileSize @@ -24,6 +27,19 @@ func (p *FilesPolicy) Merge(src FilesPolicy) { if len(p.DotIgnoreFiles) == 0 { p.DotIgnoreFiles = src.DotIgnoreFiles } + + if p.IgnoreCacheDirs == nil { + p.IgnoreCacheDirs = src.IgnoreCacheDirs + } +} + +// IgnoreCacheDirectoriesOrDefault gets the value of IgnoreCacheDirs or the provided default if not set. +func (p *FilesPolicy) IgnoreCacheDirectoriesOrDefault(def bool) bool { + if p.IgnoreCacheDirs == nil { + return def + } + + return *p.IgnoreCacheDirs } // defaultFilesPolicy is the default file ignore policy. diff --git a/snapshot/snapshotfs/upload.go b/snapshot/snapshotfs/upload.go index 93a86fcf7..f9c8cb8e7 100644 --- a/snapshot/snapshotfs/upload.go +++ b/snapshot/snapshotfs/upload.go @@ -688,7 +688,7 @@ func maybeReadDirectoryEntries(ctx context.Context, dir fs.Directory) fs.Entries return nil } - return skipCacheDirectory(ent) + return ent } func uniqueDirectories(dirs []fs.Directory) []fs.Directory { @@ -746,8 +746,6 @@ func uploadDirInternal( return "", fs.DirectorySummary{}, dirReadError{direrr} } - entries = skipCacheDirectory(entries) - var prevEntries []fs.Entries for _, d := range uniqueDirectories(previousDirs) { @@ -784,15 +782,6 @@ func uploadDirInternal( return oid, *dirManifest.Summary, err } -func skipCacheDirectory(entries fs.Entries) fs.Entries { - if entries.FindByName(repo.CacheDirMarkerFile) != nil { - // if the given directory contains a marker file used for kopia cache, pretend the directory was empty. - return nil - } - - return entries -} - func (u *Uploader) maybeIgnoreFileReadError(err error, output chan dirEntryOrError, entryRelativePath string, policyTree *policy.Tree) error { errHandlingPolicy := policyTree.EffectivePolicy().ErrorHandlingPolicy diff --git a/tests/testenv/faketimeserver.go b/tests/testenv/faketimeserver.go index 5bc3245b4..eb27619a5 100644 --- a/tests/testenv/faketimeserver.go +++ b/tests/testenv/faketimeserver.go @@ -52,7 +52,7 @@ func (s *FakeTimeServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { func NewFakeTimeServer(startTime time.Time, step time.Duration) *FakeTimeServer { return &FakeTimeServer{ nextTimeChunk: startTime, - timeChunkLength: 100 * step, // nolint:mnd + timeChunkLength: 100 * step, // nolint:gomnd step: step, } }