From 132e2eef5089bb2db7aabdf94c3e0e18f4f4f204 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Wed, 10 Mar 2021 23:04:55 -0800 Subject: [PATCH] New snapshot UX - streamlined snapshot creation and policy setting (#878) * uitask: added support for reporting string progress info * server: report current directory as task progress * snapshot: created reusable Estimate() method to be used during upload, cli estimate and via API * cli: switched to snapshotfs.Estimate() * server: added API to estimate snapshot size * kopia-ui: fixed directory selector * htmlui: streamlined new snapshot flow and cleaned up policy setting See https://youtu.be/8p6csuoB3kg --- app/public/electron.js | 1 + cli/command_snapshot_estimate.go | 168 +++++++++++------------- htmlui/src/App.css | 21 ++- htmlui/src/App.js | 10 +- htmlui/src/NewSnapshot.js | 186 +++++++++++++++++++++++++++ htmlui/src/PoliciesTable.js | 198 +++++++++++++++++++++++------ htmlui/src/PolicyEditor.js | 79 ++++++++---- htmlui/src/SourcesTable.js | 176 +++++-------------------- htmlui/src/TaskDetails.js | 80 +++++++----- htmlui/src/TasksTable.js | 54 ++++---- htmlui/src/forms.js | 5 + htmlui/src/uiutil.js | 65 +++++++++- internal/server/api_estimate.go | 121 ++++++++++++++++++ internal/server/api_sources.go | 2 + internal/server/server.go | 1 + internal/server/source_manager.go | 2 + internal/serverapi/serverapi.go | 5 + internal/uitask/uitask.go | 9 ++ internal/uitask/uitask_manager.go | 3 + internal/uitask/uitask_test.go | 9 ++ snapshot/snapshotfs/estimate.go | 152 ++++++++++++++++++++++ snapshot/snapshotfs/upload.go | 21 ++- snapshot/snapshotfs/upload_scan.go | 52 +++----- snapshot/snapshotfs/upload_test.go | 53 +++++++- 24 files changed, 1052 insertions(+), 421 deletions(-) create mode 100644 htmlui/src/NewSnapshot.js create mode 100644 internal/server/api_estimate.go create mode 100644 snapshot/snapshotfs/estimate.go diff --git a/app/public/electron.js b/app/public/electron.js index 49cf6dd68..fb0d879c0 100644 --- a/app/public/electron.js +++ b/app/public/electron.js @@ -32,6 +32,7 @@ function showRepoWindow(repoID) { autoHideMenuBar: true, webPreferences: { nodeIntegration: true, + enableRemoteModule: true, }, }) diff --git a/cli/command_snapshot_estimate.go b/cli/command_snapshot_estimate.go index 6c4c45e25..312bd6b79 100644 --- a/cli/command_snapshot_estimate.go +++ b/cli/command_snapshot_estimate.go @@ -9,11 +9,11 @@ "github.com/pkg/errors" "github.com/kopia/kopia/fs" - "github.com/kopia/kopia/fs/ignorefs" "github.com/kopia/kopia/internal/units" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/policy" + "github.com/kopia/kopia/snapshot/snapshotfs" ) var ( @@ -24,44 +24,32 @@ snapshotEstimateUploadSpeed = snapshotEstimate.Flag("upload-speed", "Upload speed to use for estimation").Default("10").PlaceHolder("mbit/s").Float64() ) -const maxExamplesPerBucket = 10 - -type bucket struct { - MinSize int64 `json:"minSize"` - Count int `json:"count"` - TotalSize int64 `json:"totalSize"` - Examples []string `json:"examples,omitempty"` +type estimateProgress struct { + stats snapshot.Stats + included snapshotfs.SampleBuckets + excluded snapshotfs.SampleBuckets + excludedDirs []string } -func (b *bucket) add(fname string, size int64) { - b.Count++ - b.TotalSize += size - - if len(b.Examples) < maxExamplesPerBucket { - b.Examples = append(b.Examples, fmt.Sprintf("%v - %v", fname, units.BytesStringBase10(size))) +func (ep *estimateProgress) Processing(ctx context.Context, dirname string) { + if !*snapshotEstimateQuiet { + log(ctx).Infof("Analyzing %v...", dirname) } } -type buckets []*bucket - -func (b buckets) add(fname string, size int64) { - for _, bucket := range b { - if size >= bucket.MinSize { - bucket.add(fname, size) - break - } +func (ep *estimateProgress) Error(ctx context.Context, filename string, err error, isIgnored bool) { + if isIgnored { + log(ctx).Warningf("Ignored error in %v: %v", filename, err) + } else { + log(ctx).Errorf("Error in %v: %v", filename, err) } } -func makeBuckets() buckets { - return buckets{ - &bucket{MinSize: 1e15}, - &bucket{MinSize: 1e12}, - &bucket{MinSize: 1e9}, - &bucket{MinSize: 1e6}, - &bucket{MinSize: 1e3}, - &bucket{MinSize: 0}, - } +func (ep *estimateProgress) Stats(ctx context.Context, st *snapshot.Stats, included, excluded snapshotfs.SampleBuckets, excludedDirs []string, final bool) { + ep.stats = *st + ep.included = included + ep.excluded = excluded + ep.excludedDirs = excludedDirs } func runSnapshotEstimateCommand(ctx context.Context, rep repo.Repository) error { @@ -76,51 +64,53 @@ func runSnapshotEstimateCommand(ctx context.Context, rep repo.Repository) error UserName: rep.ClientOptions().Username, } - var stats snapshot.Stats - - ib := makeBuckets() - eb := makeBuckets() - - onIgnoredFile := func(relativePath string, e fs.Entry) { - eb.add(relativePath, e.Size()) - - if e.IsDir() { - stats.ExcludedDirCount++ - - log(ctx).Infof("excluded dir %v", relativePath) - } else { - log(ctx).Infof("excluded file %v (%v)", relativePath, units.BytesStringBase10(e.Size())) - stats.ExcludedFileCount++ - stats.ExcludedTotalFileSize += e.Size() - } - } - entry, err := getLocalFSEntry(ctx, path) if err != nil { return err } - if dir, ok := entry.(fs.Directory); ok { - policyTree, err := policy.TreeForSource(ctx, rep, sourceInfo) - if err != nil { - return errors.Wrapf(err, "error creating policy tree for %v", sourceInfo) - } - - entry = ignorefs.New(dir, policyTree, ignorefs.ReportIgnoredFiles(onIgnoredFile)) + dir, ok := entry.(fs.Directory) + if !ok { + return errors.Errorf("invalid path: '%s': must be a directory", path) } - if err := estimate(ctx, ".", entry, &stats, ib); err != nil { - return err + var ep estimateProgress + + policyTree, err := policy.TreeForSource(ctx, rep, sourceInfo) + if err != nil { + return errors.Wrapf(err, "error creating policy tree for %v", sourceInfo) } - fmt.Printf("Snapshot includes %v files, total size %v\n", stats.TotalFileCount, units.BytesStringBase10(stats.TotalFileSize)) - showBuckets(ib) + if err := snapshotfs.Estimate(ctx, rep, dir, policyTree, &ep); err != nil { + return errors.Wrap(err, "error estimating") + } + + fmt.Printf("Snapshot includes %v files, total size %v\n", ep.stats.TotalFileCount, units.BytesStringBase10(ep.stats.TotalFileSize)) + showBuckets(ep.included, *snapshotEstimateShowFiles) fmt.Println() - fmt.Printf("Snapshot excludes %v directories and %v files with total size %v\n", stats.ExcludedDirCount, stats.ExcludedFileCount, units.BytesStringBase10(stats.ExcludedTotalFileSize)) - showBuckets(eb) + if ep.stats.ExcludedFileCount > 0 { + fmt.Printf("Snapshot excludes %v files, total size %v\n", ep.stats.ExcludedFileCount, ep.stats.ExcludedTotalFileSize) + showBuckets(ep.excluded, true) + } else { + fmt.Printf("Snapshots excludes no files.\n") + } - megabits := float64(stats.TotalFileSize) * 8 / 1000000 //nolint:gomnd + if ep.stats.ExcludedDirCount > 0 { + fmt.Printf("Snapshots excludes %v directories. Examples:\n", ep.stats.ExcludedDirCount) + + for _, ed := range ep.excludedDirs { + fmt.Printf(" - %v\n", ed) + } + } else { + fmt.Printf("Snapshots excludes no directories.\n") + } + + if ep.stats.ErrorCount > 0 { + fmt.Printf("Encountered %v errors.\n", ep.stats.ErrorCount) + } + + megabits := float64(ep.stats.TotalFileSize) * 8 / 1000000 //nolint:gomnd seconds := megabits / *snapshotEstimateUploadSpeed fmt.Println() @@ -129,49 +119,35 @@ func runSnapshotEstimateCommand(ctx context.Context, rep repo.Repository) error return nil } -func showBuckets(b buckets) { - for _, bucket := range b { +func showBuckets(buckets snapshotfs.SampleBuckets, showFiles bool) { + for i, bucket := range buckets { if bucket.Count == 0 { continue } - fmt.Printf(" with size over %-5v: %7v files, total size %v\n", units.BytesStringBase10(bucket.MinSize), bucket.Count, units.BytesStringBase10(bucket.TotalSize)) + var sizeRange string - if *snapshotEstimateShowFiles { + if i == 0 { + sizeRange = fmt.Sprintf("< %-6v", + units.BytesStringBase10(bucket.MinSize)) + } else { + sizeRange = fmt.Sprintf("%-6v...%6v", + units.BytesStringBase10(bucket.MinSize), + units.BytesStringBase10(buckets[i-1].MinSize)) + } + + fmt.Printf("%18v: %7v files, total size %v\n", + sizeRange, + bucket.Count, units.BytesStringBase10(bucket.TotalSize)) + + if showFiles { for _, sample := range bucket.Examples { - fmt.Printf(" %v\n", sample) + fmt.Printf(" - %v\n", sample) } } } } -func estimate(ctx context.Context, relativePath string, entry fs.Entry, stats *snapshot.Stats, ib buckets) error { - switch entry := entry.(type) { - case fs.Directory: - if !*snapshotEstimateQuiet { - log(ctx).Infof("Scanning %v\n", relativePath) - } - - children, err := entry.Readdir(ctx) - if err != nil { - return errors.Wrap(err, "unable to read directory") - } - - for _, child := range children { - if err := estimate(ctx, filepath.Join(relativePath, child.Name()), child, stats, ib); err != nil { - return err - } - } - - case fs.File: - ib.add(relativePath, entry.Size()) - stats.TotalFileCount++ - stats.TotalFileSize += entry.Size() - } - - return nil -} - func init() { snapshotEstimate.Action(repositoryReaderAction(runSnapshotEstimateCommand)) } diff --git a/htmlui/src/App.css b/htmlui/src/App.css index f2c88a8a3..748dc134a 100644 --- a/htmlui/src/App.css +++ b/htmlui/src/App.css @@ -113,8 +113,8 @@ div.tab-body { } .logs-table { - border: 1px solid #ccc; - background-color: #f0f0f0; + border: 1px solid #ccd; + background-color: #f0f0ff; font-family: monospace; font-size: 12px; font-weight: bold; @@ -140,4 +140,21 @@ div.tab-body { height: 40px; font-size: 125%; vertical-align: text-bottom; +} + +.nested-task { + padding: 10px; + padding-left: 40px; +} + +.estimate-results { + margin: 10px; + margin-left: 15px; + padding: 15px; + border: 1px solid #ccc; + background-color: #f8f8ff; +} + +.list-actions { + min-height: 40px; } \ No newline at end of file diff --git a/htmlui/src/App.js b/htmlui/src/App.js index 61a36b1a2..5dda500ab 100644 --- a/htmlui/src/App.js +++ b/htmlui/src/App.js @@ -15,6 +15,8 @@ import { SnapshotsTable } from "./SnapshotsTable"; import { SourcesTable } from "./SourcesTable"; import { TaskDetails } from './TaskDetails'; import { TasksTable } from './TasksTable'; +import { NewSnapshot } from './NewSnapshot'; +import { PolicyEditor } from './PolicyEditor'; function useInterval(callback, delay) { const savedCallback = useRef(); @@ -40,8 +42,8 @@ function App() { const [runningTaskCount, setRunningTaskCount] = useState(0); useInterval(() => { - axios.get('/api/v1/tasks-summary').then(result => { - setRunningTaskCount(result.data["RUNNING"] || 0); + axios.get('/api/v1/tasks-summary').then(result => { + setRunningTaskCount(result.data["RUNNING"] || 0); }).catch(error => { setRunningTaskCount(-1); }); @@ -57,7 +59,7 @@ function App() { Snapshots Policies Tasks <> - {runningTaskCount > 0 && <>({runningTaskCount})} + {runningTaskCount > 0 && <>({runningTaskCount})} Repository @@ -67,10 +69,12 @@ function App() { + + diff --git a/htmlui/src/NewSnapshot.js b/htmlui/src/NewSnapshot.js new file mode 100644 index 000000000..09c51d01d --- /dev/null +++ b/htmlui/src/NewSnapshot.js @@ -0,0 +1,186 @@ +import { faWindowClose } from '@fortawesome/free-solid-svg-icons'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import axios from 'axios'; +import React, { Component } from 'react'; +import Button from 'react-bootstrap/Button'; +import Col from 'react-bootstrap/Col'; +import Form from 'react-bootstrap/Form'; +import { handleChange, validateRequiredFields } from './forms'; +import { PolicyEditor } from './PolicyEditor'; +import { TaskDetails } from './TaskDetails'; +import { cancelTask, DirectorySelector, GoBackButton, redirectIfNotConnected } from './uiutil'; + +export class NewSnapshot extends Component { + constructor() { + super(); + this.state = { + path: "", + estimateTaskID: null, + estimateTaskVisible: false, + policyEditorVisibleFor: "n/a", + localUsername: null, + }; + + this.policyEditorRef = React.createRef(); + this.handleChange = handleChange.bind(this); + this.estimate = this.estimate.bind(this); + this.snapshotNow = this.snapshotNow.bind(this); + this.cancelEstimate = this.cancelEstimate.bind(this); + this.togglePolicyEditor = this.togglePolicyEditor.bind(this); + } + + componentDidMount() { + axios.get('/api/v1/sources').then(result => { + this.setState({ + localUsername: result.data.localUsername, + localHost: result.data.localHost, + }); + }).catch(error => { + redirectIfNotConnected(error); + }); + } + + togglePolicyEditor() { + if (!this.state.path) { + return; + } + + if (this.state.policyEditorVisibleFor === this.state.path) { + this.setState({ + policyEditorVisibleFor: "n/a", + }); + } else { + this.setState({ + policyEditorVisibleFor: this.state.path, + }); + } + } + + estimate(e) { + e.preventDefault(); + + if (!validateRequiredFields(this, ["path"])) { + return; + } + + let req = { + root: this.state.path, + } + + axios.post('/api/v1/estimate', req).then(result => { + this.setState({ + estimateTaskID: result.data.id, + estimatingPath: result.data.description, + estimateTaskVisible: true, + didEstimate: false, + }) + }).catch(error => { + if (error.response.data) { + alert(JSON.stringify(error.response.data)); + } else { + alert('failed'); + } + }); + } + + cancelEstimate() { + this.setState({ estimateTaskVisible: false }); + cancelTask(this.state.estimateTaskID); + } + + snapshotNow(e) { + e.preventDefault(); + + if (!this.state.path) { + alert('Must specify directory to snapshot.'); + return + } + + axios.post('/api/v1/sources', { + path: this.state.path, + createSnapshot: true, + }).then(result => { + this.props.history.goBack(); + }).catch(error => { + if (error.response) { + alert('Error: ' + error.response.data.error + " (" + error.response.data.code + ")"); + return + } + + this.setState({ + error, + isLoading: false + }); + }); + } + + render() { + return <> + + + + +    

New Snapshot

+ +
+ + + + this.setState({ path: p })} autoFocus placeholder="enter path to snapshot" name="path" value={this.state.path} onChange={this.handleChange} + readOnly={this.state.policyEditorVisible || this.state.estimateTaskVisible} /> + + Click the Estimate button to estimate the size of the snapshot. + To specify frequency of snapshots or exclude some files, click Policy. + + + + + + + + + + + {this.state.estimateTaskID && this.state.estimateTaskVisible && +
+
+   + Estimate Snapshot Size for {this.state.estimatingPath}... +
+ +
+ } +
+ + {this.state.path && this.state.policyEditorVisibleFor === this.state.path && + + + + } + + + + + ; + } +} diff --git a/htmlui/src/PoliciesTable.js b/htmlui/src/PoliciesTable.js index d5d8bed54..7964e7e78 100644 --- a/htmlui/src/PoliciesTable.js +++ b/htmlui/src/PoliciesTable.js @@ -1,13 +1,23 @@ +import { faUserFriends } from '@fortawesome/free-solid-svg-icons'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import axios from 'axios'; import React, { Component } from 'react'; import Badge from 'react-bootstrap/Badge'; import Button from 'react-bootstrap/Button'; +import Col from 'react-bootstrap/Col'; +import Dropdown from 'react-bootstrap/Dropdown'; import Form from 'react-bootstrap/Form'; -import { handleChange, OptionalField, RequiredField, validateRequiredFields } from './forms'; +import Row from 'react-bootstrap/Row'; +import { handleChange } from './forms'; import { PolicyEditor } from './PolicyEditor'; import MyTable from './Table'; -import { redirectIfNotConnected } from './uiutil'; +import { compare, DirectorySelector, isAbsolutePath, ownerName, redirectIfNotConnected } from './uiutil'; +const localPolicies = "Local Policies" +const allPolicies = "All Policies" +const globalPolicy = "Global Policy" +const perUserPolicies = "Per-User Policies" +const perHostPolicies = "Per-Host Policies" export class PoliciesTable extends Component { constructor() { super(); @@ -16,16 +26,16 @@ export class PoliciesTable extends Component { isLoading: false, error: null, editorTarget: null, - selectedRowIsNew: false, - targetHost: "*", - targetUsername: "*", - targetPath: "*", + selectedOwner: localPolicies, + policyPath: "", + sources: [], }; this.editorClosed = this.editorClosed.bind(this); - this.setNewPolicy = this.setNewPolicy.bind(this); + this.editPolicyForPath = this.editPolicyForPath.bind(this); this.handleChange = handleChange.bind(this); this.fetchPolicies = this.fetchPolicies.bind(this); + this.fetchSourcesWithoutSpinner = this.fetchSourcesWithoutSpinner.bind(this); } componentDidMount() { @@ -34,6 +44,20 @@ export class PoliciesTable extends Component { }); this.fetchPolicies(); + this.fetchSourcesWithoutSpinner(); + } + + sync() { + this.fetchPolicies(); + + axios.post('/api/v1/repo/sync', {}).then(result => { + this.fetchSourcesWithoutSpinner(); + }).catch(error => { + this.setState({ + error, + isLoading: false + }); + }); } fetchPolicies() { @@ -51,6 +75,25 @@ export class PoliciesTable extends Component { }); } + fetchSourcesWithoutSpinner() { + axios.get('/api/v1/sources').then(result => { + this.setState({ + localSourceName: result.data.localUsername + "@" + result.data.localHost, + localUsername: result.data.localUsername, + localHost: result.data.localHost, + multiUser: result.data.multiUser, + sources: result.data.sources, + isLoading: false, + }); + }).catch(error => { + redirectIfNotConnected(error); + this.setState({ + error, + isLoading: false + }); + }); + } + editorClosed() { this.setState({ editorTarget: null, @@ -58,34 +101,33 @@ export class PoliciesTable extends Component { this.fetchPolicies(); } - setNewPolicy() { - if (!validateRequiredFields(this, ["targetHost"])) { + editPolicyForPath(e) { + e.preventDefault(); + + if (!this.state.policyPath) { return; } - function fixInput(a) { - if (!a) { - return ""; - } - - // allow both * and empty string to indicate "all" - if (a === "*") { - return "" - } - - return a; + if (!isAbsolutePath(this.state.policyPath)) { + alert("Policies can only be defined for absolute paths."); + return; } this.setState({ editorTarget: { - userName: fixInput(this.state.targetUsername), - host: this.state.targetHost, - path: fixInput(this.state.targetPath), + userName: this.state.localUsername, + host: this.state.localHost, + path: this.state.policyPath, }, - selectedRowIsNew: true, }) } + selectOwner(h) { + this.setState({ + selectedOwner: h, + }); + } + policySummary(p) { function isEmpty(obj) { for (var key in obj) { @@ -117,13 +159,63 @@ export class PoliciesTable extends Component { } render() { - const { items, isLoading, error } = this.state; + let { items, sources, isLoading, error } = this.state; if (error) { return

{error.message}

; } if (isLoading) { return

Loading ...

; } + + + let uniqueOwners = sources.reduce((a, d) => { + const owner = ownerName(d.source); + + if (!a.includes(owner)) { a.push(owner); } + return a; + }, []); + + uniqueOwners.sort(); + + switch (this.state.selectedOwner) { + case allPolicies: + // do nothing; + break; + + case globalPolicy: + items = items.filter(x => !x.target.userName && !x.target.host && !x.target.path); + break; + + case localPolicies: + items = items.filter(x => ownerName(x.target) === this.state.localSourceName && x.target.path.startsWith(this.state.policyPath)); + break; + + case perUserPolicies: + items = items.filter(x => !!x.target.userName && !!x.target.host && !x.target.path); + break; + + case perHostPolicies: + items = items.filter(x => !x.target.userName && !!x.target.host && !x.target.path); + break; + + default: + items = items.filter(x => ownerName(x.target) === this.state.selectedOwner); + break; + }; + + items.sort((l,r) => { + const hc = compare(l.target.host,r.target.host); + if (hc) { + return hc; + } + const uc = compare(l.target.userName,r.target.userName); + if (uc) { + return uc; + } + return compare(l.target.path,r.target.path); + }); + + const columns = [{ Header: 'Username', accessor: x => x.target.userName || "*", @@ -150,23 +242,49 @@ export class PoliciesTable extends Component { }] return
-
- -
+ {!this.state.editorTarget &&
+
+ + + + +  {this.state.selectedOwner} + -
-
-
- - {OptionalField(this, "Target Username", "targetUsername", {}, "Specify * all empty to target all users")} - {RequiredField(this, "Target Host", "targetHost", {})} - {OptionalField(this, "Target Path", "targetPath", {}, "Specify * all empty to target all filesystem paths")} - - + + this.selectOwner(localPolicies)}>{localPolicies} + this.selectOwner(allPolicies)}>{allPolicies} + + this.selectOwner(globalPolicy)}>{globalPolicy} + this.selectOwner(perUserPolicies)}>{perUserPolicies} + this.selectOwner(perHostPolicies)}>{perHostPolicies} + + {uniqueOwners.map(v => this.selectOwner(v)}>{v})} + + + + {this.state.selectedOwner === localPolicies ? <> + + this.setState({ policyPath: p })} + placeholder="enter directory to find or set policy" + name="policyPath" value={this.state.policyPath} onChange={this.handleChange} /> + + + + + : } + -
-
- {this.state.editorTarget && } +
} + + {items.length > 0 ?
+

Found {items.length} policies matching criteria.

+ +
: ((this.state.selectedOwner === localPolicies && this.state.policyPath) ?

+ No policy found for directory {this.state.policyPath}. Click Set Policy to define it. +

:

No policies found.

)} + + {this.state.editorTarget && }
; } } diff --git a/htmlui/src/PolicyEditor.js b/htmlui/src/PolicyEditor.js index 2ef989b60..aca4d14d3 100644 --- a/htmlui/src/PolicyEditor.js +++ b/htmlui/src/PolicyEditor.js @@ -3,26 +3,27 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import axios from 'axios'; import React, { Component } from 'react'; import Button from 'react-bootstrap/Button'; +import Row from 'react-bootstrap/Row'; import Col from 'react-bootstrap/Col'; import Form from 'react-bootstrap/Form'; import Tab from 'react-bootstrap/Tab'; import Tabs from 'react-bootstrap/Tabs'; import { handleChange, OptionalBoolean, OptionalNumberField, RequiredBoolean, stateProperty, StringList } from './forms'; -function sourceDisplayName(s) { +function policyTypeName(s) { if (!s.host && !s.userName) { - return "global" + return "Global Policy" } if (!s.userName) { - return "host " + s.host; + return "Per-Host Policy"; } if (!s.path) { - return "user " + s.userName + "@" + s.host; + return "Per-User Policy"; } - return "user " + s.userName + "@" + s.host + " path '" + s.path + "'"; + return "Directory Policy"; } export class PolicyEditor extends Component { @@ -57,27 +58,30 @@ export class PolicyEditor extends Component { } fetchPolicy(props) { - if (props.isNew) { - this.setState({ - isLoading: false, - policy: {}, - }); - - return; - } - axios.get(this.snapshotURL(props)).then(result => { this.setState({ isLoading: false, policy: result.data, }); - }).catch(error => this.setState({ - error: error, - isLoading: false - })); + }).catch(error => { + if (error.response && error.response.data.code !== "NOT_FOUND") { + this.setState({ + error: error, + isLoading: false + }) + } else { + this.setState({ + policy: {}, + isNew: true, + isLoading: false + }) + } + }); } - saveChanges() { + saveChanges(e) { + e.preventDefault() + function removeEmpty(l) { if (!l) { return l; @@ -119,7 +123,7 @@ export class PolicyEditor extends Component { axios.put(this.snapshotURL(this.props), policy).then(result => { this.props.close(); }).catch(error => { - alert('Error saving policy: ' + error); + alert('Error saving policy: ' + JSON.stringify(error)); }); } @@ -152,8 +156,29 @@ export class PolicyEditor extends Component { } return
-

{this.props.isNew && "New "}Policy: {sourceDisplayName(this.props)}

- + {!this.props.embedded &&

+   {policyTypeName(this.props)}

} +
+ {!this.props.embedded && + + + Target User + "} /> + + + + + Target Host + "} /> + + + + + Target Directory Path + "} /> + + + }
@@ -209,7 +234,7 @@ export class PolicyEditor extends Component { name="policy.compression.compressorName" onChange={this.handleChange} value={stateProperty(this, "policy.compression.compressorName")}> - + {this.state.algorithms && this.state.algorithms.compression.map(x => )} @@ -238,14 +263,18 @@ export class PolicyEditor extends Component { {RequiredBoolean(this, "Disable Parent Policy Evaluation (prevents any parent policies from affecting this directory and subdirectories)", "policy.noParent")} - - {!this.props.isNew && <>  + + {!this.state.isNew && <>  } + + {!this.props.embedded && <>
JSON representation
{JSON.stringify(this.state.policy, null, 4)}
+ } +
; } } diff --git a/htmlui/src/SourcesTable.js b/htmlui/src/SourcesTable.js index b598828bd..d0a28a1f7 100644 --- a/htmlui/src/SourcesTable.js +++ b/htmlui/src/SourcesTable.js @@ -5,12 +5,8 @@ import moment from 'moment'; import React, { Component } from 'react'; import Badge from 'react-bootstrap/Badge'; import Button from 'react-bootstrap/Button'; -import ButtonGroup from 'react-bootstrap/ButtonGroup'; -import ButtonToolbar from 'react-bootstrap/ButtonToolbar'; +import Col from 'react-bootstrap/Col'; import Dropdown from 'react-bootstrap/Dropdown'; -import DropdownButton from 'react-bootstrap/DropdownButton'; -import FormControl from 'react-bootstrap/FormControl'; -import InputGroup from 'react-bootstrap/InputGroup'; import Row from 'react-bootstrap/Row'; import Spinner from 'react-bootstrap/Spinner'; import { Link } from 'react-router-dom'; @@ -37,13 +33,8 @@ export class SourcesTable extends Component { this.sync = this.sync.bind(this); this.fetchSourcesWithoutSpinner = this.fetchSourcesWithoutSpinner.bind(this); - this.selectDirectory = this.selectDirectory.bind(this); this.handleChange = handleChange.bind(this); - this.snapshotOnce = this.snapshotOnce.bind(this); - this.snapshotEveryDay = this.snapshotEveryDay.bind(this); - this.snapshotEveryHour = this.snapshotEveryHour.bind(this); - this.createPolicy = this.createPolicy.bind(this); this.cancelSnapshot = this.cancelSnapshot.bind(this); this.startSnapshot = this.startSnapshot.bind(this); } @@ -94,87 +85,6 @@ export class SourcesTable extends Component { }); } - selectDirectory() { - // populated in 'preload.js' in Electron - if (!window.require) { - alert('Directory selection is not supported in a web browser.\n\nPlease enter path manually.'); - return; - } - - const { dialog } = window.require('electron').remote; - try { - let dir = dialog.showOpenDialogSync({ - properties: ['openDirectory'] - }); - if (dir) { - this.setState({ - selectedDirectory: dir[0], - }); - } - } catch (e) { - window.alert('Error: ' + e); - } - } - - createSource(request) { - if (!this.state.selectedDirectory) { - alert('Must specify directory to snapshot.'); - return - } - - axios.post('/api/v1/sources', request).then(result => { - this.fetchSourcesWithoutSpinner(); - }).catch(error => { - if (error.response) { - alert('Error: ' + error.response.data.error + " (" + error.response.data.code + ")"); - return - } - - this.setState({ - error, - isLoading: false - }); - }); - } - - snapshotOnce() { - this.createSource({ - path: this.state.selectedDirectory, - createSnapshot: true, - initialPolicy: { - } - }); - } - - snapshotEveryDay() { - this.createSource({ - path: this.state.selectedDirectory, - createSnapshot: true, - initialPolicy: { - scheduling: { intervalSeconds: 86400 }, - } - }); - } - - snapshotEveryHour() { - this.createSource({ - path: this.state.selectedDirectory, - createSnapshot: true, - initialPolicy: { - scheduling: { intervalSeconds: 3600 }, - } - }); - } - - createPolicy() { - this.createSource({ - path: this.state.selectedDirectory, - createSnapshot: false, - initialPolicy: { - scheduling: { intervalSeconds: 3600 }, - } - }); - } statusCell(x, parent) { switch (x.cell.value) { @@ -216,7 +126,7 @@ export class SourcesTable extends Component { return <>  {totals}   - {x.row.original.currentTask && Details} + {x.row.original.currentTask && Details} ; default: @@ -326,63 +236,35 @@ export class SourcesTable extends Component { Cell: x => this.statusCell(x, this), }] - const selectSupported = !!window.require; - return
- {this.state.multiUser && -   - - - -  {this.state.selectedOwner} - +
+ + {this.state.multiUser && <> + + +  {this.state.selectedOwner} + - - this.selectOwner(localSnapshots)}>{localSnapshots} - this.selectOwner(allSnapshots)}>{allSnapshots} - - {uniqueOwners.map(v => this.selectOwner(v)}>{v})} - - - -   - - - - } - - - - {selectSupported && } - -   - - Snapshot Once - Snapshot Every Hour - Snapshot Every Day - {/* Create Policy */} - - -
- - - + + this.selectOwner(localSnapshots)}>{localSnapshots} + this.selectOwner(allSnapshots)}>{allSnapshots} + + {uniqueOwners.map(v => this.selectOwner(v)}>{v})} + + + } + + + + + + + + + +
+ +
; } } diff --git a/htmlui/src/TaskDetails.js b/htmlui/src/TaskDetails.js index b3e1d9d0f..4ad1a82d6 100644 --- a/htmlui/src/TaskDetails.js +++ b/htmlui/src/TaskDetails.js @@ -1,5 +1,5 @@ -import { faStopCircle } from '@fortawesome/free-solid-svg-icons'; +import { faChevronCircleDown, faChevronCircleUp, faStopCircle } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import axios from 'axios'; import React, { Component } from 'react'; @@ -9,9 +9,8 @@ import Button from 'react-bootstrap/Button'; import Col from 'react-bootstrap/Col'; import Form from 'react-bootstrap/Form'; import Spinner from 'react-bootstrap/Spinner'; -import { sizeDisplayName } from './uiutil'; import { TaskLogs } from './TaskLogs'; -import { cancelTask, formatDuration, GoBackButton, redirectIfNotConnected } from './uiutil'; +import { cancelTask, formatDuration, GoBackButton, redirectIfNotConnected, sizeDisplayName } from './uiutil'; export class TaskDetails extends Component { constructor() { @@ -23,8 +22,9 @@ export class TaskDetails extends Component { showLog: false, }; + this.taskID = this.taskID.bind(this); this.fetchTask = this.fetchTask.bind(this); - + // poll frequently, we will stop as soon as the task ends. this.interval = window.setInterval(() => this.fetchTask(this.props), 500); } @@ -43,10 +43,12 @@ export class TaskDetails extends Component { } } - fetchTask(props) { - let tid = props.match.params.tid; + taskID(props) { + return props.taskID || props.match.params.tid; + } - axios.get('/api/v1/tasks/' + tid).then(result => { + fetchTask(props) { + axios.get('/api/v1/tasks/' + this.taskID(props)).then(result => { this.setState({ task: result.data, isLoading: false, @@ -74,25 +76,35 @@ export class TaskDetails extends Component { switch (task.status) { - case "SUCCESS": - return Task succeeded after {dur}.; + case "SUCCESS": + return Task succeeded after {dur}.; - case "FAILED": - return Error: {task.errorMessage}.; + case "FAILED": + return Error: {task.errorMessage}.; - case "CANCELING": - return Cancelation requested...; + case "CANCELED": + return Task canceled.; - case "CANCELED": - return Task canceled.; + case "CANCELING": + return + Canceling {dur}: {task.progressInfo}.; - default: - return Task in progress ({dur}).; + default: + return + Running for {dur}: {task.progressInfo}.; } } + valueThreshold() { + if (this.props.showZeroCounters) { + return -1; + } + + return 0 + } + counterBadge(label, c) { - if (!c.value) { + if (c.value < this.valueThreshold()) { return ""; } @@ -153,7 +165,7 @@ export class TaskDetails extends Component { return 0; }); - return keys.map(c => (counters[c].value > 0) && this.counterBadge(c, counters[c])); + return keys.map(c => (counters[c].value > this.valueThreshold()) && this.counterBadge(c, counters[c])); } render() { @@ -167,15 +179,16 @@ export class TaskDetails extends Component { } return
- - - - {task.status === "RUNNING" && <> -   - } - - - + {this.props.history && + + + + {task.status === "RUNNING" && <> +   + } + + } + {!this.props.hideDescription && @@ -186,7 +199,7 @@ export class TaskDetails extends Component { - + } {this.summaryControl(task)} @@ -199,13 +212,16 @@ export class TaskDetails extends Component { {task.counters && - {this.sortedBadges(task.counters)} + {this.sortedBadges(task.counters)} } -
+
- {this.state.showLog ? : } + {this.state.showLog ? <> + + + : }
diff --git a/htmlui/src/TasksTable.js b/htmlui/src/TasksTable.js index 6386a708d..c650c1fde 100644 --- a/htmlui/src/TasksTable.js +++ b/htmlui/src/TasksTable.js @@ -4,10 +4,9 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import axios from 'axios'; import moment from 'moment'; import React, { Component } from 'react'; -import ButtonGroup from 'react-bootstrap/ButtonGroup'; +import Alert from 'react-bootstrap/Alert'; import Col from 'react-bootstrap/Col'; import Dropdown from 'react-bootstrap/Dropdown'; -import Alert from 'react-bootstrap/Alert'; import Form from 'react-bootstrap/Form'; import Row from 'react-bootstrap/Row'; import { Link } from 'react-router-dom'; @@ -128,35 +127,34 @@ export class TasksTable extends Component { return
+
- - - - Status: {this.state.showStatus} - - this.setState({ showStatus: "All" })}>All - - this.setState({ showStatus: "Running" })}>Running - this.setState({ showStatus: "Failed" })}>Failed - - -   - - Kind: {this.state.showKind} - - this.setState({ showKind: "All" })}>All - - {this.state.uniqueKinds.map(k => this.setState({ showKind: k })}>{k})} - - -   - - + + + Status: {this.state.showStatus} + + this.setState({ showStatus: "All" })}>All + + this.setState({ showStatus: "Running" })}>Running + this.setState({ showStatus: "Failed" })}>Failed + + + + + + Kind: {this.state.showKind} + + this.setState({ showKind: "All" })}>All + + {this.state.uniqueKinds.map(k => this.setState({ showKind: k })}>{k})} + + + + + - -
- +
{!items.length ? diff --git a/htmlui/src/forms.js b/htmlui/src/forms.js index d5faca347..3c1c02c92 100644 --- a/htmlui/src/forms.js +++ b/htmlui/src/forms.js @@ -71,6 +71,7 @@ export function RequiredField(component, label, name, props = {}, helpText = nul return {label} {label} {label} {label} component.handleChange(e, optionalBooleanValue)} @@ -216,6 +220,7 @@ export function StringList(component, label, name, helpText) { return {label} component.handleChange(e, multilineStringToList)} diff --git a/htmlui/src/uiutil.js b/htmlui/src/uiutil.js index c6ec4a71b..f4bfea25b 100644 --- a/htmlui/src/uiutil.js +++ b/htmlui/src/uiutil.js @@ -1,4 +1,4 @@ -import { faBan, faCheck, faChevronLeft, faExclamationCircle, faExclamationTriangle, faWindowClose } from '@fortawesome/free-solid-svg-icons'; +import { faBan, faCheck, faChevronLeft, faExclamationCircle, faExclamationTriangle, faFolderOpen, faWindowClose } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import axios from 'axios'; import React from 'react'; @@ -7,6 +7,9 @@ import ListGroup from 'react-bootstrap/ListGroup'; import OverlayTrigger from 'react-bootstrap/OverlayTrigger'; import Popover from 'react-bootstrap/Popover'; import Spinner from 'react-bootstrap/Spinner'; +import InputGroup from 'react-bootstrap/InputGroup'; +import Form from 'react-bootstrap/Form'; +import FormControl from 'react-bootstrap/FormControl'; const base10UnitPrefixes = ["", "K", "M", "G", "T"]; @@ -172,4 +175,64 @@ export function cancelTask(tid) { export function GoBackButton(props) { return ; +} + +function selectDirectory(onSelected) { + // populated in 'preload.js' in Electron + if (!window.require) { + alert('Directory selection is not supported in a web browser.\n\nPlease enter path manually.'); + return; + } + + const { dialog } = window.require('electron').remote; + try { + let dir = dialog.showOpenDialogSync({ + properties: ['openDirectory'] + }); + if (dir) { + onSelected(dir[0]); + } + } catch (e) { + window.alert('Error: ' + e); + } +} + +export function isAbsolutePath(p) { + // Unix-style path. + if (p.startsWith("/")) { + return true; + } + + // Windows-style X:\... path. + if (p.length >= 3 && p.substring(1,3) === ":\\") { + const letter = p.substring(0, 1).toUpperCase(); + + return letter >= "A" && letter <= "Z"; + } + + // Windows UNC path. + if (p.startsWith("\\\\")) { + return true; + } + + return false; +} + +export function DirectorySelector(props) { + const selectSupported = !!window.require; + + let { onDirectorySelected, ...inputProps } = props; + + if (!selectSupported) { + return + } + + return + + + + + ; } \ No newline at end of file diff --git a/internal/server/api_estimate.go b/internal/server/api_estimate.go new file mode 100644 index 000000000..41d601620 --- /dev/null +++ b/internal/server/api_estimate.go @@ -0,0 +1,121 @@ +package server + +import ( + "context" + "encoding/json" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + + "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/fs/localfs" + "github.com/kopia/kopia/internal/ctxutil" + "github.com/kopia/kopia/internal/serverapi" + "github.com/kopia/kopia/internal/uitask" + "github.com/kopia/kopia/snapshot" + "github.com/kopia/kopia/snapshot/policy" + "github.com/kopia/kopia/snapshot/snapshotfs" +) + +type estimateTaskProgress struct { + ctrl uitask.Controller +} + +func (p estimateTaskProgress) Processing(ctx context.Context, dirname string) { + p.ctrl.ReportProgressInfo(dirname) +} + +func (p estimateTaskProgress) Error(ctx context.Context, dirname string, err error, isIgnored bool) { + if isIgnored { + log(ctx).Warningf("ignored error in %v: %v", dirname, err) + } else { + log(ctx).Errorf("error in %v: %v", dirname, err) + } +} + +func (p estimateTaskProgress) Stats(ctx context.Context, st *snapshot.Stats, included, excluded snapshotfs.SampleBuckets, excludedDirs []string, final bool) { + p.ctrl.ReportCounters(map[string]uitask.CounterValue{ + "Bytes": uitask.NoticeBytesCounter(st.TotalFileSize), + "Files": uitask.NoticeCounter(int64(st.TotalFileCount)), + "Directories": uitask.NoticeCounter(int64(st.TotalDirectoryCount)), + "Excluded Files": uitask.SimpleCounter(int64(st.ExcludedFileCount)), + "Excluded Directories": uitask.SimpleCounter(int64(st.ExcludedDirCount)), + "Errors": uitask.ErrorCounter(int64(st.ErrorCount)), + "Ignored Errors": uitask.ErrorCounter(int64(st.IgnoredErrorCount)), + }) +} + +func resolveUserFriendlyPath(path string) string { + home := os.Getenv("HOME") + if home != "" && strings.HasPrefix(path, "~") { + return home + path[1:] + } + + if filepath.IsAbs(path) { + return path + } + + return filepath.Join(home, path) +} + +var _ snapshotfs.EstimateProgress = estimateTaskProgress{} + +func (s *Server) handleEstimate(ctx context.Context, r *http.Request, body []byte) (interface{}, *apiError) { + var req serverapi.EstimateRequest + + if err := json.Unmarshal(body, &req); err != nil { + return nil, requestError(serverapi.ErrorMalformedRequest, "malformed request body") + } + + ctx = ctxutil.Detach(ctx) + rep := s.rep + + resolvedRoot := filepath.Clean(resolveUserFriendlyPath(req.Root)) + + e, err := localfs.NewEntry(resolvedRoot) + if err != nil { + return nil, internalServerError(errors.Wrap(err, "can't get local fs entry")) + } + + dir, ok := e.(fs.Directory) + if !ok { + return nil, internalServerError(errors.Wrap(err, "estimation is only supported on directories")) + } + + taskIDChan := make(chan string) + + // launch a goroutine that will continue the estimate and can be observed in the Tasks UI. + + // nolint:errcheck + go s.taskmgr.Run(ctx, "Estimate", resolvedRoot, func(ctx context.Context, ctrl uitask.Controller) error { + taskIDChan <- ctrl.CurrentTaskID() + + estimatectx, cancel := context.WithCancel(ctx) + defer cancel() + + ctrl.OnCancel(cancel) + + policyTree, err := policy.TreeForSource(ctx, s.rep, snapshot.SourceInfo{ + Host: s.options.ConnectOptions.Hostname, + UserName: s.options.ConnectOptions.Username, + Path: resolvedRoot, + }) + if err != nil { + return errors.Wrap(err, "unable to get policy tree") + } + + return snapshotfs.Estimate(estimatectx, rep, dir, policyTree, estimateTaskProgress{ctrl}) + }) + + taskID := <-taskIDChan + + task, ok := s.taskmgr.GetTask(taskID) + if !ok { + return nil, internalServerError(errors.Errorf("task not found")) + } + + return task, nil +} diff --git a/internal/server/api_sources.go b/internal/server/api_sources.go index 7895a0c68..2d22896a5 100644 --- a/internal/server/api_sources.go +++ b/internal/server/api_sources.go @@ -51,6 +51,8 @@ func (s *Server) handleSourcesCreate(ctx context.Context, r *http.Request, body return nil, requestError(serverapi.ErrorMalformedRequest, "missing path") } + req.Path = resolveUserFriendlyPath(req.Path) + _, err := os.Stat(req.Path) if os.IsNotExist(err) { return nil, requestError(serverapi.ErrorPathNotFound, "path does not exist") diff --git a/internal/server/server.go b/internal/server/server.go index 28ab07401..766315990 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -88,6 +88,7 @@ func (s *Server) APIHandlers(legacyAPI bool) http.Handler { m.HandleFunc("/api/v1/objects/{objectID}", s.requireAuth(s.handleObjectGet)).Methods(http.MethodGet) m.HandleFunc("/api/v1/restore", s.handleAPI(requireUIUser, s.handleRestore)).Methods(http.MethodPost) + m.HandleFunc("/api/v1/estimate", s.handleAPI(requireUIUser, s.handleEstimate)).Methods(http.MethodPost) // methods that can be called by any authenticated user (UI or remote user). m.HandleFunc("/api/v1/flush", s.handleAPI(anyAuthenticatedUser, s.handleFlush)).Methods(http.MethodPost) diff --git a/internal/server/source_manager.go b/internal/server/source_manager.go index 6d9430c95..777654e76 100644 --- a/internal/server/source_manager.go +++ b/internal/server/source_manager.go @@ -449,6 +449,8 @@ func (t *uitaskProgress) UploadedBytes(numBytes int64) { func (t *uitaskProgress) StartedDirectory(dirname string) { t.p.StartedDirectory(dirname) t.maybeReport() + + t.ctrl.ReportProgressInfo(dirname) } // FinishedDirectory is emitted whenever a directory is finished uploading. diff --git a/internal/serverapi/serverapi.go b/internal/serverapi/serverapi.go index db57fd5f3..a99d9a181 100644 --- a/internal/serverapi/serverapi.go +++ b/internal/serverapi/serverapi.go @@ -213,3 +213,8 @@ type RestoreRequest struct { TarFile string `json:"tarFile"` Options restore.Options `json:"options"` } + +// EstimateRequest contains request to estimate the size of the snapshot in a given root. +type EstimateRequest struct { + Root string `json:"root"` +} diff --git a/internal/uitask/uitask.go b/internal/uitask/uitask.go index 0e5648a6f..9754c6073 100644 --- a/internal/uitask/uitask.go +++ b/internal/uitask/uitask.go @@ -52,6 +52,7 @@ type Info struct { Kind string `json:"kind"` // Maintenance, Snapshot, Restore, etc. Description string `json:"description"` Status Status `json:"status"` + ProgressInfo string `json:"progressInfo"` ErrorMessage string `json:"errorMessage,omitempty"` Counters map[string]CounterValue `json:"counters"` LogLines []LogEntry `json:"-"` @@ -101,6 +102,14 @@ func (t *runningTaskInfo) cancel() { } } +// ReportProgressInfo implements the Controller interface. +func (t *runningTaskInfo) ReportProgressInfo(pi string) { + t.mu.Lock() + defer t.mu.Unlock() + + t.ProgressInfo = pi +} + // ReportCounters implements the Controller interface. func (t *runningTaskInfo) ReportCounters(c map[string]CounterValue) { t.mu.Lock() diff --git a/internal/uitask/uitask_manager.go b/internal/uitask/uitask_manager.go index 5df103c29..21c6436de 100644 --- a/internal/uitask/uitask_manager.go +++ b/internal/uitask/uitask_manager.go @@ -32,6 +32,7 @@ type Controller interface { CurrentTaskID() string OnCancel(cancelFunc context.CancelFunc) ReportCounters(counters map[string]CounterValue) + ReportProgressInfo(text string) } // TaskFunc represents a task function. @@ -178,6 +179,8 @@ func (m *Manager) completeTask(r *runningTaskInfo, err error) { r.Status = StatusCanceled } + r.ProgressInfo = "" + now := clock.Now() r.EndTime = &now diff --git a/internal/uitask/uitask_test.go b/internal/uitask/uitask_test.go index 885d04128..3561f210b 100644 --- a/internal/uitask/uitask_test.go +++ b/internal/uitask/uitask_test.go @@ -79,6 +79,7 @@ func TestUITask(t *testing.T) { "fff", }) + ctrl.ReportProgressInfo("doing something") ctrl.ReportCounters(map[string]uitask.CounterValue{ "foo": uitask.SimpleCounter(1), "bar": uitask.BytesCounter(2), @@ -103,6 +104,10 @@ func TestUITask(t *testing.T) { t.Fatalf("unexpected counters, diff: %v", diff) } + if got, want := tsk.ProgressInfo, "doing something"; got != want { + t.Fatalf("invalid progress info: %v, want %v", got, want) + } + return nil }) @@ -116,6 +121,10 @@ func TestUITask(t *testing.T) { t.Fatalf("task not found") } + if got, want := tsk.ProgressInfo, ""; got != want { + t.Fatalf("invalid progress info: %v, want %v", got, want) + } + if got, want := tsk.Description, "test-1"; got != want { t.Fatalf("invalid task description %v, want %v", got, want) } diff --git a/snapshot/snapshotfs/estimate.go b/snapshot/snapshotfs/estimate.go new file mode 100644 index 000000000..f08306201 --- /dev/null +++ b/snapshot/snapshotfs/estimate.go @@ -0,0 +1,152 @@ +package snapshotfs + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/fs/ignorefs" + "github.com/kopia/kopia/internal/units" + "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/snapshot" + "github.com/kopia/kopia/snapshot/policy" +) + +const maxExamplesPerBucket = 10 + +// SampleBucket keeps track of count and total size of files above in certain size range and +// includes small number of examples of such files. +type SampleBucket struct { + MinSize int64 `json:"minSize"` + Count int `json:"count"` + TotalSize int64 `json:"totalSize"` + Examples []string `json:"examples,omitempty"` +} + +func (b *SampleBucket) add(fname string, size int64) { + b.Count++ + b.TotalSize += size + + if len(b.Examples) < maxExamplesPerBucket { + b.Examples = append(b.Examples, fmt.Sprintf("%v - %v", fname, units.BytesStringBase10(size))) + } +} + +// SampleBuckets is a collection of buckets for interesting file sizes sorted in descending order. +type SampleBuckets []*SampleBucket + +func (b SampleBuckets) add(fname string, size int64) { + for _, bucket := range b { + if size >= bucket.MinSize { + bucket.add(fname, size) + break + } + } +} + +func makeBuckets() SampleBuckets { + return SampleBuckets{ + &SampleBucket{MinSize: 1e15}, + &SampleBucket{MinSize: 1e14}, + &SampleBucket{MinSize: 1e13}, + &SampleBucket{MinSize: 1e12}, + &SampleBucket{MinSize: 1e11}, + &SampleBucket{MinSize: 1e10}, + &SampleBucket{MinSize: 1e9}, + &SampleBucket{MinSize: 1e8}, + &SampleBucket{MinSize: 1e7}, + &SampleBucket{MinSize: 1e6}, + &SampleBucket{MinSize: 1e5}, + &SampleBucket{MinSize: 1e4}, + &SampleBucket{MinSize: 1e3}, + &SampleBucket{MinSize: 0}, + } +} + +// EstimateProgress must be provided by the caller of Estimate to report results. +type EstimateProgress interface { + Processing(ctx context.Context, dirname string) + Error(ctx context.Context, filename string, err error, isIgnored bool) + Stats(ctx context.Context, s *snapshot.Stats, includedFiles, excludedFiles SampleBuckets, excludedDirs []string, final bool) +} + +// Estimate walks the provided directory tree and invokes provided progress callback as it discovers +// items to be snapshotted. +func Estimate(ctx context.Context, rep repo.Repository, entry fs.Directory, policyTree *policy.Tree, progress EstimateProgress) error { + stats := &snapshot.Stats{} + ed := []string{} + ib := makeBuckets() + eb := makeBuckets() + + // report final stats just before returning + defer func() { + progress.Stats(ctx, stats, ib, eb, ed, true) + }() + + onIgnoredFile := func(relativePath string, e fs.Entry) { + if e.IsDir() { + if len(ed) < maxExamplesPerBucket { + ed = append(ed, relativePath) + } + + stats.ExcludedDirCount++ + + log(ctx).Debugf("excluded dir %v", relativePath) + } else { + log(ctx).Debugf("excluded file %v (%v)", relativePath, units.BytesStringBase10(e.Size())) + stats.ExcludedFileCount++ + stats.ExcludedTotalFileSize += e.Size() + eb.add(relativePath, e.Size()) + } + } + + entry = ignorefs.New(entry, policyTree, ignorefs.ReportIgnoredFiles(onIgnoredFile)) + + return estimate(ctx, ".", entry, policyTree, stats, ib, eb, &ed, progress) +} + +func estimate(ctx context.Context, relativePath string, entry fs.Entry, policyTree *policy.Tree, stats *snapshot.Stats, ib, eb SampleBuckets, ed *[]string, progress EstimateProgress) error { + // see if the context got canceled + select { + case <-ctx.Done(): + return ctx.Err() + + default: + } + + switch entry := entry.(type) { + case fs.Directory: + stats.TotalDirectoryCount++ + + progress.Processing(ctx, relativePath) + + children, err := entry.Readdir(ctx) + if err != nil { + isIgnored := policyTree.EffectivePolicy().ErrorHandlingPolicy.IgnoreDirectoryErrorsOrDefault(false) + + if isIgnored { + stats.IgnoredErrorCount++ + } else { + stats.ErrorCount++ + } + + progress.Error(ctx, relativePath, err, isIgnored) + } else { + for _, child := range children { + if err := estimate(ctx, filepath.Join(relativePath, child.Name()), child, policyTree.Child(child.Name()), stats, ib, eb, ed, progress); err != nil { + return err + } + } + } + + progress.Stats(ctx, stats, ib, eb, *ed, false) + + case fs.File: + ib.add(relativePath, entry.Size()) + stats.TotalFileCount++ + stats.TotalFileSize += entry.Size() + } + + return nil +} diff --git a/snapshot/snapshotfs/upload.go b/snapshot/snapshotfs/upload.go index cf8aa28c8..0b7bff4cf 100644 --- a/snapshot/snapshotfs/upload.go +++ b/snapshot/snapshotfs/upload.go @@ -1127,6 +1127,16 @@ func (u *Uploader) Upload( } } + scanWG.Add(1) + + go func() { + defer scanWG.Done() + + ds, _ := u.scanDirectory(scanctx, entry, policyTree) + + u.Progress.EstimatedDataSize(ds.numFiles, ds.totalFileSize) + }() + entry = ignorefs.New(entry, policyTree, ignorefs.ReportIgnoredFiles(func(fname string, md fs.Entry) { if md.IsDir() { u.Progress.ExcludedDir(fname) @@ -1136,17 +1146,6 @@ func (u *Uploader) Upload( u.stats.AddExcluded(md) })) - - scanWG.Add(1) - - go func() { - defer scanWG.Done() - - ds, _ := u.scanDirectory(scanctx, entry) - - u.Progress.EstimatedDataSize(ds.numFiles, ds.totalFileSize) - }() - s.RootEntry, err = u.uploadDirWithCheckpointing(ctx, entry, policyTree, previousDirs, sourceInfo) case fs.File: diff --git a/snapshot/snapshotfs/upload_scan.go b/snapshot/snapshotfs/upload_scan.go index 179927f4d..96cf872ca 100644 --- a/snapshot/snapshotfs/upload_scan.go +++ b/snapshot/snapshotfs/upload_scan.go @@ -3,9 +3,9 @@ import ( "context" - "github.com/pkg/errors" - "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/snapshot" + "github.com/kopia/kopia/snapshot/policy" ) type scanResults struct { @@ -13,45 +13,29 @@ type scanResults struct { totalFileSize int64 } +func (e *scanResults) Error(ctx context.Context, filename string, err error, isIgnored bool) {} + +func (e *scanResults) Processing(ctx context.Context, pathname string) {} + +func (e *scanResults) Stats(ctx context.Context, s *snapshot.Stats, includedFiles, excludedFiles SampleBuckets, excludedDirs []string, final bool) { + if final { + e.numFiles = int(s.TotalFileCount) + e.totalFileSize = s.TotalFileSize + } +} + +var _ EstimateProgress = (*scanResults)(nil) + // scanDirectory computes the number of files and their total size in a given directory recursively descending // into subdirectories. The scan teminates early as soon as the provided context is canceled. -func (u *Uploader) scanDirectory(ctx context.Context, dir fs.Directory) (scanResults, error) { +func (u *Uploader) scanDirectory(ctx context.Context, dir fs.Directory, policyTree *policy.Tree) (scanResults, error) { var res scanResults if u.disableEstimation { return res, nil } - entries, err := dir.Readdir(ctx) - if err != nil { - return res, errors.Wrap(err, "unable to read directory") - } + err := Estimate(ctx, u.repo, dir, policyTree, &res) - for _, e := range entries { - if err := ctx.Err(); err != nil { - // terminate early if context got canceled - // nolint:wrapcheck - return res, err - } - - switch e := e.(type) { - case fs.Directory: - dr, err := u.scanDirectory(ctx, e) - res.numFiles += dr.numFiles - res.totalFileSize += dr.totalFileSize - - if err != nil { - return res, err - } - - case fs.File: - res.numFiles++ - res.totalFileSize += e.Size() - - case fs.StreamingFile: - res.numFiles++ - } - } - - return res, nil + return res, err } diff --git a/snapshot/snapshotfs/upload_test.go b/snapshot/snapshotfs/upload_test.go index 83bbc4f63..20ff84b3f 100644 --- a/snapshot/snapshotfs/upload_test.go +++ b/snapshot/snapshotfs/upload_test.go @@ -547,7 +547,7 @@ func TestUploadWithCheckpointing(t *testing.T) { } } -func TestUploadScan(t *testing.T) { +func TestUploadScanStopsOnContextCancel(t *testing.T) { ctx := testlogging.Context(t) th := newUploadTestHarness(ctx, t) @@ -561,7 +561,7 @@ func TestUploadScan(t *testing.T) { cancel() }) - result, err := u.scanDirectory(scanctx, th.sourceDir) + result, err := u.scanDirectory(scanctx, th.sourceDir, nil) if !errors.Is(err, scanctx.Err()) { t.Fatalf("invalid scan error: %v", err) } @@ -571,6 +571,47 @@ func TestUploadScan(t *testing.T) { } } +func TestUploadScanIgnoresFiles(t *testing.T) { + ctx := testlogging.Context(t) + th := newUploadTestHarness(ctx, t) + + defer th.cleanup() + + u := NewUploader(th.repo) + + // set up a policy tree where that ignores some files. + policyTree := policy.BuildTree(map[string]*policy.Policy{ + ".": { + FilesPolicy: policy.FilesPolicy{ + IgnoreRules: []string{"f1"}, + }, + }, + }, policy.DefaultPolicy) + + // no policy + result1, err := u.scanDirectory(ctx, th.sourceDir, nil) + must(t, err) + + result2, err := u.scanDirectory(ctx, th.sourceDir, policyTree) + must(t, err) + + if result1.numFiles == 0 { + t.Fatalf("no files scanned") + } + + if result2.numFiles == 0 { + t.Fatalf("no files scanned") + } + + if got, want := result2.numFiles, result1.numFiles; got >= want { + t.Fatalf("expected lower number of files %v, wanted %v", got, want) + } + + if got, want := result2.totalFileSize, result1.totalFileSize; got >= want { + t.Fatalf("expected lower file size %v, wanted %v", got, want) + } +} + func TestUpload_VirtualDirectoryWithStreamingFile(t *testing.T) { ctx := testlogging.Context(t) th := newUploadTestHarness(ctx, t) @@ -625,3 +666,11 @@ func TestUpload_VirtualDirectoryWithStreamingFile(t *testing.T) { t.Fatalf("unexpected manifest file count: %v, want %v", got, want) } } + +func must(t *testing.T, err error) { + t.Helper() + + if err != nil { + t.Fatal(err) + } +}