mirror/kopia
1
0
Fork 0
mirror of https://github.com/kopia/kopia.git synced 2026-05-16 10:44:40 -04:00
Code Issues Packages Projects Releases Wiki Activity

New snapshot UX - streamlined snapshot creation and policy setting (#878)

Browse Source 
* uitask: added support for reporting string progress info

* server: report current directory as task progress

* snapshot: created reusable Estimate() method to be used during upload, cli estimate and via API

* cli: switched to snapshotfs.Estimate()

* server: added API to estimate snapshot size

* kopia-ui: fixed directory selector

* htmlui: streamlined new snapshot flow and cleaned up policy setting

See https://youtu.be/8p6csuoB3kg
This commit is contained in:
Jarek Kowalski
2021-03-10 23:04:55 -08:00
committed by GitHub
parent f04ec7ebed
commit 132e2eef50
24 changed files with 1052 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
app/public/electron.js
View File

@@ -32,6 +32,7 @@ function showRepoWindow(repoID) {
autoHideMenuBar: true,
webPreferences: {
nodeIntegration: true,
enableRemoteModule: true,
},
})

168
cli/command_snapshot_estimate.go
View File

@@ -9,11 +9,11 @@
"github.com/pkg/errors"
"github.com/kopia/kopia/fs"
"github.com/kopia/kopia/fs/ignorefs"
"github.com/kopia/kopia/internal/units"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/snapshot"
"github.com/kopia/kopia/snapshot/policy"
"github.com/kopia/kopia/snapshot/snapshotfs"
)
var (
@@ -24,44 +24,32 @@
snapshotEstimateUploadSpeed = snapshotEstimate.Flag("upload-speed", "Upload speed to use for estimation").Default("10").PlaceHolder("mbit/s").Float64()
)
const maxExamplesPerBucket = 10
type bucket struct {
MinSize int64 `json:"minSize"`
Count int `json:"count"`
TotalSize int64 `json:"totalSize"`
Examples []string `json:"examples,omitempty"`
type estimateProgress struct {
stats snapshot.Stats
included snapshotfs.SampleBuckets
excluded snapshotfs.SampleBuckets
excludedDirs []string
}
func (b *bucket) add(fname string, size int64) {
b.Count++
b.TotalSize += size
if len(b.Examples) < maxExamplesPerBucket {
b.Examples = append(b.Examples, fmt.Sprintf("%v - %v", fname, units.BytesStringBase10(size)))
func (ep *estimateProgress) Processing(ctx context.Context, dirname string) {
if !*snapshotEstimateQuiet {
log(ctx).Infof("Analyzing %v...", dirname)
}
}
type buckets []*bucket
func (b buckets) add(fname string, size int64) {
for _, bucket := range b {
if size >= bucket.MinSize {
bucket.add(fname, size)
break
}
func (ep *estimateProgress) Error(ctx context.Context, filename string, err error, isIgnored bool) {
if isIgnored {
log(ctx).Warningf("Ignored error in %v: %v", filename, err)
} else {
log(ctx).Errorf("Error in %v: %v", filename, err)
}
}
func makeBuckets() buckets {
return buckets{
&bucket{MinSize: 1e15},
&bucket{MinSize: 1e12},
&bucket{MinSize: 1e9},
&bucket{MinSize: 1e6},
&bucket{MinSize: 1e3},
&bucket{MinSize: 0},
}
func (ep *estimateProgress) Stats(ctx context.Context, st *snapshot.Stats, included, excluded snapshotfs.SampleBuckets, excludedDirs []string, final bool) {
ep.stats = *st
ep.included = included
ep.excluded = excluded
ep.excludedDirs = excludedDirs
}
func runSnapshotEstimateCommand(ctx context.Context, rep repo.Repository) error {
@@ -76,51 +64,53 @@ func runSnapshotEstimateCommand(ctx context.Context, rep repo.Repository) error
UserName: rep.ClientOptions().Username,
}
var stats snapshot.Stats
ib := makeBuckets()
eb := makeBuckets()
onIgnoredFile := func(relativePath string, e fs.Entry) {
eb.add(relativePath, e.Size())
if e.IsDir() {
stats.ExcludedDirCount++
log(ctx).Infof("excluded dir %v", relativePath)
} else {
log(ctx).Infof("excluded file %v (%v)", relativePath, units.BytesStringBase10(e.Size()))
stats.ExcludedFileCount++
stats.ExcludedTotalFileSize += e.Size()
}
}
entry, err := getLocalFSEntry(ctx, path)
if err != nil {
return err
}
if dir, ok := entry.(fs.Directory); ok {
policyTree, err := policy.TreeForSource(ctx, rep, sourceInfo)
if err != nil {
return errors.Wrapf(err, "error creating policy tree for %v", sourceInfo)
}
entry = ignorefs.New(dir, policyTree, ignorefs.ReportIgnoredFiles(onIgnoredFile))
dir, ok := entry.(fs.Directory)
if !ok {
return errors.Errorf("invalid path: '%s': must be a directory", path)
}
if err := estimate(ctx, ".", entry, &stats, ib); err != nil {
return err
var ep estimateProgress
policyTree, err := policy.TreeForSource(ctx, rep, sourceInfo)
if err != nil {
return errors.Wrapf(err, "error creating policy tree for %v", sourceInfo)
}
fmt.Printf("Snapshot includes %v files, total size %v\n", stats.TotalFileCount, units.BytesStringBase10(stats.TotalFileSize))
showBuckets(ib)
if err := snapshotfs.Estimate(ctx, rep, dir, policyTree, &ep); err != nil {
return errors.Wrap(err, "error estimating")
}
fmt.Printf("Snapshot includes %v files, total size %v\n", ep.stats.TotalFileCount, units.BytesStringBase10(ep.stats.TotalFileSize))
showBuckets(ep.included, *snapshotEstimateShowFiles)
fmt.Println()
fmt.Printf("Snapshot excludes %v directories and %v files with total size %v\n", stats.ExcludedDirCount, stats.ExcludedFileCount, units.BytesStringBase10(stats.ExcludedTotalFileSize))
showBuckets(eb)
if ep.stats.ExcludedFileCount > 0 {
fmt.Printf("Snapshot excludes %v files, total size %v\n", ep.stats.ExcludedFileCount, ep.stats.ExcludedTotalFileSize)
showBuckets(ep.excluded, true)
} else {
fmt.Printf("Snapshots excludes no files.\n")
}
megabits := float64(stats.TotalFileSize) * 8 / 1000000 //nolint:gomnd
if ep.stats.ExcludedDirCount > 0 {
fmt.Printf("Snapshots excludes %v directories. Examples:\n", ep.stats.ExcludedDirCount)
for _, ed := range ep.excludedDirs {
fmt.Printf(" - %v\n", ed)
}
} else {
fmt.Printf("Snapshots excludes no directories.\n")
}
if ep.stats.ErrorCount > 0 {
fmt.Printf("Encountered %v errors.\n", ep.stats.ErrorCount)
}
megabits := float64(ep.stats.TotalFileSize) * 8 / 1000000 //nolint:gomnd
seconds := megabits / *snapshotEstimateUploadSpeed
fmt.Println()
@@ -129,49 +119,35 @@ func runSnapshotEstimateCommand(ctx context.Context, rep repo.Repository) error
return nil
}
func showBuckets(b buckets) {
for _, bucket := range b {
func showBuckets(buckets snapshotfs.SampleBuckets, showFiles bool) {
for i, bucket := range buckets {
if bucket.Count == 0 {
continue
}
fmt.Printf(" with size over %-5v: %7v files, total size %v\n", units.BytesStringBase10(bucket.MinSize), bucket.Count, units.BytesStringBase10(bucket.TotalSize))
var sizeRange string
if *snapshotEstimateShowFiles {
if i == 0 {
sizeRange = fmt.Sprintf("< %-6v",
units.BytesStringBase10(bucket.MinSize))
} else {
sizeRange = fmt.Sprintf("%-6v...%6v",
units.BytesStringBase10(bucket.MinSize),
units.BytesStringBase10(buckets[i-1].MinSize))
}
fmt.Printf("%18v: %7v files, total size %v\n",
sizeRange,
bucket.Count, units.BytesStringBase10(bucket.TotalSize))
if showFiles {
for _, sample := range bucket.Examples {
fmt.Printf(" %v\n", sample)
fmt.Printf(" - %v\n", sample)
}
}
}
}
func estimate(ctx context.Context, relativePath string, entry fs.Entry, stats *snapshot.Stats, ib buckets) error {
switch entry := entry.(type) {
case fs.Directory:
if !*snapshotEstimateQuiet {
log(ctx).Infof("Scanning %v\n", relativePath)
}
children, err := entry.Readdir(ctx)
if err != nil {
return errors.Wrap(err, "unable to read directory")
}
for _, child := range children {
if err := estimate(ctx, filepath.Join(relativePath, child.Name()), child, stats, ib); err != nil {
return err
}
}
case fs.File:
ib.add(relativePath, entry.Size())
stats.TotalFileCount++
stats.TotalFileSize += entry.Size()
}
return nil
}
func init() {
snapshotEstimate.Action(repositoryReaderAction(runSnapshotEstimateCommand))
}

21
htmlui/src/App.css
View File

@@ -113,8 +113,8 @@ div.tab-body {
}
.logs-table {
border: 1px solid #ccc;
background-color: #f0f0f0;
border: 1px solid #ccd;
background-color: #f0f0ff;
font-family: monospace;
font-size: 12px;
font-weight: bold;
@@ -140,4 +140,21 @@ div.tab-body {
height: 40px;
font-size: 125%;
vertical-align: text-bottom;
}
.nested-task {
padding: 10px;
padding-left: 40px;
}
.estimate-results {
margin: 10px;
margin-left: 15px;
padding: 15px;
border: 1px solid #ccc;
background-color: #f8f8ff;
}
.list-actions {
min-height: 40px;
}

10
htmlui/src/App.js
View File

@@ -15,6 +15,8 @@ import { SnapshotsTable } from "./SnapshotsTable";
import { SourcesTable } from "./SourcesTable";
import { TaskDetails } from './TaskDetails';
import { TasksTable } from './TasksTable';
import { NewSnapshot } from './NewSnapshot';
import { PolicyEditor } from './PolicyEditor';
function useInterval(callback, delay) {
const savedCallback = useRef();
@@ -40,8 +42,8 @@ function App() {
const [runningTaskCount, setRunningTaskCount] = useState(0);
useInterval(() => {
axios.get('/api/v1/tasks-summary').then(result => {
setRunningTaskCount(result.data["RUNNING"] || 0);
axios.get('/api/v1/tasks-summary').then(result => {
setRunningTaskCount(result.data["RUNNING"] || 0);
}).catch(error => {
setRunningTaskCount(-1);
});
@@ -57,7 +59,7 @@ function App() {
<NavLink className="nav-link" activeClassName="active" to="/snapshots">Snapshots</NavLink>
<NavLink className="nav-link" activeClassName="active" to="/policies">Policies</NavLink>
<NavLink className="nav-link" activeClassName="active" to="/tasks">Tasks <>
{runningTaskCount > 0 && <>({runningTaskCount})</>}
{runningTaskCount > 0 && <>({runningTaskCount})</>}
</>
</NavLink>
<NavLink className="nav-link" activeClassName="active" to="/repo">Repository</NavLink>
@@ -67,10 +69,12 @@ function App() {
<Container fluid>
<Switch>
<Route path="/snapshots/new" component={NewSnapshot} />
<Route path="/snapshots/single-source/" component={SnapshotsTable} />
<Route path="/snapshots/dir/:oid/restore" component={BeginRestore} />
<Route path="/snapshots/dir/:oid" component={DirectoryObject} />
<Route path="/snapshots" component={SourcesTable} />
<Route path="/policies/edit/" component={PolicyEditor} />
<Route path="/policies" component={PoliciesTable} />
<Route path="/tasks/:tid" component={TaskDetails} />
<Route path="/tasks" component={TasksTable} />

186
htmlui/src/NewSnapshot.js Normal file
View File

@@ -0,0 +1,186 @@
import { faWindowClose } from '@fortawesome/free-solid-svg-icons';
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import axios from 'axios';
import React, { Component } from 'react';
import Button from 'react-bootstrap/Button';
import Col from 'react-bootstrap/Col';
import Form from 'react-bootstrap/Form';
import { handleChange, validateRequiredFields } from './forms';
import { PolicyEditor } from './PolicyEditor';
import { TaskDetails } from './TaskDetails';
import { cancelTask, DirectorySelector, GoBackButton, redirectIfNotConnected } from './uiutil';
export class NewSnapshot extends Component {
constructor() {
super();
this.state = {
path: "",
estimateTaskID: null,
estimateTaskVisible: false,
policyEditorVisibleFor: "n/a",
localUsername: null,
};
this.policyEditorRef = React.createRef();
this.handleChange = handleChange.bind(this);
this.estimate = this.estimate.bind(this);
this.snapshotNow = this.snapshotNow.bind(this);
this.cancelEstimate = this.cancelEstimate.bind(this);
this.togglePolicyEditor = this.togglePolicyEditor.bind(this);
}
componentDidMount() {
axios.get('/api/v1/sources').then(result => {
this.setState({
localUsername: result.data.localUsername,
localHost: result.data.localHost,
});
}).catch(error => {
redirectIfNotConnected(error);
});
}
togglePolicyEditor() {
if (!this.state.path) {
return;
}
if (this.state.policyEditorVisibleFor === this.state.path) {
this.setState({
policyEditorVisibleFor: "n/a",
});
} else {
this.setState({
policyEditorVisibleFor: this.state.path,
});
}
}
estimate(e) {
e.preventDefault();
if (!validateRequiredFields(this, ["path"])) {
return;
}
let req = {
root: this.state.path,
}
axios.post('/api/v1/estimate', req).then(result => {
this.setState({
estimateTaskID: result.data.id,
estimatingPath: result.data.description,
estimateTaskVisible: true,
didEstimate: false,
})
}).catch(error => {
if (error.response.data) {
alert(JSON.stringify(error.response.data));
} else {
alert('failed');
}
});
}
cancelEstimate() {
this.setState({ estimateTaskVisible: false });
cancelTask(this.state.estimateTaskID);
}
snapshotNow(e) {
e.preventDefault();
if (!this.state.path) {
alert('Must specify directory to snapshot.');
return
}
axios.post('/api/v1/sources', {
path: this.state.path,
createSnapshot: true,
}).then(result => {
this.props.history.goBack();
}).catch(error => {
if (error.response) {
alert('Error: ' + error.response.data.error + " (" + error.response.data.code + ")");
return
}
this.setState({
error,
isLoading: false
});
});
}
render() {
return <>
<Form.Row>
<Form.Group>
<GoBackButton onClick={this.props.history.goBack} />
</Form.Group>
&nbsp;&nbsp;&nbsp;<h4>New Snapshot</h4>
</Form.Row>
<br />
<Form.Row>
<Col>
<Form.Group>
<DirectorySelector onDirectorySelected={p => this.setState({ path: p })} autoFocus placeholder="enter path to snapshot" name="path" value={this.state.path} onChange={this.handleChange}
readOnly={this.state.policyEditorVisible || this.state.estimateTaskVisible} />
<Form.Text>
Click the <code>Estimate</code> button to estimate the size of the snapshot.
To specify frequency of snapshots or exclude some files, click <code>Policy</code>.
</Form.Text>
</Form.Group>
</Col>
<Col xs="auto">
<Button
size="sm"
disabled={!this.state.path}
title="Edit Policy"
variant="primary"
onClick={this.estimate}>Estimate</Button>
</Col>
<Col xs="auto">
<Button
size="sm"
disabled={!this.state.path}
title="Estimate"
variant="primary"
onClick={this.togglePolicyEditor}>Policy</Button>
</Col>
</Form.Row>
{this.state.estimateTaskID && this.state.estimateTaskVisible &&
<div className="estimate-results">
<h5>
<Button onClick={this.cancelEstimate} title="stop estimation" variant="light" size="sm"><FontAwesomeIcon color="#888" icon={faWindowClose} /></Button>&nbsp;
Estimate Snapshot Size for {this.state.estimatingPath}...
</h5>
<TaskDetails taskID={this.state.estimateTaskID} hideDescription={true} showZeroCounters={true} />
</div>
}
<br />
{this.state.path && this.state.policyEditorVisibleFor === this.state.path && <Form.Row>
<Col xs={12}>
<PolicyEditor ref={this.policyEditorRef}
embedded
host={this.state.localHost}
userName={this.state.localUsername}
path={this.state.path}
close={this.togglePolicyEditor} />
</Col>
</Form.Row>}
<Form.Row>
<Button size="sm"
disabled={!this.state.path}
title="Snapshot Now"
variant="primary"
onClick={this.snapshotNow}
>Snapshot Now</Button>
</Form.Row>
</>;
}
}

198
htmlui/src/PoliciesTable.js
View File

@@ -1,13 +1,23 @@
import { faUserFriends } from '@fortawesome/free-solid-svg-icons';
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import axios from 'axios';
import React, { Component } from 'react';
import Badge from 'react-bootstrap/Badge';
import Button from 'react-bootstrap/Button';
import Col from 'react-bootstrap/Col';
import Dropdown from 'react-bootstrap/Dropdown';
import Form from 'react-bootstrap/Form';
import { handleChange, OptionalField, RequiredField, validateRequiredFields } from './forms';
import Row from 'react-bootstrap/Row';
import { handleChange } from './forms';
import { PolicyEditor } from './PolicyEditor';
import MyTable from './Table';
import { redirectIfNotConnected } from './uiutil';
import { compare, DirectorySelector, isAbsolutePath, ownerName, redirectIfNotConnected } from './uiutil';
const localPolicies = "Local Policies"
const allPolicies = "All Policies"
const globalPolicy = "Global Policy"
const perUserPolicies = "Per-User Policies"
const perHostPolicies = "Per-Host Policies"
export class PoliciesTable extends Component {
constructor() {
super();
@@ -16,16 +26,16 @@ export class PoliciesTable extends Component {
isLoading: false,
error: null,
editorTarget: null,
selectedRowIsNew: false,
targetHost: "*",
targetUsername: "*",
targetPath: "*",
selectedOwner: localPolicies,
policyPath: "",
sources: [],
};
this.editorClosed = this.editorClosed.bind(this);
this.setNewPolicy = this.setNewPolicy.bind(this);
this.editPolicyForPath = this.editPolicyForPath.bind(this);
this.handleChange = handleChange.bind(this);
this.fetchPolicies = this.fetchPolicies.bind(this);
this.fetchSourcesWithoutSpinner = this.fetchSourcesWithoutSpinner.bind(this);
}
componentDidMount() {
@@ -34,6 +44,20 @@ export class PoliciesTable extends Component {
});
this.fetchPolicies();
this.fetchSourcesWithoutSpinner();
}
sync() {
this.fetchPolicies();
axios.post('/api/v1/repo/sync', {}).then(result => {
this.fetchSourcesWithoutSpinner();
}).catch(error => {
this.setState({
error,
isLoading: false
});
});
}
fetchPolicies() {
@@ -51,6 +75,25 @@ export class PoliciesTable extends Component {
});
}
fetchSourcesWithoutSpinner() {
axios.get('/api/v1/sources').then(result => {
this.setState({
localSourceName: result.data.localUsername + "@" + result.data.localHost,
localUsername: result.data.localUsername,
localHost: result.data.localHost,
multiUser: result.data.multiUser,
sources: result.data.sources,
isLoading: false,
});
}).catch(error => {
redirectIfNotConnected(error);
this.setState({
error,
isLoading: false
});
});
}
editorClosed() {
this.setState({
editorTarget: null,
@@ -58,34 +101,33 @@ export class PoliciesTable extends Component {
this.fetchPolicies();
}
setNewPolicy() {
if (!validateRequiredFields(this, ["targetHost"])) {
editPolicyForPath(e) {
e.preventDefault();
if (!this.state.policyPath) {
return;
}
function fixInput(a) {
if (!a) {
return "";
}
// allow both * and empty string to indicate "all"
if (a === "*") {
return ""
}
return a;
if (!isAbsolutePath(this.state.policyPath)) {
alert("Policies can only be defined for absolute paths.");
return;
}
this.setState({
editorTarget: {
userName: fixInput(this.state.targetUsername),
host: this.state.targetHost,
path: fixInput(this.state.targetPath),
userName: this.state.localUsername,
host: this.state.localHost,
path: this.state.policyPath,
},
selectedRowIsNew: true,
})
}
selectOwner(h) {
this.setState({
selectedOwner: h,
});
}
policySummary(p) {
function isEmpty(obj) {
for (var key in obj) {
@@ -117,13 +159,63 @@ export class PoliciesTable extends Component {
}
render() {
const { items, isLoading, error } = this.state;
let { items, sources, isLoading, error } = this.state;
if (error) {
return <p>{error.message}</p>;
}
if (isLoading) {
return <p>Loading ...</p>;
}
let uniqueOwners = sources.reduce((a, d) => {
const owner = ownerName(d.source);
if (!a.includes(owner)) { a.push(owner); }
return a;
}, []);
uniqueOwners.sort();
switch (this.state.selectedOwner) {
case allPolicies:
// do nothing;
break;
case globalPolicy:
items = items.filter(x => !x.target.userName && !x.target.host && !x.target.path);
break;
case localPolicies:
items = items.filter(x => ownerName(x.target) === this.state.localSourceName && x.target.path.startsWith(this.state.policyPath));
break;
case perUserPolicies:
items = items.filter(x => !!x.target.userName && !!x.target.host && !x.target.path);
break;
case perHostPolicies:
items = items.filter(x => !x.target.userName && !!x.target.host && !x.target.path);
break;
default:
items = items.filter(x => ownerName(x.target) === this.state.selectedOwner);
break;
};
items.sort((l,r) => {
const hc = compare(l.target.host,r.target.host);
if (hc) {
return hc;
}
const uc = compare(l.target.userName,r.target.userName);
if (uc) {
return uc;
}
return compare(l.target.path,r.target.path);
});
const columns = [{
Header: 'Username',
accessor: x => x.target.userName || "*",
@@ -150,23 +242,49 @@ export class PoliciesTable extends Component {
}]
return <div className="padded">
<div className={this.state.editorTarget ? "hidden" : "normal"}>
<MyTable data={items} columns={columns} />
</div>
{!this.state.editorTarget && <div className="list-actions">
<Form onSubmit={this.editPolicyForPath}>
<Row>
<Col xs="auto">
<Dropdown>
<Dropdown.Toggle size="sm" variant="outline-primary" id="dropdown-basic">
<FontAwesomeIcon icon={faUserFriends} />&nbsp;{this.state.selectedOwner}
</Dropdown.Toggle>
<div className={this.state.editorTarget ? "hidden" : "normal"}>
<hr />
<div className="new-policy-panel"><Form>
<Form.Row>
{OptionalField(this, "Target Username", "targetUsername", {}, "Specify * all empty to target all users")}
{RequiredField(this, "Target Host", "targetHost", {})}
{OptionalField(this, "Target Path", "targetPath", {}, "Specify * all empty to target all filesystem paths")}
</Form.Row>
<Button size="sm" variant="primary" onClick={this.setNewPolicy}>Set New Policy</Button>
<Dropdown.Menu>
<Dropdown.Item onClick={() => this.selectOwner(localPolicies)}>{localPolicies}</Dropdown.Item>
<Dropdown.Item onClick={() => this.selectOwner(allPolicies)}>{allPolicies}</Dropdown.Item>
<Dropdown.Divider />
<Dropdown.Item onClick={() => this.selectOwner(globalPolicy)}>{globalPolicy}</Dropdown.Item>
<Dropdown.Item onClick={() => this.selectOwner(perUserPolicies)}>{perUserPolicies}</Dropdown.Item>
<Dropdown.Item onClick={() => this.selectOwner(perHostPolicies)}>{perHostPolicies}</Dropdown.Item>
<Dropdown.Divider />
{uniqueOwners.map(v => <Dropdown.Item key={v} onClick={() => this.selectOwner(v)}>{v}</Dropdown.Item>)}
</Dropdown.Menu>
</Dropdown>
</Col>
{this.state.selectedOwner === localPolicies ? <>
<Col>
<DirectorySelector autoFocus onDirectorySelected={p => this.setState({ policyPath: p })}
placeholder="enter directory to find or set policy"
name="policyPath" value={this.state.policyPath} onChange={this.handleChange} />
</Col>
<Col xs="auto">
<Button disabled={!this.state.policyPath} size="sm" type="submit" onClick={this.editPolicyForPath}>Set Policy</Button>
</Col>
</> : <Col />}
</Row>
</Form>
</div>
</div>
{this.state.editorTarget && <PolicyEditor host={this.state.editorTarget.host} userName={this.state.editorTarget.userName} path={this.state.editorTarget.path} close={this.editorClosed} isNew={this.state.selectedRowIsNew} />}
</div>}
{items.length > 0 ? <div className={this.state.editorTarget ? "hidden" : "normal"}>
<p>Found {items.length} policies matching criteria.</p>
<MyTable data={items} columns={columns} />
</div> : ((this.state.selectedOwner === localPolicies && this.state.policyPath) ? <p>
No policy found for directory <code>{this.state.policyPath}</code>. Click <b>Set Policy</b> to define it.
</p> : <p>No policies found.</p>)}
{this.state.editorTarget && <PolicyEditor host={this.state.editorTarget.host} userName={this.state.editorTarget.userName} path={this.state.editorTarget.path} close={this.editorClosed} />}
</div>;
}
}

79
htmlui/src/PolicyEditor.js
View File

@@ -3,26 +3,27 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import axios from 'axios';
import React, { Component } from 'react';
import Button from 'react-bootstrap/Button';
import Row from 'react-bootstrap/Row';
import Col from 'react-bootstrap/Col';
import Form from 'react-bootstrap/Form';
import Tab from 'react-bootstrap/Tab';
import Tabs from 'react-bootstrap/Tabs';
import { handleChange, OptionalBoolean, OptionalNumberField, RequiredBoolean, stateProperty, StringList } from './forms';
function sourceDisplayName(s) {
function policyTypeName(s) {
if (!s.host && !s.userName) {
return "global"
return "Global Policy"
}
if (!s.userName) {
return "host " + s.host;
return "Per-Host Policy";
}
if (!s.path) {
return "user " + s.userName + "@" + s.host;
return "Per-User Policy";
}
return "user " + s.userName + "@" + s.host + " path '" + s.path + "'";
return "Directory Policy";
}
export class PolicyEditor extends Component {
@@ -57,27 +58,30 @@ export class PolicyEditor extends Component {
}
fetchPolicy(props) {
if (props.isNew) {
this.setState({
isLoading: false,
policy: {},
});
return;
}
axios.get(this.snapshotURL(props)).then(result => {
this.setState({
isLoading: false,
policy: result.data,
});
}).catch(error => this.setState({
error: error,
isLoading: false
}));
}).catch(error => {
if (error.response && error.response.data.code !== "NOT_FOUND") {
this.setState({
error: error,
isLoading: false
})
} else {
this.setState({
policy: {},
isNew: true,
isLoading: false
})
}
});
}
saveChanges() {
saveChanges(e) {
e.preventDefault()
function removeEmpty(l) {
if (!l) {
return l;
@@ -119,7 +123,7 @@ export class PolicyEditor extends Component {
axios.put(this.snapshotURL(this.props), policy).then(result => {
this.props.close();
}).catch(error => {
alert('Error saving policy: ' + error);
alert('Error saving policy: ' + JSON.stringify(error));
});
}
@@ -152,8 +156,29 @@ export class PolicyEditor extends Component {
}
return <div className="padded">
<h3><Button size="sm" variant="outline-secondary" onClick={this.props.close} ><FontAwesomeIcon icon={faChevronLeft} /> Return </Button> {this.props.isNew && "New "}Policy: {sourceDisplayName(this.props)}</h3>
{!this.props.embedded && <h4><Button size="sm" variant="outline-secondary" onClick={this.props.close} ><FontAwesomeIcon icon={faChevronLeft} /> Return </Button>
&nbsp;&nbsp;{policyTypeName(this.props)}</h4>}
<Form onSubmit={this.saveChanges}>
{!this.props.embedded && <Row>
<Col xs="2">
<Form.Group>
<Form.Label className="required">Target User</Form.Label>
<Form.Control size="sm" type="text" readOnly value={this.props.userName || "<any user>"} />
</Form.Group>
</Col>
<Col xs="2">
<Form.Group>
<Form.Label className="required">Target Host</Form.Label>
<Form.Control size="sm" type="text" readOnly value={this.props.host || "<any host>"} />
</Form.Group>
</Col>
<Col xs="8">
<Form.Group>
<Form.Label className="required">Target Directory Path</Form.Label>
<Form.Control size="sm" type="text" readOnly value={this.props.path || "<any path>"} />
</Form.Group>
</Col>
</Row>}
<Tabs defaultActiveKey="retention">
<Tab eventKey="retention" title="Retention">
<div className="tab-body">
@@ -209,7 +234,7 @@ export class PolicyEditor extends Component {
name="policy.compression.compressorName"
onChange={this.handleChange}
value={stateProperty(this, "policy.compression.compressorName")}>
<option value="">(none)</option>
<option value="">(none)</option>
{this.state.algorithms && this.state.algorithms.compression.map(x => <option key={x} value={x}>{x}</option>)}
</Form.Control>
</Form.Group>
@@ -238,14 +263,18 @@ export class PolicyEditor extends Component {
{RequiredBoolean(this, "Disable Parent Policy Evaluation (prevents any parent policies from affecting this directory and subdirectories)", "policy.noParent")}
</Form.Row>
<Button size="sm" variant="success" onClick={this.saveChanges}>Save Policy</Button>
{!this.props.isNew && <>&nbsp;
<Button size="sm" variant="success" type="submit" onClick={this.saveChanges}>Save Policy</Button>
{!this.state.isNew && <>&nbsp;
<Button size="sm" variant="danger" disabled={this.isGlobal()} onClick={this.deletePolicy}>Delete Policy</Button>
</>}
{!this.props.embedded && <>
<hr />
<h5>JSON representation</h5>
<pre className="debug-json">{JSON.stringify(this.state.policy, null, 4)}
</pre>
</>}
</Form>
</div>;
}
}

176
htmlui/src/SourcesTable.js
View File

@@ -5,12 +5,8 @@ import moment from 'moment';
import React, { Component } from 'react';
import Badge from 'react-bootstrap/Badge';
import Button from 'react-bootstrap/Button';
import ButtonGroup from 'react-bootstrap/ButtonGroup';
import ButtonToolbar from 'react-bootstrap/ButtonToolbar';
import Col from 'react-bootstrap/Col';
import Dropdown from 'react-bootstrap/Dropdown';
import DropdownButton from 'react-bootstrap/DropdownButton';
import FormControl from 'react-bootstrap/FormControl';
import InputGroup from 'react-bootstrap/InputGroup';
import Row from 'react-bootstrap/Row';
import Spinner from 'react-bootstrap/Spinner';
import { Link } from 'react-router-dom';
@@ -37,13 +33,8 @@ export class SourcesTable extends Component {
this.sync = this.sync.bind(this);
this.fetchSourcesWithoutSpinner = this.fetchSourcesWithoutSpinner.bind(this);
this.selectDirectory = this.selectDirectory.bind(this);
this.handleChange = handleChange.bind(this);
this.snapshotOnce = this.snapshotOnce.bind(this);
this.snapshotEveryDay = this.snapshotEveryDay.bind(this);
this.snapshotEveryHour = this.snapshotEveryHour.bind(this);
this.createPolicy = this.createPolicy.bind(this);
this.cancelSnapshot = this.cancelSnapshot.bind(this);
this.startSnapshot = this.startSnapshot.bind(this);
}
@@ -94,87 +85,6 @@ export class SourcesTable extends Component {
});
}
selectDirectory() {
// populated in 'preload.js' in Electron
if (!window.require) {
alert('Directory selection is not supported in a web browser.\n\nPlease enter path manually.');
return;
}
const { dialog } = window.require('electron').remote;
try {
let dir = dialog.showOpenDialogSync({
properties: ['openDirectory']
});
if (dir) {
this.setState({
selectedDirectory: dir[0],
});
}
} catch (e) {
window.alert('Error: ' + e);
}
}
createSource(request) {
if (!this.state.selectedDirectory) {
alert('Must specify directory to snapshot.');
return
}
axios.post('/api/v1/sources', request).then(result => {
this.fetchSourcesWithoutSpinner();
}).catch(error => {
if (error.response) {
alert('Error: ' + error.response.data.error + " (" + error.response.data.code + ")");
return
}
this.setState({
error,
isLoading: false
});
});
}
snapshotOnce() {
this.createSource({
path: this.state.selectedDirectory,
createSnapshot: true,
initialPolicy: {
}
});
}
snapshotEveryDay() {
this.createSource({
path: this.state.selectedDirectory,
createSnapshot: true,
initialPolicy: {
scheduling: { intervalSeconds: 86400 },
}
});
}
snapshotEveryHour() {
this.createSource({
path: this.state.selectedDirectory,
createSnapshot: true,
initialPolicy: {
scheduling: { intervalSeconds: 3600 },
}
});
}
createPolicy() {
this.createSource({
path: this.state.selectedDirectory,
createSnapshot: false,
initialPolicy: {
scheduling: { intervalSeconds: 3600 },
}
});
}
statusCell(x, parent) {
switch (x.cell.value) {
@@ -216,7 +126,7 @@ export class SourcesTable extends Component {
return <>
<Spinner animation="border" variant="primary" size="sm" title={title} />&nbsp;{totals}
&nbsp;
{x.row.original.currentTask && <Link to={"/tasks/"+x.row.original.currentTask}>Details</Link>}
{x.row.original.currentTask && <Link to={"/tasks/" + x.row.original.currentTask}>Details</Link>}
</>;
default:
@@ -326,63 +236,35 @@ export class SourcesTable extends Component {
Cell: x => this.statusCell(x, this),
}]
const selectSupported = !!window.require;
return <div className="padded">
{this.state.multiUser && <ButtonToolbar className="float-sm-right">
&nbsp;
<ButtonGroup>
<Dropdown>
<Dropdown.Toggle size="sm" variant="outline-primary" id="dropdown-basic">
<FontAwesomeIcon icon={faUserFriends} />&nbsp;{this.state.selectedOwner}
</Dropdown.Toggle>
<div class="list-actions">
<Row>
{this.state.multiUser && <><Col xs="auto">
<Dropdown>
<Dropdown.Toggle size="sm" variant="outline-primary" id="dropdown-basic">
<FontAwesomeIcon icon={faUserFriends} />&nbsp;{this.state.selectedOwner}
</Dropdown.Toggle>
<Dropdown.Menu>
<Dropdown.Item onClick={() => this.selectOwner(localSnapshots)}>{localSnapshots}</Dropdown.Item>
<Dropdown.Item onClick={() => this.selectOwner(allSnapshots)}>{allSnapshots}</Dropdown.Item>
<Dropdown.Divider />
{uniqueOwners.map(v => <Dropdown.Item key={v} onClick={() => this.selectOwner(v)}>{v}</Dropdown.Item>)}
</Dropdown.Menu>
</Dropdown>
</ButtonGroup>
&nbsp;
<ButtonGroup>
<Button size="sm" variant="primary"><FontAwesomeIcon icon={faSync} /></Button>
</ButtonGroup>
</ButtonToolbar>}
<ButtonToolbar>
<InputGroup>
<FormControl
id="snapshot-path"
placeholder="Enter source path to create new snapshot"
name="selectedDirectory"
value={this.state.selectedDirectory}
onChange={this.handleChange}
size="sm"
/>
{selectSupported && <Button as={InputGroup.Prepend}
title="Snapshot"
variant="primary"
id="input-group-dropdown-2"
onClick={this.selectDirectory}>...</Button>}
</InputGroup>
&nbsp;
<DropdownButton
as={InputGroup.Append}
variant="success"
title="New Snapshot"
id="dropdown1"
size="sm">
<Dropdown.Item href="#" onClick={this.snapshotOnce}>Snapshot Once</Dropdown.Item>
<Dropdown.Item href="#" onClick={this.snapshotEveryHour}>Snapshot Every Hour</Dropdown.Item>
<Dropdown.Item href="#" onClick={this.snapshotEveryDay}>Snapshot Every Day</Dropdown.Item>
{/* <Dropdown.Item href="#" onClick={this.createPolicy}>Create Policy</Dropdown.Item> */}
</DropdownButton>
</ButtonToolbar>
<hr />
<Row>
<MyTable data={sources} columns={columns} />
</Row>
<Dropdown.Menu>
<Dropdown.Item onClick={() => this.selectOwner(localSnapshots)}>{localSnapshots}</Dropdown.Item>
<Dropdown.Item onClick={() => this.selectOwner(allSnapshots)}>{allSnapshots}</Dropdown.Item>
<Dropdown.Divider />
{uniqueOwners.map(v => <Dropdown.Item key={v} onClick={() => this.selectOwner(v)}>{v}</Dropdown.Item>)}
</Dropdown.Menu>
</Dropdown>
</Col></>}
<Col xs="auto">
<Button size="sm" variant="success" href="/snapshots/new">New Snapshot</Button>
</Col>
<Col>
</Col>
<Col xs="auto">
<Button size="sm" variant="primary"><FontAwesomeIcon icon={faSync} /></Button>
</Col>
</Row>
</div>
<MyTable data={sources} columns={columns} />
</div>;
}
}

80
htmlui/src/TaskDetails.js
View File

@@ -1,5 +1,5 @@
import { faStopCircle } from '@fortawesome/free-solid-svg-icons';
import { faChevronCircleDown, faChevronCircleUp, faStopCircle } from '@fortawesome/free-solid-svg-icons';
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import axios from 'axios';
import React, { Component } from 'react';
@@ -9,9 +9,8 @@ import Button from 'react-bootstrap/Button';
import Col from 'react-bootstrap/Col';
import Form from 'react-bootstrap/Form';
import Spinner from 'react-bootstrap/Spinner';
import { sizeDisplayName } from './uiutil';
import { TaskLogs } from './TaskLogs';
import { cancelTask, formatDuration, GoBackButton, redirectIfNotConnected } from './uiutil';
import { cancelTask, formatDuration, GoBackButton, redirectIfNotConnected, sizeDisplayName } from './uiutil';
export class TaskDetails extends Component {
constructor() {
@@ -23,8 +22,9 @@ export class TaskDetails extends Component {
showLog: false,
};
this.taskID = this.taskID.bind(this);
this.fetchTask = this.fetchTask.bind(this);
// poll frequently, we will stop as soon as the task ends.
this.interval = window.setInterval(() => this.fetchTask(this.props), 500);
}
@@ -43,10 +43,12 @@ export class TaskDetails extends Component {
}
}
fetchTask(props) {
let tid = props.match.params.tid;
taskID(props) {
return props.taskID || props.match.params.tid;
}
axios.get('/api/v1/tasks/' + tid).then(result => {
fetchTask(props) {
axios.get('/api/v1/tasks/' + this.taskID(props)).then(result => {
this.setState({
task: result.data,
isLoading: false,
@@ -74,25 +76,35 @@ export class TaskDetails extends Component {
switch (task.status) {
case "SUCCESS":
return <Alert variant="success">Task succeeded after {dur}.</Alert>;
case "SUCCESS":
return <Alert variant="success">Task succeeded after {dur}.</Alert>;
case "FAILED":
return <Alert variant="danger"><b>Error:</b> {task.errorMessage}.</Alert>;
case "FAILED":
return <Alert variant="danger"><b>Error:</b> {task.errorMessage}.</Alert>;
case "CANCELING":
return <Alert variant="warning">Cancelation requested...</Alert>;
case "CANCELED":
return <Alert variant="warning">Task canceled.</Alert>;
case "CANCELED":
return <Alert variant="warning">Task canceled.</Alert>;
case "CANCELING":
return <Alert variant="primary">
<Spinner animation="border" variant="warning" size="sm" /> Canceling {dur}: {task.progressInfo}.</Alert>;
default:
return <Alert variant="primary"> <Spinner animation="border" variant="primary" size="sm" /> Task in progress ({dur}).</Alert>;
default:
return <Alert variant="primary">
<Spinner animation="border" variant="primary" size="sm" /> Running for {dur}: {task.progressInfo}.</Alert>;
}
}
valueThreshold() {
if (this.props.showZeroCounters) {
return -1;
}
return 0
}
counterBadge(label, c) {
if (!c.value) {
if (c.value < this.valueThreshold()) {
return "";
}
@@ -153,7 +165,7 @@ export class TaskDetails extends Component {
return 0;
});
return keys.map(c => (counters[c].value > 0) && this.counterBadge(c, counters[c]));
return keys.map(c => (counters[c].value > this.valueThreshold()) && this.counterBadge(c, counters[c]));
}
render() {
@@ -167,15 +179,16 @@ export class TaskDetails extends Component {
}
return <Form>
<Form.Row>
<Form.Group>
<GoBackButton onClick={this.props.history.goBack} />
{task.status === "RUNNING" && <>
&nbsp;<Button size="sm" variant="danger" onClick={() => cancelTask(task.id)} ><FontAwesomeIcon icon={faStopCircle} /> Stop </Button>
</>}
</Form.Group>
</Form.Row>
<Form.Row>
{this.props.history &&
<Form.Row>
<Form.Group>
<GoBackButton onClick={this.props.history.goBack} />
{task.status === "RUNNING" && <>
&nbsp;<Button size="sm" variant="danger" onClick={() => cancelTask(task.id)} ><FontAwesomeIcon icon={faStopCircle} /> Stop </Button>
</>}
</Form.Group>
</Form.Row>}
{!this.props.hideDescription && <Form.Row>
<Col xs={3} >
<Form.Group>
<Form.Control type="text" readOnly={true} value={task.kind} />
@@ -186,7 +199,7 @@ export class TaskDetails extends Component {
<Form.Control type="text" readOnly={true} value={task.description} />
</Form.Group>
</Col>
</Form.Row>
</Form.Row>}
<Form.Row>
<Col xs={9}>
{this.summaryControl(task)}
@@ -199,13 +212,16 @@ export class TaskDetails extends Component {
</Form.Row>
{task.counters && <Form.Row>
<Col>
{this.sortedBadges(task.counters)}
{this.sortedBadges(task.counters)}
</Col>
</Form.Row>}
<hr/>
<hr />
<Form.Row>
<Col>
{this.state.showLog ? <TaskLogs taskID={this.props.match.params.tid} /> : <Button onClick={() => this.setState({showLog:true})}>Show Log</Button>}
{this.state.showLog ? <>
<Button size="sm" onClick={() => this.setState({ showLog: false })}><FontAwesomeIcon icon={faChevronCircleUp} /> Hide Log</Button>
<TaskLogs taskID={this.taskID(this.props)} />
</> : <Button size="sm" onClick={() => this.setState({ showLog: true })}><FontAwesomeIcon icon={faChevronCircleDown} /> Show Log</Button>}
</Col>
</Form.Row>
</Form>

54
htmlui/src/TasksTable.js
View File

@@ -4,10 +4,9 @@ import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import axios from 'axios';
import moment from 'moment';
import React, { Component } from 'react';
import ButtonGroup from 'react-bootstrap/ButtonGroup';
import Alert from 'react-bootstrap/Alert';
import Col from 'react-bootstrap/Col';
import Dropdown from 'react-bootstrap/Dropdown';
import Alert from 'react-bootstrap/Alert';
import Form from 'react-bootstrap/Form';
import Row from 'react-bootstrap/Row';
import { Link } from 'react-router-dom';
@@ -128,35 +127,34 @@ export class TasksTable extends Component {
return <div className="padded">
<Form>
<div class="list-actions">
<Row>
<Col>
<ButtonGroup>
<Dropdown>
<Dropdown.Toggle size="sm" variant="primary">Status: {this.state.showStatus}</Dropdown.Toggle>
<Dropdown.Menu>
<Dropdown.Item onClick={() => this.setState({ showStatus: "All" })}>All</Dropdown.Item>
<Dropdown.Divider />
<Dropdown.Item onClick={() => this.setState({ showStatus: "Running" })}>Running</Dropdown.Item>
<Dropdown.Item onClick={() => this.setState({ showStatus: "Failed" })}>Failed</Dropdown.Item>
</Dropdown.Menu>
</Dropdown>
&nbsp;
<Dropdown>
<Dropdown.Toggle size="sm" variant="primary">Kind: {this.state.showKind}</Dropdown.Toggle>
<Dropdown.Menu>
<Dropdown.Item onClick={() => this.setState({ showKind: "All" })}>All</Dropdown.Item>
<Dropdown.Divider />
{this.state.uniqueKinds.map(k => <Dropdown.Item onClick={() => this.setState({ showKind: k })}>{k}</Dropdown.Item>)}
</Dropdown.Menu>
</Dropdown>
&nbsp;
<Form.Control size="sm" type="text" name="searchDescription" placeholder="search description" value={this.state.searchDescription} onChange={this.handleChange} autoFocus={true} />
</ButtonGroup>
<Col xs="auto">
<Dropdown>
<Dropdown.Toggle size="sm" variant="primary">Status: {this.state.showStatus}</Dropdown.Toggle>
<Dropdown.Menu>
<Dropdown.Item onClick={() => this.setState({ showStatus: "All" })}>All</Dropdown.Item>
<Dropdown.Divider />
<Dropdown.Item onClick={() => this.setState({ showStatus: "Running" })}>Running</Dropdown.Item>
<Dropdown.Item onClick={() => this.setState({ showStatus: "Failed" })}>Failed</Dropdown.Item>
</Dropdown.Menu>
</Dropdown>
</Col>
<Col xs="auto">
<Dropdown>
<Dropdown.Toggle size="sm" variant="primary">Kind: {this.state.showKind}</Dropdown.Toggle>
<Dropdown.Menu>
<Dropdown.Item onClick={() => this.setState({ showKind: "All" })}>All</Dropdown.Item>
<Dropdown.Divider />
{this.state.uniqueKinds.map(k => <Dropdown.Item onClick={() => this.setState({ showKind: k })}>{k}</Dropdown.Item>)}
</Dropdown.Menu>
</Dropdown>
</Col>
<Col xs="4">
<Form.Control size="sm" type="text" name="searchDescription" placeholder="search description" value={this.state.searchDescription} onChange={this.handleChange} autoFocus={true} />
</Col>
</Row>
<Form.Row>
<hr />
</Form.Row>
</div>
<Form.Row>
<Col>
{!items.length ?

5
htmlui/src/forms.js
View File

@@ -71,6 +71,7 @@ export function RequiredField(component, label, name, props = {}, helpText = nul
return <Form.Group as={Col}>
<Form.Label className="required">{label}</Form.Label>
<Form.Control
size="sm"
isInvalid={stateProperty(component, name, null) === ''}
name={name}
value={stateProperty(component, name)}
@@ -86,6 +87,7 @@ export function OptionalField(component, label, name, props = {}, helpText = nul
return <Form.Group as={Col}>
<Form.Label>{label}</Form.Label>
<Form.Control
size="sm"
name={name}
value={stateProperty(component, name)}
data-testid={'control-' + name}
@@ -125,6 +127,7 @@ export function OptionalNumberField(component, label, name, props = {}) {
return <Form.Group as={Col}>
<Form.Label>{label}</Form.Label>
<Form.Control
size="sm"
name={name}
isInvalid={isInvalidNumber(stateProperty(component, name))}
value={stateProperty(component, name)}
@@ -184,6 +187,7 @@ export function OptionalBoolean(component, label, name, defaultLabel) {
return <Form.Group as={Col}>
<Form.Label>{label}</Form.Label>
<Form.Control
size="sm"
name={name}
value={stateProperty(component, name)}
onChange={e => component.handleChange(e, optionalBooleanValue)}
@@ -216,6 +220,7 @@ export function StringList(component, label, name, helpText) {
return <Form.Group as={Col}>
<Form.Label>{label}</Form.Label>
<Form.Control
size="sm"
name={name}
value={listToMultilineString(stateProperty(component, name))}
onChange={e => component.handleChange(e, multilineStringToList)}

65
htmlui/src/uiutil.js
View File

@@ -1,4 +1,4 @@
import { faBan, faCheck, faChevronLeft, faExclamationCircle, faExclamationTriangle, faWindowClose } from '@fortawesome/free-solid-svg-icons';
import { faBan, faCheck, faChevronLeft, faExclamationCircle, faExclamationTriangle, faFolderOpen, faWindowClose } from '@fortawesome/free-solid-svg-icons';
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import axios from 'axios';
import React from 'react';
@@ -7,6 +7,9 @@ import ListGroup from 'react-bootstrap/ListGroup';
import OverlayTrigger from 'react-bootstrap/OverlayTrigger';
import Popover from 'react-bootstrap/Popover';
import Spinner from 'react-bootstrap/Spinner';
import InputGroup from 'react-bootstrap/InputGroup';
import Form from 'react-bootstrap/Form';
import FormControl from 'react-bootstrap/FormControl';
const base10UnitPrefixes = ["", "K", "M", "G", "T"];
@@ -172,4 +175,64 @@ export function cancelTask(tid) {
export function GoBackButton(props) {
return <Button size="sm" variant="outline-secondary" {...props}><FontAwesomeIcon icon={faChevronLeft} /> Return </Button>;
}
function selectDirectory(onSelected) {
// populated in 'preload.js' in Electron
if (!window.require) {
alert('Directory selection is not supported in a web browser.\n\nPlease enter path manually.');
return;
}
const { dialog } = window.require('electron').remote;
try {
let dir = dialog.showOpenDialogSync({
properties: ['openDirectory']
});
if (dir) {
onSelected(dir[0]);
}
} catch (e) {
window.alert('Error: ' + e);
}
}
export function isAbsolutePath(p) {
// Unix-style path.
if (p.startsWith("/")) {
return true;
}
// Windows-style X:\... path.
if (p.length >= 3 && p.substring(1,3) === ":\\") {
const letter = p.substring(0, 1).toUpperCase();
return letter >= "A" && letter <= "Z";
}
// Windows UNC path.
if (p.startsWith("\\\\")) {
return true;
}
return false;
}
export function DirectorySelector(props) {
const selectSupported = !!window.require;
let { onDirectorySelected, ...inputProps } = props;
if (!selectSupported) {
return <Form.Control size="sm" {...inputProps} />
}
return <InputGroup>
<FormControl size="sm" {...inputProps} />
<InputGroup.Append>
<Button size="sm" onClick={() => selectDirectory(onDirectorySelected)}>
<FontAwesomeIcon icon={faFolderOpen} />
</Button>
</InputGroup.Append>
</InputGroup>;
}

121
internal/server/api_estimate.go Normal file
View File

@@ -0,0 +1,121 @@
package server
import (
"context"
"encoding/json"
"net/http"
"os"
"path/filepath"
"strings"
"github.com/pkg/errors"
"github.com/kopia/kopia/fs"
"github.com/kopia/kopia/fs/localfs"
"github.com/kopia/kopia/internal/ctxutil"
"github.com/kopia/kopia/internal/serverapi"
"github.com/kopia/kopia/internal/uitask"
"github.com/kopia/kopia/snapshot"
"github.com/kopia/kopia/snapshot/policy"
"github.com/kopia/kopia/snapshot/snapshotfs"
)
type estimateTaskProgress struct {
ctrl uitask.Controller
}
func (p estimateTaskProgress) Processing(ctx context.Context, dirname string) {
p.ctrl.ReportProgressInfo(dirname)
}
func (p estimateTaskProgress) Error(ctx context.Context, dirname string, err error, isIgnored bool) {
if isIgnored {
log(ctx).Warningf("ignored error in %v: %v", dirname, err)
} else {
log(ctx).Errorf("error in %v: %v", dirname, err)
}
}
func (p estimateTaskProgress) Stats(ctx context.Context, st *snapshot.Stats, included, excluded snapshotfs.SampleBuckets, excludedDirs []string, final bool) {
p.ctrl.ReportCounters(map[string]uitask.CounterValue{
"Bytes": uitask.NoticeBytesCounter(st.TotalFileSize),
"Files": uitask.NoticeCounter(int64(st.TotalFileCount)),
"Directories": uitask.NoticeCounter(int64(st.TotalDirectoryCount)),
"Excluded Files": uitask.SimpleCounter(int64(st.ExcludedFileCount)),
"Excluded Directories": uitask.SimpleCounter(int64(st.ExcludedDirCount)),
"Errors": uitask.ErrorCounter(int64(st.ErrorCount)),
"Ignored Errors": uitask.ErrorCounter(int64(st.IgnoredErrorCount)),
})
}
func resolveUserFriendlyPath(path string) string {
home := os.Getenv("HOME")
if home != "" && strings.HasPrefix(path, "~") {
return home + path[1:]
}
if filepath.IsAbs(path) {
return path
}
return filepath.Join(home, path)
}
var _ snapshotfs.EstimateProgress = estimateTaskProgress{}
func (s *Server) handleEstimate(ctx context.Context, r *http.Request, body []byte) (interface{}, *apiError) {
var req serverapi.EstimateRequest
if err := json.Unmarshal(body, &req); err != nil {
return nil, requestError(serverapi.ErrorMalformedRequest, "malformed request body")
}
ctx = ctxutil.Detach(ctx)
rep := s.rep
resolvedRoot := filepath.Clean(resolveUserFriendlyPath(req.Root))
e, err := localfs.NewEntry(resolvedRoot)
if err != nil {
return nil, internalServerError(errors.Wrap(err, "can't get local fs entry"))
}
dir, ok := e.(fs.Directory)
if !ok {
return nil, internalServerError(errors.Wrap(err, "estimation is only supported on directories"))
}
taskIDChan := make(chan string)
// launch a goroutine that will continue the estimate and can be observed in the Tasks UI.
// nolint:errcheck
go s.taskmgr.Run(ctx, "Estimate", resolvedRoot, func(ctx context.Context, ctrl uitask.Controller) error {
taskIDChan <- ctrl.CurrentTaskID()
estimatectx, cancel := context.WithCancel(ctx)
defer cancel()
ctrl.OnCancel(cancel)
policyTree, err := policy.TreeForSource(ctx, s.rep, snapshot.SourceInfo{
Host: s.options.ConnectOptions.Hostname,
UserName: s.options.ConnectOptions.Username,
Path: resolvedRoot,
})
if err != nil {
return errors.Wrap(err, "unable to get policy tree")
}
return snapshotfs.Estimate(estimatectx, rep, dir, policyTree, estimateTaskProgress{ctrl})
})
taskID := <-taskIDChan
task, ok := s.taskmgr.GetTask(taskID)
if !ok {
return nil, internalServerError(errors.Errorf("task not found"))
}
return task, nil
}

2
internal/server/api_sources.go
View File

@@ -51,6 +51,8 @@ func (s *Server) handleSourcesCreate(ctx context.Context, r *http.Request, body
return nil, requestError(serverapi.ErrorMalformedRequest, "missing path")
}
req.Path = resolveUserFriendlyPath(req.Path)
_, err := os.Stat(req.Path)
if os.IsNotExist(err) {
return nil, requestError(serverapi.ErrorPathNotFound, "path does not exist")

1
internal/server/server.go
View File

@@ -88,6 +88,7 @@ func (s *Server) APIHandlers(legacyAPI bool) http.Handler {
m.HandleFunc("/api/v1/objects/{objectID}", s.requireAuth(s.handleObjectGet)).Methods(http.MethodGet)
m.HandleFunc("/api/v1/restore", s.handleAPI(requireUIUser, s.handleRestore)).Methods(http.MethodPost)
m.HandleFunc("/api/v1/estimate", s.handleAPI(requireUIUser, s.handleEstimate)).Methods(http.MethodPost)
// methods that can be called by any authenticated user (UI or remote user).
m.HandleFunc("/api/v1/flush", s.handleAPI(anyAuthenticatedUser, s.handleFlush)).Methods(http.MethodPost)

2
internal/server/source_manager.go
View File

@@ -449,6 +449,8 @@ func (t *uitaskProgress) UploadedBytes(numBytes int64) {
func (t *uitaskProgress) StartedDirectory(dirname string) {
t.p.StartedDirectory(dirname)
t.maybeReport()
t.ctrl.ReportProgressInfo(dirname)
}
// FinishedDirectory is emitted whenever a directory is finished uploading.

5
internal/serverapi/serverapi.go
View File

@@ -213,3 +213,8 @@ type RestoreRequest struct {
TarFile string `json:"tarFile"`
Options restore.Options `json:"options"`
}
// EstimateRequest contains request to estimate the size of the snapshot in a given root.
type EstimateRequest struct {
Root string `json:"root"`
}

9
internal/uitask/uitask.go
View File

@@ -52,6 +52,7 @@ type Info struct {
Kind string `json:"kind"` // Maintenance, Snapshot, Restore, etc.
Description string `json:"description"`
Status Status `json:"status"`
ProgressInfo string `json:"progressInfo"`
ErrorMessage string `json:"errorMessage,omitempty"`
Counters map[string]CounterValue `json:"counters"`
LogLines []LogEntry `json:"-"`
@@ -101,6 +102,14 @@ func (t *runningTaskInfo) cancel() {
}
}
// ReportProgressInfo implements the Controller interface.
func (t *runningTaskInfo) ReportProgressInfo(pi string) {
t.mu.Lock()
defer t.mu.Unlock()
t.ProgressInfo = pi
}
// ReportCounters implements the Controller interface.
func (t *runningTaskInfo) ReportCounters(c map[string]CounterValue) {
t.mu.Lock()

3
internal/uitask/uitask_manager.go
View File

@@ -32,6 +32,7 @@ type Controller interface {
CurrentTaskID() string
OnCancel(cancelFunc context.CancelFunc)
ReportCounters(counters map[string]CounterValue)
ReportProgressInfo(text string)
}
// TaskFunc represents a task function.
@@ -178,6 +179,8 @@ func (m *Manager) completeTask(r *runningTaskInfo, err error) {
r.Status = StatusCanceled
}
r.ProgressInfo = ""
now := clock.Now()
r.EndTime = &now

9
internal/uitask/uitask_test.go
View File

@@ -79,6 +79,7 @@ func TestUITask(t *testing.T) {
"fff",
})
ctrl.ReportProgressInfo("doing something")
ctrl.ReportCounters(map[string]uitask.CounterValue{
"foo": uitask.SimpleCounter(1),
"bar": uitask.BytesCounter(2),
@@ -103,6 +104,10 @@ func TestUITask(t *testing.T) {
t.Fatalf("unexpected counters, diff: %v", diff)
}
if got, want := tsk.ProgressInfo, "doing something"; got != want {
t.Fatalf("invalid progress info: %v, want %v", got, want)
}
return nil
})
@@ -116,6 +121,10 @@ func TestUITask(t *testing.T) {
t.Fatalf("task not found")
}
if got, want := tsk.ProgressInfo, ""; got != want {
t.Fatalf("invalid progress info: %v, want %v", got, want)
}
if got, want := tsk.Description, "test-1"; got != want {
t.Fatalf("invalid task description %v, want %v", got, want)
}

152
snapshot/snapshotfs/estimate.go Normal file
View File

@@ -0,0 +1,152 @@
package snapshotfs
import (
"context"
"fmt"
"path/filepath"
"github.com/kopia/kopia/fs"
"github.com/kopia/kopia/fs/ignorefs"
"github.com/kopia/kopia/internal/units"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/snapshot"
"github.com/kopia/kopia/snapshot/policy"
)
const maxExamplesPerBucket = 10
// SampleBucket keeps track of count and total size of files above in certain size range and
// includes small number of examples of such files.
type SampleBucket struct {
MinSize int64 `json:"minSize"`
Count int `json:"count"`
TotalSize int64 `json:"totalSize"`
Examples []string `json:"examples,omitempty"`
}
func (b *SampleBucket) add(fname string, size int64) {
b.Count++
b.TotalSize += size
if len(b.Examples) < maxExamplesPerBucket {
b.Examples = append(b.Examples, fmt.Sprintf("%v - %v", fname, units.BytesStringBase10(size)))
}
}
// SampleBuckets is a collection of buckets for interesting file sizes sorted in descending order.
type SampleBuckets []*SampleBucket
func (b SampleBuckets) add(fname string, size int64) {
for _, bucket := range b {
if size >= bucket.MinSize {
bucket.add(fname, size)
break
}
}
}
func makeBuckets() SampleBuckets {
return SampleBuckets{
&SampleBucket{MinSize: 1e15},
&SampleBucket{MinSize: 1e14},
&SampleBucket{MinSize: 1e13},
&SampleBucket{MinSize: 1e12},
&SampleBucket{MinSize: 1e11},
&SampleBucket{MinSize: 1e10},
&SampleBucket{MinSize: 1e9},
&SampleBucket{MinSize: 1e8},
&SampleBucket{MinSize: 1e7},
&SampleBucket{MinSize: 1e6},
&SampleBucket{MinSize: 1e5},
&SampleBucket{MinSize: 1e4},
&SampleBucket{MinSize: 1e3},
&SampleBucket{MinSize: 0},
}
}
// EstimateProgress must be provided by the caller of Estimate to report results.
type EstimateProgress interface {
Processing(ctx context.Context, dirname string)
Error(ctx context.Context, filename string, err error, isIgnored bool)
Stats(ctx context.Context, s *snapshot.Stats, includedFiles, excludedFiles SampleBuckets, excludedDirs []string, final bool)
}
// Estimate walks the provided directory tree and invokes provided progress callback as it discovers
// items to be snapshotted.
func Estimate(ctx context.Context, rep repo.Repository, entry fs.Directory, policyTree *policy.Tree, progress EstimateProgress) error {
stats := &snapshot.Stats{}
ed := []string{}
ib := makeBuckets()
eb := makeBuckets()
// report final stats just before returning
defer func() {
progress.Stats(ctx, stats, ib, eb, ed, true)
}()
onIgnoredFile := func(relativePath string, e fs.Entry) {
if e.IsDir() {
if len(ed) < maxExamplesPerBucket {
ed = append(ed, relativePath)
}
stats.ExcludedDirCount++
log(ctx).Debugf("excluded dir %v", relativePath)
} else {
log(ctx).Debugf("excluded file %v (%v)", relativePath, units.BytesStringBase10(e.Size()))
stats.ExcludedFileCount++
stats.ExcludedTotalFileSize += e.Size()
eb.add(relativePath, e.Size())
}
}
entry = ignorefs.New(entry, policyTree, ignorefs.ReportIgnoredFiles(onIgnoredFile))
return estimate(ctx, ".", entry, policyTree, stats, ib, eb, &ed, progress)
}
func estimate(ctx context.Context, relativePath string, entry fs.Entry, policyTree *policy.Tree, stats *snapshot.Stats, ib, eb SampleBuckets, ed *[]string, progress EstimateProgress) error {
// see if the context got canceled
select {
case <-ctx.Done():
return ctx.Err()
default:
}
switch entry := entry.(type) {
case fs.Directory:
stats.TotalDirectoryCount++
progress.Processing(ctx, relativePath)
children, err := entry.Readdir(ctx)
if err != nil {
isIgnored := policyTree.EffectivePolicy().ErrorHandlingPolicy.IgnoreDirectoryErrorsOrDefault(false)
if isIgnored {
stats.IgnoredErrorCount++
} else {
stats.ErrorCount++
}
progress.Error(ctx, relativePath, err, isIgnored)
} else {
for _, child := range children {
if err := estimate(ctx, filepath.Join(relativePath, child.Name()), child, policyTree.Child(child.Name()), stats, ib, eb, ed, progress); err != nil {
return err
}
}
}
progress.Stats(ctx, stats, ib, eb, *ed, false)
case fs.File:
ib.add(relativePath, entry.Size())
stats.TotalFileCount++
stats.TotalFileSize += entry.Size()
}
return nil
}

21
snapshot/snapshotfs/upload.go
View File

@@ -1127,6 +1127,16 @@ func (u *Uploader) Upload(
}
}
scanWG.Add(1)
go func() {
defer scanWG.Done()
ds, _ := u.scanDirectory(scanctx, entry, policyTree)
u.Progress.EstimatedDataSize(ds.numFiles, ds.totalFileSize)
}()
entry = ignorefs.New(entry, policyTree, ignorefs.ReportIgnoredFiles(func(fname string, md fs.Entry) {
if md.IsDir() {
u.Progress.ExcludedDir(fname)
@@ -1136,17 +1146,6 @@ func (u *Uploader) Upload(
u.stats.AddExcluded(md)
}))
scanWG.Add(1)
go func() {
defer scanWG.Done()
ds, _ := u.scanDirectory(scanctx, entry)
u.Progress.EstimatedDataSize(ds.numFiles, ds.totalFileSize)
}()
s.RootEntry, err = u.uploadDirWithCheckpointing(ctx, entry, policyTree, previousDirs, sourceInfo)
case fs.File:

52
snapshot/snapshotfs/upload_scan.go
View File

@@ -3,9 +3,9 @@
import (
"context"
"github.com/pkg/errors"
"github.com/kopia/kopia/fs"
"github.com/kopia/kopia/snapshot"
"github.com/kopia/kopia/snapshot/policy"
)
type scanResults struct {
@@ -13,45 +13,29 @@ type scanResults struct {
totalFileSize int64
}
func (e *scanResults) Error(ctx context.Context, filename string, err error, isIgnored bool) {}
func (e *scanResults) Processing(ctx context.Context, pathname string) {}
func (e *scanResults) Stats(ctx context.Context, s *snapshot.Stats, includedFiles, excludedFiles SampleBuckets, excludedDirs []string, final bool) {
if final {
e.numFiles = int(s.TotalFileCount)
e.totalFileSize = s.TotalFileSize
}
}
var _ EstimateProgress = (*scanResults)(nil)
// scanDirectory computes the number of files and their total size in a given directory recursively descending
// into subdirectories. The scan teminates early as soon as the provided context is canceled.
func (u *Uploader) scanDirectory(ctx context.Context, dir fs.Directory) (scanResults, error) {
func (u *Uploader) scanDirectory(ctx context.Context, dir fs.Directory, policyTree *policy.Tree) (scanResults, error) {
var res scanResults
if u.disableEstimation {
return res, nil
}
entries, err := dir.Readdir(ctx)
if err != nil {
return res, errors.Wrap(err, "unable to read directory")
}
err := Estimate(ctx, u.repo, dir, policyTree, &res)
for _, e := range entries {
if err := ctx.Err(); err != nil {
// terminate early if context got canceled
// nolint:wrapcheck
return res, err
}
switch e := e.(type) {
case fs.Directory:
dr, err := u.scanDirectory(ctx, e)
res.numFiles += dr.numFiles
res.totalFileSize += dr.totalFileSize
if err != nil {
return res, err
}
case fs.File:
res.numFiles++
res.totalFileSize += e.Size()
case fs.StreamingFile:
res.numFiles++
}
}
return res, nil
return res, err
}

53
snapshot/snapshotfs/upload_test.go
View File

@@ -547,7 +547,7 @@ func TestUploadWithCheckpointing(t *testing.T) {
}
}
func TestUploadScan(t *testing.T) {
func TestUploadScanStopsOnContextCancel(t *testing.T) {
ctx := testlogging.Context(t)
th := newUploadTestHarness(ctx, t)
@@ -561,7 +561,7 @@ func TestUploadScan(t *testing.T) {
cancel()
})
result, err := u.scanDirectory(scanctx, th.sourceDir)
result, err := u.scanDirectory(scanctx, th.sourceDir, nil)
if !errors.Is(err, scanctx.Err()) {
t.Fatalf("invalid scan error: %v", err)
}
@@ -571,6 +571,47 @@ func TestUploadScan(t *testing.T) {
}
}
func TestUploadScanIgnoresFiles(t *testing.T) {
ctx := testlogging.Context(t)
th := newUploadTestHarness(ctx, t)
defer th.cleanup()
u := NewUploader(th.repo)
// set up a policy tree where that ignores some files.
policyTree := policy.BuildTree(map[string]*policy.Policy{
".": {
FilesPolicy: policy.FilesPolicy{
IgnoreRules: []string{"f1"},
},
},
}, policy.DefaultPolicy)
// no policy
result1, err := u.scanDirectory(ctx, th.sourceDir, nil)
must(t, err)
result2, err := u.scanDirectory(ctx, th.sourceDir, policyTree)
must(t, err)
if result1.numFiles == 0 {
t.Fatalf("no files scanned")
}
if result2.numFiles == 0 {
t.Fatalf("no files scanned")
}
if got, want := result2.numFiles, result1.numFiles; got >= want {
t.Fatalf("expected lower number of files %v, wanted %v", got, want)
}
if got, want := result2.totalFileSize, result1.totalFileSize; got >= want {
t.Fatalf("expected lower file size %v, wanted %v", got, want)
}
}
func TestUpload_VirtualDirectoryWithStreamingFile(t *testing.T) {
ctx := testlogging.Context(t)
th := newUploadTestHarness(ctx, t)
@@ -625,3 +666,11 @@ func TestUpload_VirtualDirectoryWithStreamingFile(t *testing.T) {
t.Fatalf("unexpected manifest file count: %v, want %v", got, want)
}
}
func must(t *testing.T, err error) {
t.Helper()
if err != nil {
t.Fatal(err)
}
}