mirror/kopia
1
0 Fork 0
mirror of https://github.com/kopia/kopia.git synced 2026-10-27 08:38:55 -04:00
Code Issues Packages Projects Releases Wiki Activity

ui: major improvements to new snapshot flow (#1565)

Browse Source 
* ui: changed how PolicyEditor is instantiated via a route

* server: added paths/resolve API

* server: refresh affected source manager after policy change

Also switched 15-second refresh cycle which is way too aggressive
to 30-minute cycle (manual refresh button can be used if needed).

* policy: allow overriding top-level policy for estimation

* server: changed source create API to always require policy

* ui: streamlined new snapshot and estimate flow

* linter fix
This commit is contained in:
Jarek Kowalski authored and GitHub committed 2021-12-04 22:13:10 -08:00
1 parent 2cb05f7501
commit 5f04fad003
23 files changed
+487 -202
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

htmlui/src/App.css
+4 -8
View File
@@ -143,8 +143,7 @@ div.tab-body {
}
.logs-table {
border: 1px solid #ccd;
background-color: #f0f0ff;
background-color: #f8f8ff;
font-family: monospace;
font-size: 12px;
font-weight: bold;
@@ -175,12 +174,9 @@ div.tab-body {
padding-left: 40px;
}
.estimate-results {
margin: 10px;
margin-left: 15px;
padding: 15px;
border: 1px solid #ccc;
background-color: #f8f8ff;
.estimateResults {
color: #008;
font-size: 95%;
}
.list-actions {
htmlui/src/App.js
+4 -4
View File
@@ -14,9 +14,9 @@ import { RepoStatus } from "./RepoStatus";
import { SnapshotsTable } from "./SnapshotsTable";
import { SourcesTable } from "./SourcesTable";
import { TaskDetails } from './TaskDetails';
import { TasksTable } from './TasksTable';
import { TasksTable } from './TasksTable';
import { NewSnapshot } from './NewSnapshot';
import { PolicyEditor } from './PolicyEditor';
import { PolicyEditorPage } from './PolicyEditorPage';
function useInterval(callback, delay) {
const savedCallback = useRef();
@@ -72,9 +72,9 @@ function App() {
<Route path="/snapshots/new" component={NewSnapshot} />
<Route path="/snapshots/single-source/" component={SnapshotsTable} />
<Route path="/snapshots/dir/:oid/restore" component={BeginRestore} />
<Route path="/snapshots/dir/:oid" component={DirectoryObject} />
<Route path="/snapshots/dir/:oid" component={DirectoryObject} />
<Route path="/snapshots" component={SourcesTable} />
<Route path="/policies/edit/" component={PolicyEditor} />
<Route path="/policies/edit/" component={PolicyEditorPage} />
<Route path="/policies" component={PoliciesTable} />
<Route path="/tasks/:tid" component={TaskDetails} />
<Route path="/tasks" component={TasksTable} />
htmlui/src/EstimateResults.js
+116
View File
@@ -0,0 +1,116 @@
import { faChevronCircleDown, faChevronCircleUp, faStopCircle } from '@fortawesome/free-solid-svg-icons';
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import axios from 'axios';
import React, { Component } from 'react';
import Button from 'react-bootstrap/Button';
import Spinner from 'react-bootstrap/esm/Spinner';
import Form from 'react-bootstrap/Form';
import { TaskLogs } from './TaskLogs';
import { cancelTask, redirectIfNotConnected, sizeDisplayName } from './uiutil';
export class EstimateResults extends Component {
constructor() {
super();
this.state = {
isLoading: true,
error: null,
showLog: false,
};
this.taskID = this.taskID.bind(this);
this.fetchTask = this.fetchTask.bind(this);
// poll frequently, we will stop as soon as the task ends.
this.interval = window.setInterval(() => this.fetchTask(this.props), 500);
}
componentDidMount() {
this.setState({
isLoading: true,
});
this.fetchTask(this.props);
}
componentWillUnmount() {
if (this.interval) {
window.clearInterval(this.interval);
}
}
taskID(props) {
return props.taskID || props.match.params.tid;
}
fetchTask(props) {
axios.get('/api/v1/tasks/' + this.taskID(props)).then(result => {
this.setState({
task: result.data,
isLoading: false,
});
if (result.data.endTime) {
window.clearInterval(this.interval);
this.interval = null;
}
}).catch(error => {
redirectIfNotConnected(error);
this.setState({
error,
isLoading: false
});
});
}
componentDidUpdate(prevProps) {
if (prevProps !== this.props) {
this.fetchTask(this.props);
}
}
taskStatusDescription(task) {
if (task.status === "RUNNING") {
return <><Spinner animation="border" variant="primary" size="sm" /></>
}
if (task.status === "SUCCESS") {
return "Total"
}
if (task.status === "CANCELED") {
return "(Canceled)"
}
return task.status;
}
render() {
const { task, isLoading, error } = this.state;
if (error) {
return <p>{error.message}</p>;
}
if (isLoading) {
return <p>Loading ...</p>;
}
return <>
{task.counters && <Form.Text className="estimateResults">
{this.taskStatusDescription(task)} Bytes: <b>{sizeDisplayName(task.counters["Bytes"]?.value)}</b> (<b>{sizeDisplayName(task.counters["Excluded Bytes"]?.value)}</b> excluded)
Files: <b>{task.counters["Files"]?.value}</b> (<b>{task.counters["Excluded Files"]?.value}</b> excluded)
Directories: <b>{task.counters["Directories"]?.value}</b> (<b>{task.counters["Excluded Directories"]?.value}</b> excluded)
Errors: <b>{task.counters["Errors"]?.value}</b> (<b>{task.counters["Ignored Errors"]?.value}</b> ignored)
</Form.Text>
}
{task.status === "RUNNING" && <>
&nbsp;<Button size="sm" variant="light" onClick={() => cancelTask(task.id)} ><FontAwesomeIcon icon={faStopCircle} color="red" /> Cancel </Button>
</>}
{this.state.showLog ? <>
<Button size="sm" variant="light" onClick={() => this.setState({ showLog: false })}><FontAwesomeIcon icon={faChevronCircleUp} /> Hide Log</Button>
<TaskLogs taskID={this.taskID(this.props)} />
</> : <Button size="sm" variant="light" onClick={() => this.setState({ showLog: true })}><FontAwesomeIcon icon={faChevronCircleDown} /> Show Log</Button>}
</>
;
}
}
htmlui/src/NewSnapshot.js
+54 -68
View File
@@ -1,15 +1,13 @@
import { faWindowClose } from '@fortawesome/free-solid-svg-icons';
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import axios from 'axios';
import React, { Component } from 'react';
import Button from 'react-bootstrap/Button';
import Col from 'react-bootstrap/Col';
import Form from 'react-bootstrap/Form';
import Row from 'react-bootstrap/Row';
import { handleChange, validateRequiredFields } from './forms';
import { handleChange } from './forms';
import { PolicyEditor } from './PolicyEditor';
import { TaskDetails } from './TaskDetails';
import { cancelTask, DirectorySelector, errorAlert, GoBackButton, redirectIfNotConnected } from './uiutil';
import { EstimateResults } from './EstimateResults';
import { DirectorySelector, errorAlert, GoBackButton, redirectIfNotConnected } from './uiutil';
export class NewSnapshot extends Component {
constructor() {
@@ -18,6 +16,7 @@ export class NewSnapshot extends Component {
path: "",
estimateTaskID: null,
estimateTaskVisible: false,
lastEstimatedPath: "",
policyEditorVisibleFor: "n/a",
localUsername: null,
};
@@ -26,8 +25,6 @@ export class NewSnapshot extends Component {
this.handleChange = handleChange.bind(this);
this.estimate = this.estimate.bind(this);
this.snapshotNow = this.snapshotNow.bind(this);
this.cancelEstimate = this.cancelEstimate.bind(this);
this.togglePolicyEditor = this.togglePolicyEditor.bind(this);
}
componentDidMount() {
@@ -41,36 +38,46 @@ export class NewSnapshot extends Component {
});
}
togglePolicyEditor() {
if (!this.state.path) {
return;
componentDidUpdate() {
if (this.state.lastResolvedPath !== this.state.path) {
axios.post('/api/v1/paths/resolve', { path: this.state.path }).then(result => {
this.setState({
lastResolvedPath: this.state.path,
resolvedSource: result.data.source,
});
}).catch(error => {
redirectIfNotConnected(error);
});
}
if (this.state.policyEditorVisibleFor === this.state.path) {
if (this.state.estimateTaskVisible && this.state.lastEstimatedPath !== this.state.resolvedSource.path) {
this.setState({
policyEditorVisibleFor: "n/a",
});
} else {
this.setState({
policyEditorVisibleFor: this.state.path,
});
estimateTaskVisible: false,
})
}
}
estimate(e) {
e.preventDefault();
if (!validateRequiredFields(this, ["path"])) {
if (!this.state.resolvedSource.path) {
return;
}
const pe = this.policyEditorRef.current;
if (!pe) {
return;
}
let req = {
root: this.state.path,
root: this.state.resolvedSource.path,
maxExamplesPerBucket: 10,
policyOverride: pe.state.policy,
}
axios.post('/api/v1/estimate', req).then(result => {
this.setState({
lastEstimatedPath: this.state.resolvedSource.path,
estimateTaskID: result.data.id,
estimatingPath: result.data.description,
estimateTaskVisible: true,
@@ -81,22 +88,23 @@ export class NewSnapshot extends Component {
});
}
cancelEstimate() {
this.setState({ estimateTaskVisible: false });
cancelTask(this.state.estimateTaskID);
}
snapshotNow(e) {
e.preventDefault();
if (!this.state.path) {
if (!this.state.resolvedSource.path) {
alert('Must specify directory to snapshot.');
return
}
const pe = this.policyEditorRef.current;
if (!pe) {
return;
}
axios.post('/api/v1/sources', {
path: this.state.path,
path: this.state.resolvedSource.path,
createSnapshot: true,
policy: pe.state.policy,
}).then(result => {
this.props.history.goBack();
}).catch(error => {
@@ -121,62 +129,40 @@ export class NewSnapshot extends Component {
<Row>
<Col>
<Form.Group>
<DirectorySelector onDirectorySelected={p => this.setState({ path: p })} autoFocus placeholder="enter path to snapshot" name="path" value={this.state.path} onChange={this.handleChange}
readOnly={this.state.policyEditorVisible || this.state.estimateTaskVisible} />
<Form.Text>
Click the <code>Estimate</code> button to estimate the size of the snapshot.
To specify frequency of snapshots or exclude some files, click <code>Policy</code>.
</Form.Text>
<DirectorySelector onDirectorySelected={p => this.setState({ path: p })} autoFocus placeholder="enter path to snapshot" name="path" value={this.state.path} onChange={this.handleChange} />
</Form.Group>
</Col>
<Col xs="auto">
<Button
size="sm"
disabled={!this.state.path}
title="Edit Policy"
variant="primary"
onClick={this.estimate}>Estimate</Button>
</Col>
<Col xs="auto">
<Button
size="sm"
disabled={!this.state.path}
disabled={!this.state.resolvedSource?.path}
title="Estimate"
variant="primary"
onClick={this.togglePolicyEditor}>Policy</Button>
onClick={this.estimate}>Estimate</Button>
&nbsp;
<Button size="sm"
disabled={!this.state.resolvedSource?.path}
title="Snapshot Now"
variant="success"
onClick={this.snapshotNow}>Snapshot Now</Button>
</Col>
</Row>
{this.state.estimateTaskID && this.state.estimateTaskVisible &&
<div className="estimate-results">
<h5>
<Button onClick={this.cancelEstimate} title="stop estimation" variant="light" size="sm"><FontAwesomeIcon color="#888" icon={faWindowClose} /></Button>&nbsp;
Estimate Snapshot Size for {this.state.estimatingPath}...
</h5>
<TaskDetails taskID={this.state.estimateTaskID} hideDescription={true} showZeroCounters={true} />
</div>
<EstimateResults taskID={this.state.estimateTaskID} hideDescription={true} showZeroCounters={true} />
}
<br />
{this.state.path && this.state.policyEditorVisibleFor === this.state.path && <Row>
<Col xs={12}>
<PolicyEditor ref={this.policyEditorRef}
embedded
host={this.state.localHost}
userName={this.state.localUsername}
path={this.state.path}
close={this.togglePolicyEditor} />
</Col>
</Row>}
{this.state.resolvedSource && <Col xs={12}>
<Form.Text>
{this.state.resolvedSource ? this.state.resolvedSource.path : this.state.path}
</Form.Text>
<Row>
<Col>
<Button size="sm"
disabled={!this.state.path}
title="Snapshot Now"
variant="primary"
onClick={this.snapshotNow}>Snapshot Now</Button>
</Col>
</Row>
<PolicyEditor ref={this.policyEditorRef}
embedded
host={this.state.resolvedSource.host}
userName={this.state.resolvedSource.userName}
path={this.state.resolvedSource.path} />
</Col>}
</>;
}
}
htmlui/src/PoliciesTable.js
+9 -26
View File
@@ -8,10 +8,10 @@ import Col from 'react-bootstrap/Col';
import Dropdown from 'react-bootstrap/Dropdown';
import Form from 'react-bootstrap/Form';
import Row from 'react-bootstrap/Row';
import { Link } from 'react-router-dom';
import { handleChange } from './forms';
import { PolicyEditor } from './PolicyEditor';
import MyTable from './Table';
import { compare, DirectorySelector, isAbsolutePath, ownerName, redirectIfNotConnected } from './uiutil';
import { compare, DirectorySelector, isAbsolutePath, ownerName, policyEditorURL, redirectIfNotConnected } from './uiutil';
const localPolicies = "Local Policies"
const allPolicies = "All Policies"
@@ -31,7 +31,6 @@ export class PoliciesTable extends Component {
sources: [],
};
this.editorClosed = this.editorClosed.bind(this);
this.editPolicyForPath = this.editPolicyForPath.bind(this);
this.handleChange = handleChange.bind(this);
this.fetchPolicies = this.fetchPolicies.bind(this);
@@ -94,13 +93,6 @@ export class PoliciesTable extends Component {
});
}
editorClosed() {
this.setState({
editorTarget: null,
});
this.fetchPolicies();
}
editPolicyForPath(e) {
e.preventDefault();
@@ -113,13 +105,11 @@ export class PoliciesTable extends Component {
return;
}
this.setState({
editorTarget: {
userName: this.state.localUsername,
host: this.state.localHost,
path: this.state.policyPath,
},
})
this.props.history.push(policyEditorURL({
userName: this.state.localUsername,
host: this.state.localHost,
path: this.state.policyPath,
}));
}
selectOwner(h) {
@@ -233,12 +223,7 @@ export class PoliciesTable extends Component {
id: 'edit',
Header: '',
width: 50,
Cell: x => <Button size="sm" onClick={() => {
this.setState({
editorTarget: x.row.original.target,
selectedRowIsNew: false,
})
}}>Edit</Button>,
Cell: x => <Link to={policyEditorURL(x.row.original.target)}><Button size="sm">Edit</Button></Link>,
}]
return <div className="padded">
@@ -277,14 +262,12 @@ export class PoliciesTable extends Component {
</Form>
</div>}
{items.length > 0 ? <div className={this.state.editorTarget ? "hidden" : "normal"}>
{items.length > 0 ? <div>
<p>Found {items.length} policies matching criteria.</p>
<MyTable data={items} columns={columns} />
</div> : ((this.state.selectedOwner === localPolicies && this.state.policyPath) ? <p>
No policy found for directory <code>{this.state.policyPath}</code>. Click <b>Set Policy</b> to define it.
</p> : <p>No policies found.</p>)}
{this.state.editorTarget && <PolicyEditor host={this.state.editorTarget.host} userName={this.state.editorTarget.userName} path={this.state.editorTarget.path} close={this.editorClosed} />}
</div>;
}
}
htmlui/src/PolicyEditor.js
+11 -25
View File
@@ -1,4 +1,4 @@
import { faCalendarTimes, faChevronLeft, faClock, faExclamationTriangle, faFileAlt, faFileArchive, faFolderOpen, faMagic } from '@fortawesome/free-solid-svg-icons';
import { faCalendarTimes, faClock, faExclamationTriangle, faFileAlt, faFileArchive, faFolderOpen, faMagic } from '@fortawesome/free-solid-svg-icons';
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import axios from 'axios';
import moment from 'moment';
@@ -10,24 +10,9 @@ import Row from 'react-bootstrap/Row';
import Spinner from 'react-bootstrap/Spinner';
import Accordion from 'react-bootstrap/Accordion';
import { handleChange, LogDetailSelector, OptionalBoolean, OptionalNumberField, RequiredBoolean, stateProperty, StringList, valueToNumber } from './forms';
import { errorAlert, sourceQueryStringParams } from './uiutil';
import { errorAlert, PolicyEditorLink, sourceQueryStringParams } from './uiutil';
import { getDeepStateProperty } from './deepstate';
function PolicyTypeName(s) {
if (!s.host && !s.userName) {
return "Global Policy"
}
if (!s.userName) {
return "Host: " + s.host;
}
if (!s.path) {
return "User: " + s.userName + "@" + s.host;
}
return "Directory: " + s.userName + "@" + s.host + ":" + s.path;
}
function LabelColumn(props) {
return <Col xs={12} sm={4} className="policyFieldColumn">
@@ -118,7 +103,7 @@ function UpcomingSnapshotTimes(times) {
function SectionHeaderRow() {
return <Row>
<LabelColumn />
<ValueColumn><div className="policyEditorHeader">Values</div></ValueColumn>
<ValueColumn><div className="policyEditorHeader"></div></ValueColumn>
<EffectiveValueColumn><div className="policyEditorHeader">Effective</div></EffectiveValueColumn>
</Row>;
}
@@ -152,7 +137,11 @@ export class PolicyEditor extends Component {
});
}
componentDidUpdate() {
componentDidUpdate(prevProps) {
if (sourceQueryStringParams(this.props) !== sourceQueryStringParams(prevProps)) {
this.fetchPolicy(this.props);
}
const pjs = JSON.stringify(this.state.policy);
if (pjs !== this.lastResolvedPolicy) {
this.resolvePolicy(this.props);
@@ -203,7 +192,7 @@ export class PolicyEditor extends Component {
return "(Defined by this policy)";
}
return <>Defined by <b>{PolicyTypeName(p)}</b></>;
return <>Defined by {PolicyEditorLink(p)}</>;
}
saveChanges(e) {
@@ -288,10 +277,7 @@ export class PolicyEditor extends Component {
}
return <div className="padded">
{!this.props.embedded && <h4><Button size="sm" variant="outline-secondary" onClick={this.props.close} ><FontAwesomeIcon icon={faChevronLeft} /> Return </Button>
&nbsp;&nbsp;{PolicyTypeName(this.props)}</h4>}
<Form onSubmit={this.saveChanges}>
<Accordion defaultActiveKey="scheduling">
<Accordion.Item eventKey="retention">
<Accordion.Header><FontAwesomeIcon icon={faCalendarTimes} />&nbsp;Snapshot Retention</Accordion.Header>
@@ -538,8 +524,8 @@ export class PolicyEditor extends Component {
</Accordion.Item>
</Accordion>
<Button size="sm" variant="success" type="submit" onClick={this.saveChanges} data-testid="button-save" disabled={this.state.saving}>Save Policy</Button>
{!this.state.isNew && <>&nbsp;
{!this.props.embedded && <Button size="sm" variant="success" type="submit" onClick={this.saveChanges} data-testid="button-save" disabled={this.state.saving}>Save Policy</Button>}
{!this.state.isNew && !this.props.embedded && <>&nbsp;
<Button size="sm" variant="danger" disabled={this.isGlobal() || this.state.saving} onClick={this.deletePolicy}>Delete Policy</Button>
</>}
{this.state.saving && <>
htmlui/src/PolicyEditor.test.js
+3 -2
View File
@@ -121,8 +121,9 @@ it('e2e', async () => {
if (false) {
logDOM(getByTestId("definition-retention.keepHourly"));
}
await waitFor(() => expect(getByTestId("definition-retention.keepLatest").innerHTML).toEqual("Defined by <b>Directory: some-user@h1:some-path</b>"));
await waitFor(() => expect(getByTestId("definition-retention.keepMonthly").innerHTML).toEqual("Defined by <b>Directory: u1@h1:p1</b>"));
await waitFor(() => expect(getByTestId("definition-retention.keepLatest").innerHTML).toContain(`Directory: some-user@h1:some-path`));
await waitFor(() => expect(getByTestId("definition-retention.keepMonthly").innerHTML).toContain("Directory: u1@h1:p1"));
const expectedUpcomingSnapshotTimes = [
`<li>${moment(ust1).format('L LT')} (${moment(ust1).fromNow()})</li>`,
htmlui/src/PolicyEditorPage.js
+23
View File
@@ -0,0 +1,23 @@
import React, { Component, createRef } from 'react';
import { PolicyEditor } from './PolicyEditor';
import { GoBackButton, parseQuery, PolicyTypeName } from './uiutil';
export class PolicyEditorPage extends Component {
constructor() {
super();
this.editorRef = createRef();
}
render() {
const source = parseQuery(this.props.location.search);
const { userName, host, path } = source;
return <>
<h4>
<GoBackButton onClick={this.props.history.goBack} />
&nbsp;&nbsp;{PolicyTypeName(source)}</h4>
<PolicyEditor ref={this.editorRef} userName={userName} host={host} path={path} close={this.props.history.goBack} />
</>;
}
}
htmlui/src/SourcesTable.js
+3 -2
View File
@@ -12,7 +12,7 @@ import Spinner from 'react-bootstrap/Spinner';
import { Link } from 'react-router-dom';
import { handleChange } from './forms';
import MyTable from './Table';
import { compare, errorAlert, ownerName, redirectIfNotConnected, sizeDisplayName, sizeWithFailures, sourceQueryStringParams } from './uiutil';
import { compare, errorAlert, ownerName, policyEditorURL, redirectIfNotConnected, sizeDisplayName, sizeWithFailures, sourceQueryStringParams } from './uiutil';
const localSnapshots = "Local Snapshots"
const allSnapshots = "All Snapshots"
@@ -96,6 +96,7 @@ export class SourcesTable extends Component {
<Button variant="primary" size="sm" onClick={() => {
parent.startSnapshot(x.row.original.source);
}}>Snapshot now</Button>
&nbsp;<Link to={policyEditorURL(x.row.original.source)}><Button size="sm" variant="info">Policy</Button></Link>
</>;
case "PENDING":
@@ -233,7 +234,7 @@ export class SourcesTable extends Component {
</> : '',
}, {
id: 'status',
Header: 'Status',
Header: '',
width: 300,
accessor: x => x.status,
Cell: x => this.statusCell(x, this),
htmlui/src/uiutil.js
+25
View File
@@ -7,6 +7,7 @@ import Form from 'react-bootstrap/Form';
import FormControl from 'react-bootstrap/FormControl';
import InputGroup from 'react-bootstrap/InputGroup';
import Spinner from 'react-bootstrap/Spinner';
import { Link } from 'react-router-dom';
const base10UnitPrefixes = ["", "K", "M", "G", "T"];
@@ -165,6 +166,30 @@ export function GoBackButton(props) {
return <Button size="sm" variant="outline-secondary" {...props}><FontAwesomeIcon icon={faChevronLeft} /> Return </Button>;
}
export function PolicyTypeName(s) {
if (!s.host && !s.userName) {
return "Global Policy"
}
if (!s.userName) {
return "Host: " + s.host;
}
if (!s.path) {
return "User: " + s.userName + "@" + s.host;
}
return "Directory: " + s.userName + "@" + s.host + ":" + s.path;
}
export function policyEditorURL(s) {
return '/policies/edit?' + sourceQueryStringParams(s);
}
export function PolicyEditorLink(s) {
return <Link to={policyEditorURL(s)}>{PolicyTypeName(s)}</Link>;
}
export function sourceQueryStringParams(src) {
return 'userName=' + encodeURIComponent(src.userName) + '&host=' + encodeURIComponent(src.host) + '&path=' + encodeURIComponent(src.path);
}
internal/server/api_estimate.go
+3 -2
View File
@@ -39,6 +39,7 @@ func (p estimateTaskProgress) Error(ctx context.Context, dirname string, err err
func (p estimateTaskProgress) Stats(ctx context.Context, st *snapshot.Stats, included, excluded snapshotfs.SampleBuckets, excludedDirs []string, final bool) {
p.ctrl.ReportCounters(map[string]uitask.CounterValue{
"Bytes": uitask.BytesCounter(st.TotalFileSize),
"Excluded Bytes": uitask.BytesCounter(st.ExcludedTotalFileSize),
"Files": uitask.SimpleCounter(int64(st.TotalFileCount)),
"Directories": uitask.SimpleCounter(int64(st.TotalDirectoryCount)),
"Excluded Files": uitask.SimpleCounter(int64(st.ExcludedFileCount)),
@@ -129,11 +130,11 @@ func (s *Server) handleEstimate(ctx context.Context, r *http.Request, body []byt
ctrl.OnCancel(cancel)
policyTree, err := policy.TreeForSource(ctx, s.rep, snapshot.SourceInfo{
policyTree, err := policy.TreeForSourceWithOverride(ctx, s.rep, snapshot.SourceInfo{
Host: s.rep.ClientOptions().Hostname,
UserName: s.rep.ClientOptions().Username,
Path: resolvedRoot,
})
}, req.PolicyOverride)
if err != nil {
return errors.Wrap(err, "unable to get policy tree")
}
internal/server/api_paths.go
+28
View File
@@ -0,0 +1,28 @@
package server
import (
"context"
"encoding/json"
"net/http"
"path/filepath"
"github.com/kopia/kopia/internal/ospath"
"github.com/kopia/kopia/internal/serverapi"
"github.com/kopia/kopia/snapshot"
)
func (s *Server) handlePathResolve(ctx context.Context, r *http.Request, body []byte) (interface{}, *apiError) {
var req serverapi.ResolvePathRequest
if err := json.Unmarshal(body, &req); err != nil {
return nil, requestError(serverapi.ErrorMalformedRequest, "malformed request body")
}
return &serverapi.ResolvePathResponse{
SourceInfo: snapshot.SourceInfo{
Path: filepath.Clean(ospath.ResolveUserFriendlyPath(req.Path, true)),
Host: s.rep.ClientOptions().Hostname,
UserName: s.rep.ClientOptions().Username,
},
}, nil
}
internal/server/api_paths_test.go
+36
View File
@@ -0,0 +1,36 @@
package server_test
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/kopia/kopia/internal/apiclient"
"github.com/kopia/kopia/internal/repotesting"
"github.com/kopia/kopia/internal/serverapi"
"github.com/kopia/kopia/internal/testutil"
)
func TestPathsAPI(t *testing.T) {
ctx, env := repotesting.NewEnvironment(t, repotesting.FormatNotImportant)
srvInfo := startServer(t, env, false)
cli, err := apiclient.NewKopiaAPIClient(apiclient.Options{
BaseURL: srvInfo.BaseURL,
TrustedServerCertificateFingerprint: srvInfo.TrustedServerCertificateFingerprint,
Username: testUIUsername,
Password: testUIPassword,
})
require.NoError(t, err)
dir0 := testutil.TempDirectory(t)
req := &serverapi.ResolvePathRequest{
Path: dir0,
}
resp := &serverapi.ResolvePathResponse{}
require.NoError(t, cli.Post(ctx, "paths/resolve", req, resp))
require.Equal(t, localSource(env, dir0), resp.SourceInfo)
}
internal/server/api_policies.go
+19 -13
View File
@@ -73,7 +73,7 @@ func (s *Server) handlePolicyResolve(ctx context.Context, r *http.Request, body
target := getPolicyTargetFromURL(r.URL)
// build a list of parents
policies, err := policy.GetPolicyHierarchy(ctx, s.rep, target)
policies, err := policy.GetPolicyHierarchy(ctx, s.rep, target, nil)
if err != nil {
return nil, internalServerError(err)
}
@@ -106,18 +106,21 @@ func (s *Server) handlePolicyResolve(ctx context.Context, r *http.Request, body
}
func (s *Server) handlePolicyDelete(ctx context.Context, r *http.Request, body []byte) (interface{}, *apiError) {
w, ok := s.rep.(repo.RepositoryWriter)
if !ok {
if _, ok := s.rep.(repo.RepositoryWriter); !ok {
return nil, repositoryNotWritableError()
}
if err := policy.RemovePolicy(ctx, w, getPolicyTargetFromURL(r.URL)); err != nil {
sourceInfo := getPolicyTargetFromURL(r.URL)
if err := repo.WriteSession(ctx, s.rep, repo.WriteSessionOptions{
Purpose: "PolicyDelete",
}, func(ctx context.Context, w repo.RepositoryWriter) error {
return errors.Wrap(policy.RemovePolicy(ctx, w, sourceInfo), "unable to delete policy")
}); err != nil {
return nil, internalServerError(err)
}
if err := w.Flush(ctx); err != nil {
return nil, internalServerError(err)
}
s.triggerRefreshSource(sourceInfo)
return &serverapi.Empty{}, nil
}
@@ -128,18 +131,21 @@ func (s *Server) handlePolicyPut(ctx context.Context, r *http.Request, body []by
return nil, requestError(serverapi.ErrorMalformedRequest, "malformed request body")
}
w, ok := s.rep.(repo.RepositoryWriter)
if !ok {
if _, ok := s.rep.(repo.RepositoryWriter); !ok {
return nil, repositoryNotWritableError()
}
if err := policy.SetPolicy(ctx, w, getPolicyTargetFromURL(r.URL), newPolicy); err != nil {
sourceInfo := getPolicyTargetFromURL(r.URL)
if err := repo.WriteSession(ctx, s.rep, repo.WriteSessionOptions{
Purpose: "PolicyPut",
}, func(ctx context.Context, w repo.RepositoryWriter) error {
return errors.Wrap(policy.SetPolicy(ctx, w, sourceInfo, newPolicy), "unable to set policy")
}); err != nil {
return nil, internalServerError(err)
}
if err := w.Flush(ctx); err != nil {
return nil, internalServerError(err)
}
s.triggerRefreshSource(sourceInfo)
return &serverapi.Empty{}, nil
}
internal/server/api_sources.go
+13 -27
View File
@@ -52,6 +52,10 @@ func (s *Server) handleSourcesCreate(ctx context.Context, r *http.Request, body
return nil, requestError(serverapi.ErrorMalformedRequest, "missing path")
}
if req.Policy == nil {
return nil, requestError(serverapi.ErrorMalformedRequest, "missing policy")
}
req.Path = ospath.ResolveUserFriendlyPath(req.Path, true)
_, err := os.Stat(req.Path)
@@ -71,33 +75,13 @@ func (s *Server) handleSourcesCreate(ctx context.Context, r *http.Request, body
resp := &serverapi.CreateSnapshotSourceResponse{}
// ensure we have the policy for this source, otherwise it will not show up in the
// list of sources at all.
_, err = policy.GetDefinedPolicy(ctx, s.rep, sourceInfo)
switch {
case err == nil:
// already have policy, do nothing
log(ctx).Debugf("policy for %v already exists", sourceInfo)
resp.Created = false
case errors.Is(err, policy.ErrPolicyNotFound):
resp.Created = true
// don't have policy - create an empty one
log(ctx).Debugf("policy for %v not found, creating empty one", sourceInfo)
if err = repo.WriteSession(ctx, s.rep, repo.WriteSessionOptions{
Purpose: "handleSourcesCreate",
}, func(ctx context.Context, w repo.RepositoryWriter) error {
// nolint:wrapcheck
return policy.SetPolicy(ctx, w, sourceInfo, &req.InitialPolicy)
}); err != nil {
return nil, internalServerError(errors.Wrap(err, "unable to set initial policy"))
}
default:
return nil, internalServerError(err)
if err = repo.WriteSession(ctx, s.rep, repo.WriteSessionOptions{
Purpose: "handleSourcesCreate",
}, func(ctx context.Context, w repo.RepositoryWriter) error {
// nolint:wrapcheck
return policy.SetPolicy(ctx, w, sourceInfo, req.Policy)
}); err != nil {
return nil, internalServerError(errors.Wrap(err, "unable to set initial policy"))
}
// upgrade to exclusive lock to ensure we have source manager
@@ -108,6 +92,8 @@ func (s *Server) handleSourcesCreate(ctx context.Context, r *http.Request, body
sm := newSourceManager(sourceInfo, s)
s.sourceManagers[sourceInfo] = sm
sm.refreshStatus(ctx)
go sm.run(ctx)
}
s.mu.Unlock()
internal/server/api_sources_test.go
+62 -1
View File
@@ -9,6 +9,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/kopia/kopia/internal/apiclient"
"github.com/kopia/kopia/internal/clock"
"github.com/kopia/kopia/internal/repotesting"
"github.com/kopia/kopia/internal/serverapi"
"github.com/kopia/kopia/internal/testutil"
@@ -33,7 +34,7 @@ func TestSnapshotCounters(t *testing.T) {
dir := testutil.TempDirectory(t)
si := localSource(env, dir)
mustCreateSource(t, cli, dir)
mustCreateSource(t, cli, dir, &policy.Policy{})
require.Len(t, mustListSources(t, cli, &snapshot.SourceInfo{}), 1)
mustSetPolicy(t, cli, si, &policy.Policy{
@@ -73,3 +74,63 @@ func TestSnapshotCounters(t *testing.T) {
require.Equal(t, ut.Counters["Excluded Files"], uitask.SimpleCounter(1))
require.Equal(t, ut.Counters["Processed Files"], uitask.SimpleCounter(3))
}
func TestSourceRefreshesAfterPolicy(t *testing.T) {
ctx, env := repotesting.NewEnvironment(t, repotesting.FormatNotImportant)
srvInfo := startServer(t, env, false)
_ = ctx
cli, err := apiclient.NewKopiaAPIClient(apiclient.Options{
BaseURL: srvInfo.BaseURL,
TrustedServerCertificateFingerprint: srvInfo.TrustedServerCertificateFingerprint,
Username: testUIUsername,
Password: testUIPassword,
})
require.NoError(t, err)
dir := testutil.TempDirectory(t)
si := localSource(env, dir)
currentHour := clock.Now().Hour()
mustCreateSource(t, cli, dir, &policy.Policy{
SchedulingPolicy: policy.SchedulingPolicy{
TimesOfDay: []policy.TimeOfDay{
{Hour: (currentHour + 2) % 24, Minute: 33},
},
},
})
sources := mustListSources(t, cli, &snapshot.SourceInfo{})
require.Len(t, sources, 1)
require.NotNil(t, sources[0].NextSnapshotTime)
require.Equal(t, 33, sources[0].NextSnapshotTime.Minute())
mustSetPolicy(t, cli, si, &policy.Policy{
SchedulingPolicy: policy.SchedulingPolicy{
TimesOfDay: []policy.TimeOfDay{
{Hour: (currentHour + 2) % 24, Minute: 55},
},
},
})
// make sure that soon after setting policy, the next snapshot time is up-to-date.
match := false
for attempt := 0; attempt < 3; attempt++ {
sources = mustListSources(t, cli, &snapshot.SourceInfo{})
require.Len(t, sources, 1)
require.NotNil(t, sources[0].NextSnapshotTime)
if sources[0].NextSnapshotTime.Minute() == 55 {
match = true
break
}
time.Sleep(time.Second)
}
require.True(t, match)
}
internal/server/server.go
+15
View File
@@ -96,6 +96,9 @@ func (s *Server) APIHandlers(legacyAPI bool) http.Handler {
m.HandleFunc("/api/v1/restore", s.handleAPI(requireUIUser, s.handleRestore)).Methods(http.MethodPost)
m.HandleFunc("/api/v1/estimate", s.handleAPI(requireUIUser, s.handleEstimate)).Methods(http.MethodPost)
// path APIs
m.HandleFunc("/api/v1/paths/resolve", s.handleAPI(requireUIUser, s.handlePathResolve)).Methods(http.MethodPost)
// methods that can be called by any authenticated user (UI or remote user).
m.HandleFunc("/api/v1/flush", s.handleAPI(anyAuthenticatedUser, s.handleFlush)).Methods(http.MethodPost)
m.HandleFunc("/api/v1/repo/status", s.handleAPIPossiblyNotConnected(anyAuthenticatedUser, s.handleRepoStatus)).Methods(http.MethodGet)
@@ -432,6 +435,18 @@ func (s *Server) endUpload(ctx context.Context, src snapshot.SourceInfo) {
<-s.uploadSemaphore
}
func (s *Server) triggerRefreshSource(sourceInfo snapshot.SourceInfo) {
sm := s.sourceManagers[sourceInfo]
if sm == nil {
return
}
select {
case sm.refreshRequested <- struct{}{}:
default:
}
}
// SetRepository sets the repository (nil is allowed and indicates server that is not
// connected to the repository).
func (s *Server) SetRepository(ctx context.Context, rep repo.Repository) error {
internal/server/source_manager.go
+9 -4
View File
@@ -21,7 +21,7 @@ import (
)
const (
statusRefreshInterval = 15 * time.Second // how frequently to refresh source status
statusRefreshInterval = 30 * time.Minute
failedSnapshotRetryInterval = 5 * time.Minute
refreshTimeout = 30 * time.Second // max amount of time to refresh a single source
oneDay = 24 * time.Hour
@@ -42,6 +42,7 @@ type sourceManager struct {
src snapshot.SourceInfo
closed chan struct{}
snapshotRequests chan struct{}
refreshRequested chan struct{} // tickled externally to trigger refresh
wg sync.WaitGroup
mu sync.RWMutex
@@ -142,6 +143,10 @@ func (s *sourceManager) runLocal(ctx context.Context) {
continue
case <-s.refreshRequested:
s.refreshStatus(ctx)
continue
case <-time.After(statusRefreshInterval):
s.refreshStatus(ctx)
@@ -360,12 +365,11 @@ func (s *sourceManager) refreshStatus(ctx context.Context) {
break
}
}
s.nextSnapshotTime = s.findClosestNextSnapshotTime()
} else {
s.nextSnapshotTime = nil
s.lastSnapshot = nil
}
s.nextSnapshotTime = s.findClosestNextSnapshotTime()
}
type uitaskProgress struct {
@@ -476,6 +480,7 @@ func newSourceManager(src snapshot.SourceInfo, server *Server) *sourceManager {
state: "UNKNOWN",
closed: make(chan struct{}),
snapshotRequests: make(chan struct{}, 1),
refreshRequested: make(chan struct{}, 1),
progress: &snapshotfs.CountingUploadProgress{},
}
internal/server/util_test.go
+3 -2
View File
@@ -16,11 +16,12 @@ import (
"github.com/kopia/kopia/snapshot/policy"
)
func mustCreateSource(t *testing.T, cli *apiclient.KopiaAPIClient, path string) {
func mustCreateSource(t *testing.T, cli *apiclient.KopiaAPIClient, path string, pol *policy.Policy) {
t.Helper()
_, err := serverapi.CreateSnapshotSource(testlogging.Context(t), cli, &serverapi.CreateSnapshotSourceRequest{
Path: path,
Path: path,
Policy: pol,
})
require.NoError(t, err)
}
internal/serverapi/serverapi.go
+16 -6
View File
@@ -136,14 +136,13 @@ type SupportedAlgorithmsResponse struct {
// CreateSnapshotSourceRequest contains request to create snapshot source and optionally create first snapshot.
type CreateSnapshotSourceRequest struct {
Path string `json:"path"`
CreateSnapshot bool `json:"createSnapshot"`
InitialPolicy policy.Policy `json:"initialPolicy"` // policy to set on the source when first created, ignored if already exists
Path string `json:"path"`
CreateSnapshot bool `json:"createSnapshot"`
Policy *policy.Policy `json:"policy"` // policy to set on the path
}
// CreateSnapshotSourceResponse contains response of creating snapshot source.
type CreateSnapshotSourceResponse struct {
Created bool `json:"created"` // whether the source was created (false==previously existed)
SnapshotStarted bool `json:"snapshotted"` // whether snapshotting has been started
}
@@ -217,8 +216,9 @@ type RestoreRequest struct {
// EstimateRequest contains request to estimate the size of the snapshot in a given root.
type EstimateRequest struct {
Root string `json:"root"`
MaxExamplesPerBucket int `json:"maxExamplesPerBucket"`
Root string `json:"root"`
MaxExamplesPerBucket int `json:"maxExamplesPerBucket"`
PolicyOverride *policy.Policy `json:"policyOverride"`
}
// ResolvePolicyRequest contains request structure to ResolvePolicy.
@@ -234,3 +234,13 @@ type ResolvePolicyResponse struct {
Defined *policy.Policy `json:"defined"`
UpcomingSnapshotTimes []time.Time `json:"upcomingSnapshotTimes"`
}
// ResolvePathRequest contains request to resolve a particular path to ResolvePathResponse.
type ResolvePathRequest struct {
Path string `json:"path"`
}
// ResolvePathResponse contains response to resolve a particular path.
type ResolvePathResponse struct {
SourceInfo snapshot.SourceInfo `json:"source"`
}
snapshot/policy/policy_manager.go
+25 -6
View File
@@ -41,7 +41,14 @@ var log = logging.Module("kopia/snapshot/policy")
// with parent policies. The source must contain a path.
// Returns the effective policies and all source policies that contributed to that (most specific first).
func GetEffectivePolicy(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) (effective *Policy, definition *Definition, sources []*Policy, e error) {
policies, err := GetPolicyHierarchy(ctx, rep, si)
return GetEffectivePolicyWithOverride(ctx, rep, si, nil)
}
// GetEffectivePolicyWithOverride calculates effective snapshot policy for a given source by combining the source-specifc policy (if any)
// with parent policies. The source must contain a path.
// Returns the effective policies and all source policies that contributed to that (most specific first).
func GetEffectivePolicyWithOverride(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo, optionalPolicyOverride *Policy) (effective *Policy, definition *Definition, sources []*Policy, e error) {
policies, err := GetPolicyHierarchy(ctx, rep, si, optionalPolicyOverride)
if err != nil {
return nil, nil, nil, errors.Wrap(err, "unable to get parent policies")
}
@@ -52,7 +59,7 @@ func GetEffectivePolicy(ctx context.Context, rep repo.Repository, si snapshot.So
}
// GetPolicyHierarchy returns the set of parent policies that apply to the path in most-specific-to-most-general order.
func GetPolicyHierarchy(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) ([]*Policy, error) {
func GetPolicyHierarchy(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo, optionalPolicyOverride *Policy) ([]*Policy, error) {
var md []*manifest.EntryMetadata
// Find policies applying to paths all the way up to the root.
@@ -62,7 +69,9 @@ func GetPolicyHierarchy(ctx context.Context, rep repo.Repository, si snapshot.So
return nil, errors.Wrapf(err, "unable to find manifest for source %v", tmp)
}
md = append(md, manifests...)
if si != tmp || optionalPolicyOverride == nil {
md = append(md, manifests...)
}
parentPath := getParentPathOSIndependent(tmp.Path)
if parentPath == tmp.Path {
@@ -98,6 +107,11 @@ func GetPolicyHierarchy(ctx context.Context, rep repo.Repository, si snapshot.So
var policies []*Policy
if optionalPolicyOverride != nil {
optionalPolicyOverride.Labels = LabelsForSource(si)
policies = append(policies, optionalPolicyOverride)
}
for _, em := range md {
p := &Policy{}
if err := loadPolicyFromManifest(ctx, rep, em.ID, p); err != nil {
@@ -232,7 +246,12 @@ func (m SubdirectoryPolicyMap) GetPolicyForPath(relativePath string) (*Policy, e
// TreeForSource returns policy Tree for a given source.
func TreeForSource(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) (*Tree, error) {
pols, err := applicablePoliciesForSource(ctx, rep, si)
return TreeForSourceWithOverride(ctx, rep, si, nil)
}
// TreeForSourceWithOverride returns policy Tree for a given source with the root policy overridden.
func TreeForSourceWithOverride(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo, optionalPolicyOverride *Policy) (*Tree, error) {
pols, err := applicablePoliciesForSource(ctx, rep, si, optionalPolicyOverride)
if err != nil {
return nil, errors.Wrap(err, "unable to get policies")
}
@@ -240,10 +259,10 @@ func TreeForSource(ctx context.Context, rep repo.Repository, si snapshot.SourceI
return BuildTree(pols, DefaultPolicy), nil
}
func applicablePoliciesForSource(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) (map[string]*Policy, error) {
func applicablePoliciesForSource(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo, optionalPolicyOverride *Policy) (map[string]*Policy, error) {
result := map[string]*Policy{}
pol, _, _, err := GetEffectivePolicy(ctx, rep, si)
pol, _, _, err := GetEffectivePolicyWithOverride(ctx, rep, si, optionalPolicyOverride)
if err != nil {
return nil, err
}
snapshot/policy/policy_manager_test.go
+1 -1
View File
@@ -432,7 +432,7 @@ func TestApplicablePoliciesForSource(t *testing.T) {
for _, tc := range cases {
tc := tc
t.Run(tc.si.String(), func(t *testing.T) {
res, err := applicablePoliciesForSource(ctx, env.RepositoryWriter, tc.si)
res, err := applicablePoliciesForSource(ctx, env.RepositoryWriter, tc.si, nil)
if err != nil {
t.Fatalf("error in applicablePoliciesForSource(%v): %v", tc.si, err)
}
tests/end_to_end_test/server_start_test.go
+5 -5
View File
@@ -127,11 +127,11 @@ func TestServerStart(t *testing.T) {
require.Equal(t, int64(0), et.Counters["Ignored Errors"].Value)
createResp, err := serverapi.CreateSnapshotSource(ctx, cli, &serverapi.CreateSnapshotSourceRequest{
Path: sharedTestDataDir2,
Path: sharedTestDataDir2,
Policy: &policy.Policy{},
})
require.NoError(t, err)
require.True(t, createResp.Created)
require.False(t, createResp.SnapshotStarted)
verifySourceCount(t, cli, nil, 2)
@@ -162,13 +162,13 @@ func TestServerStart(t *testing.T) {
keepDaily := 77
createResp, err = serverapi.CreateSnapshotSource(ctx, cli, &serverapi.CreateSnapshotSourceRequest{
Path: sharedTestDataDir3,
CreateSnapshot: true,
InitialPolicy: policy.Policy{
Path: sharedTestDataDir3,
Policy: &policy.Policy{
RetentionPolicy: policy.RetentionPolicy{
KeepDaily: &keepDaily,
},
},
CreateSnapshot: true,
})
require.NoError(t, err)