ui: Policy Editor - show effective value and definition point for policy fields (#1545)

* policy: resolve API for policy editor

* htmlui: enhanced Policy Editor UI to preview effective values
This commit is contained in:
Jarek Kowalski authored and GitHub committed 2021-11-30 21:40:41 -08:00
1 parent 93930d20cb
commit dc964bee43
14 files changed
+634 -48

No files matched your search

+12 -1
View File
@@ -33,8 +33,19 @@ body {
margin-top: 4px;
}
.policyEffectiveValue {
background-color: #fff;
}
.policyEditorHeader {
font-size: 110%;
font-weight: bold;
padding-top: 4px;
padding-bottom: 4px;
}
.policyValue {
margin-bottom: 4px;
background-color: #f8f8f8;
}
.policyField {
+165 -13
View File
@@ -1,6 +1,7 @@
import { faCalendarTimes, faChevronLeft, faClock, faExclamationTriangle, faFileAlt, faFileArchive, faFolderOpen, faMagic } from '@fortawesome/free-solid-svg-icons';
import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
import axios from 'axios';
import moment from 'moment';
import React, { Component } from 'react';
import Button from 'react-bootstrap/Button';
import Col from 'react-bootstrap/Col';
@@ -10,6 +11,7 @@ import Spinner from 'react-bootstrap/Spinner';
import Accordion from 'react-bootstrap/Accordion';
import { handleChange, LogDetailSelector, OptionalBoolean, OptionalNumberField, RequiredBoolean, stateProperty, StringList, valueToNumber } from './forms';
import { errorAlert, sourceQueryStringParams } from './uiutil';
import { getDeepStateProperty } from './deepstate';
function PolicyTypeName(s) {
if (!s.host && !s.userName) {
@@ -29,7 +31,7 @@ function PolicyTypeName(s) {
function LabelColumn(props) {
return <Col xs={12} sm={4} className="policyFieldColumn">
<span class="policyField">{props.name}</span>
<span className="policyField">{props.name}</span>
{props.help && <><p className="label-help">{props.help}</p></>}
</Col>
}
@@ -42,6 +44,85 @@ function WideValueColumn(props) {
return <Col xs={12} sm={4} className="policyValue">{props.children}</Col>;
}
function EffectiveValue(component, policyField) {
const dsp = getDeepStateProperty(component, "resolved.definition." + policyField, undefined);
return <EffectiveValueColumn>
<Form.Group>
<Form.Control
data-testid={'effective-' + policyField}
size="sm"
value={getDeepStateProperty(component, "resolved.effective." + policyField, undefined)}
readOnly={true} />
<Form.Text data-testid={'definition-' + policyField}>
{component.PolicyDefinitionPoint(dsp)}
</Form.Text>
</Form.Group>
</EffectiveValueColumn>;
}
function EffectiveTextAreaValue(component, policyField) {
const dsp = getDeepStateProperty(component, "resolved.definition." + policyField, undefined);
return <EffectiveValueColumn>
<Form.Group>
<Form.Control
data-testid={'effective-' + policyField}
size="sm"
as="textarea"
rows="5"
value={getDeepStateProperty(component, "resolved.effective." + policyField, undefined)}
readOnly={true} />
<Form.Text data-testid={'definition-' + policyField}>
{component.PolicyDefinitionPoint(dsp)}
</Form.Text>
</Form.Group>
</EffectiveValueColumn>;
}
function EffectiveBooleanValue(component, policyField) {
const dsp = getDeepStateProperty(component, "resolved.definition." + policyField, undefined);
return <EffectiveValueColumn>
<Form.Group>
<Form.Check
data-testid={'effective-' + policyField}
size="sm"
checked={getDeepStateProperty(component, "resolved.effective." + policyField, undefined)}
readOnly={true} />
<Form.Text data-testid={'definition-' + policyField}>
{component.PolicyDefinitionPoint(dsp)}
</Form.Text>
</Form.Group>
</EffectiveValueColumn>;
}
function EffectiveValueColumn(props) {
return <Col xs={12} sm={4} className="policyEffectiveValue">{props.children}</Col>;
}
function UpcomingSnapshotTimes(times) {
if (!times) {
return <LabelColumn name="No upcoming snapshots" />
}
return <>
<LabelColumn name-="Upcoming" />
<ul data-testid="upcoming-snapshot-times">
{times.map(x => <li key={x}>{moment(x).format('L LT')} ({moment(x).fromNow()})</li>)}
</ul>
</>;
}
function SectionHeaderRow() {
return <Row>
<LabelColumn />
<ValueColumn><div className="policyEditorHeader">Values</div></ValueColumn>
<EffectiveValueColumn><div className="policyEditorHeader">Effective</div></EffectiveValueColumn>
</Row>;
}
export class PolicyEditor extends Component {
constructor() {
super();
@@ -56,7 +137,9 @@ export class PolicyEditor extends Component {
this.saveChanges = this.saveChanges.bind(this);
this.isGlobal = this.isGlobal.bind(this);
this.deletePolicy = this.deletePolicy.bind(this);
this.snapshotURL = this.snapshotURL.bind(this);
this.policyURL = this.policyURL.bind(this);
this.resolvePolicy = this.resolvePolicy.bind(this);
this.PolicyDefinitionPoint = this.PolicyDefinitionPoint.bind(this);
}
componentDidMount() {
@@ -69,12 +152,16 @@ export class PolicyEditor extends Component {
});
}
componentWillReceiveProps(props) {
this.fetchPolicy(props);
componentDidUpdate() {
const pjs = JSON.stringify(this.state.policy);
if (pjs !== this.lastResolvedPolicy) {
this.resolvePolicy(this.props);
this.lastResolvedPolicy = pjs;
}
}
fetchPolicy(props) {
axios.get(this.snapshotURL(props)).then(result => {
axios.get(this.policyURL(props)).then(result => {
this.setState({
isLoading: false,
policy: result.data,
@@ -95,6 +182,30 @@ export class PolicyEditor extends Component {
});
}
resolvePolicy(props) {
const u = '/api/v1/policy/resolve?' + sourceQueryStringParams(props);
axios.post(u, {
"updates": this.state.policy,
"numUpcomingSnapshotTimes": 5,
}).then(result => {
this.setState({ resolved: result.data });
}).catch(error => {
});
}
PolicyDefinitionPoint(p) {
if (!p) {
return "";
}
if (p.userName === this.props.userName && p.host === this.props.host && p.path === this.props.path) {
return "(Defined by this policy)";
}
return <>Defined by <b>{PolicyTypeName(p)}</b></>;
}
saveChanges(e) {
e.preventDefault()
@@ -137,7 +248,7 @@ export class PolicyEditor extends Component {
}
this.setState({ saving: true });
axios.put(this.snapshotURL(this.props), policy).then(result => {
axios.put(this.policyURL(this.props), policy).then(result => {
this.props.close();
}).catch(error => {
this.setState({ saving: false });
@@ -149,7 +260,7 @@ export class PolicyEditor extends Component {
if (window.confirm('Are you sure you want to delete this policy?')) {
this.setState({ saving: true });
axios.delete(this.snapshotURL(this.props)).then(result => {
axios.delete(this.policyURL(this.props)).then(result => {
this.props.close();
}).catch(error => {
this.setState({ saving: false });
@@ -158,7 +269,7 @@ export class PolicyEditor extends Component {
}
}
snapshotURL(props) {
policyURL(props) {
return '/api/v1/policy?' + sourceQueryStringParams(props);
}
@@ -185,83 +296,100 @@ export class PolicyEditor extends Component {
<Accordion.Item eventKey="retention">
<Accordion.Header><FontAwesomeIcon icon={faCalendarTimes} />&nbsp;Snapshot Retention</Accordion.Header>
<Accordion.Body>
<SectionHeaderRow />
<Row>
<LabelColumn name="Latest Snapshots" help="Number of the most recent snapshots to retain per source." />
<ValueColumn>{OptionalNumberField(this, null, "policy.retention.keepLatest", { placeholder: "# of latest snapshots" })}</ValueColumn>
{EffectiveValue(this, "retention.keepLatest")}
</Row>
<Row>
<LabelColumn name="Hourly" help="How many hourly snapshots to retain per source. The latest snapshot from each hour will be retained." />
<ValueColumn>{OptionalNumberField(this, null, "policy.retention.keepHourly", { placeholder: "# of hourly snapshots" })}</ValueColumn>
{EffectiveValue(this, "retention.keepHourly")}
</Row>
<Row>
<LabelColumn name="Daily" help="How many daily snapshots to retain per source. The latest snapshot from each day will be retained." />
<ValueColumn>{OptionalNumberField(this, null, "policy.retention.keepDaily", { placeholder: "# of daily snapshots" })}</ValueColumn>
{EffectiveValue(this, "retention.keepDaily")}
</Row>
<Row>
<LabelColumn name="Weekly" help="How many weekly snapshots to retain per source. The latest snapshot from each week will be retained." />
<ValueColumn>{OptionalNumberField(this, null, "policy.retention.keepWeekly", { placeholder: "# of weekly snapshots" })}</ValueColumn>
{EffectiveValue(this, "retention.keepWeekly")}
</Row>
<Row>
<LabelColumn name="Monthly" help="How many monthly snapshots to retain per source. The latest snapshot from each calendar month will be retained." />
<ValueColumn>{OptionalNumberField(this, null, "policy.retention.keepMonthly", { placeholder: "# of monthly snapshots" })}</ValueColumn>
{EffectiveValue(this, "retention.keepMonthly")}
</Row>
<Row>
<LabelColumn name="Annual" help="How many annual snapshots to retain per source. The latest snapshot from each calendar year will be retained." />
<ValueColumn>{OptionalNumberField(this, null, "policy.retention.keepAnnual", { placeholder: "# of annual snapshots" })}</ValueColumn>
{EffectiveValue(this, "retention.keepAnnual")}
</Row>
</Accordion.Body>
</Accordion.Item>
<Accordion.Item eventKey="files">
<Accordion.Header><FontAwesomeIcon icon={faFolderOpen} />&nbsp;Files</Accordion.Header>
<Accordion.Body>
<SectionHeaderRow />
<Row>
<LabelColumn name="Ignore Files" help="List of file and directory names to ignore. The patterns should be specified as relative to the directory they are defined in and not absolute. Wilcards are allowed." />
<WideValueColumn>{StringList(this, "policy.files.ignore")}</WideValueColumn>
{EffectiveTextAreaValue(this, "files.ignore")}
</Row>
<Row>
<LabelColumn name="Ignore Rules From Parent Directories" help="When set, ignore rules from the parent directory are ignored." />
<ValueColumn>
{RequiredBoolean(this, "", "policy.files.noParentIgnore")}
</ValueColumn>
<ValueColumn>{RequiredBoolean(this, "", "policy.files.noParentIgnore")}</ValueColumn>
<EffectiveValueColumn />
</Row>
<Row>
<LabelColumn name="Ignore Rule Files" help="List of additional files containing ignore rules. Each file configures ignore rules for the directory and its subdirectories." />
<ValueColumn>{StringList(this, "policy.files.ignoreDotFiles")}</ValueColumn>
{EffectiveTextAreaValue(this, "files.ignoreDotFiles")}
</Row>
<Row>
<LabelColumn name="Ignore Rule Files From Parent Directories" help="When set, the files specifying ignore rules (.kopiaignore, etc.) from the parent directory are ignored." />
<ValueColumn>{RequiredBoolean(this, "", "policy.files.noParentDotFiles")}</ValueColumn>
<EffectiveValueColumn />
</Row>
<Row>
<LabelColumn name="Ignore Well-Known Cache Directories" help="Ignore directories containing CACHEDIR.TAG and similar." />
<ValueColumn>{OptionalBoolean(this, null, "policy.files.ignoreCacheDirs", "inherit from parent")}</ValueColumn>
{EffectiveBooleanValue(this, "files.ignoreCacheDirs")}
</Row>
<Row>
<LabelColumn name="Scan only one filesystem" help="Do not cross filesystem boundaries when snapshotting." />
<ValueColumn>{OptionalBoolean(this, null, "policy.files.oneFileSystem", "inherit from parent")}</ValueColumn>
{EffectiveBooleanValue(this, "files.oneFileSystem")}
</Row>
</Accordion.Body>
</Accordion.Item>
<Accordion.Item eventKey="errors">
<Accordion.Header><FontAwesomeIcon icon={faExclamationTriangle} />&nbsp;Error Handling</Accordion.Header>
<Accordion.Body>
<SectionHeaderRow />
<Row>
<LabelColumn name="Ignore Directory Errors" help="Treat directory read errors as non-fatal." />
<ValueColumn>{OptionalBoolean(this, null, "policy.errorHandling.ignoreDirectoryErrors", "inherit from parent")}</ValueColumn>
{EffectiveBooleanValue(this, "errorHandling.ignoreDirectoryErrors")}
</Row>
<Row>
<LabelColumn name="Ignore File Errors" help="Treat file read errors as non-fatal." />
<ValueColumn>{OptionalBoolean(this, null, "policy.errorHandling.ignoreFileErrors", "inherit from parent")}</ValueColumn>
{EffectiveBooleanValue(this, "errorHandling.ignoreFileErrors")}
</Row>
<Row>
<LabelColumn name="Ignore Unknown Directory Entries" help="Treat unrecognized/unsupported directory entries as non-fatal errors." />
<ValueColumn>{OptionalBoolean(this, null, "policy.errorHandling.ignoreUnknownTypes", "inherit from parent")}</ValueColumn>
{EffectiveBooleanValue(this, "errorHandling.ignoreUnknownTypes")}
</Row>
</Accordion.Body>
</Accordion.Item>
<Accordion.Item eventKey="compression">
<Accordion.Header><FontAwesomeIcon icon={faFileArchive} />&nbsp;Compression</Accordion.Header>
<Accordion.Body>
<SectionHeaderRow />
<Row>
<LabelColumn name="Compression Algorithm" help="Specify compression algorithm to use when snapshotting files in this directory and subdirectories." />
<WideValueColumn>
@@ -273,32 +401,38 @@ export class PolicyEditor extends Component {
{this.state.algorithms && this.state.algorithms.compression.map(x => <option key={x} value={x}>{x}</option>)}
</Form.Control>
</WideValueColumn>
{EffectiveValue(this, "compression.compressorName")}
</Row>
<Row>
<LabelColumn name="Minimum File Size" help="Files whose size is below the provided value will not be compressed." />
<ValueColumn>{OptionalNumberField(this, "", "policy.compression.minSize", { placeholder: "minimum file size in bytes" })}</ValueColumn>
{EffectiveValue(this, "compression.minSize")}
</Row>
<Row>
<LabelColumn name="Max File Size" help="Files whose size exceeds the provided value will not be compressed." />
<ValueColumn>{OptionalNumberField(this, "", "policy.compression.maxSize", { placeholder: "maximum file size in bytes" })}</ValueColumn>
{EffectiveValue(this, "compression.maxSize")}
</Row>
<Row>
<LabelColumn name="Only Compress Extensions" help="Only compress files with the following file extensions (one extension per line)" />
<WideValueColumn>
{StringList(this, "policy.compression.onlyCompress")}
</WideValueColumn>
{EffectiveTextAreaValue(this, "compression.onlyCompress")}
</Row>
<Row>
<LabelColumn name="Never Compress Extensions" help="Never compress the following file extensions (one extension per line)" />
<WideValueColumn>
{StringList(this, "policy.compression.neverCompress")}
</WideValueColumn>
{EffectiveTextAreaValue(this, "compression.neverCompress")}
</Row>
</Accordion.Body>
</Accordion.Item>
<Accordion.Item eventKey="scheduling">
<Accordion.Header><FontAwesomeIcon icon={faClock} />&nbsp;Scheduling</Accordion.Header>
<Accordion.Body>
<SectionHeaderRow />
<Row>
<LabelColumn name="Snapshot Frequency" help="How frequently to create snapshots in KopiaUI or kopia server. This option has no effect outside of the server mode." />
<WideValueColumn>
@@ -317,53 +451,70 @@ export class PolicyEditor extends Component {
<option value="43200">every 12 hours</option>
</Form.Control>
</WideValueColumn>
{EffectiveValue(this, "scheduling.intervalSeconds")}
</Row>
<Row>
<LabelColumn name="Manual Snapshots Only" help="Only create snapshots manually (disables scheduled snapshots)." />
<ValueColumn>
{OptionalBoolean(this, "", "policy.scheduling.manual", "inherit from parent")}
</ValueColumn>
{EffectiveBooleanValue(this, "scheduling.manual")}
</Row>
<Row>
<LabelColumn name="Upcoming Snapshots" help="Times of upcoming snapshots calculated based on policy parameters." />
<ValueColumn>
</ValueColumn>
<EffectiveValueColumn>
{UpcomingSnapshotTimes(this.state?.resolved?.upcomingSnapshotTimes)}
</EffectiveValueColumn>
</Row>
</Accordion.Body>
</Accordion.Item>
<Accordion.Item eventKey="logging">
<Accordion.Header><FontAwesomeIcon icon={faFileAlt} />&nbsp;Logging</Accordion.Header>
<Accordion.Body>
<SectionHeaderRow />
<Row>
<LabelColumn name="Directory Snapshotted" help="Log verbosity when a directory is snapshotted." />
<WideValueColumn>
{LogDetailSelector(this, "policy.logging.directories.snapshotted")}
</WideValueColumn>
{EffectiveValue(this, "logging.directories.snapshotted")}
</Row>
<Row>
<LabelColumn name="Directory Ignored" help="Log verbosity when a directory is ignored." />
<WideValueColumn>
{LogDetailSelector(this, "policy.logging.directories.ignored")}
</WideValueColumn>
{EffectiveValue(this, "logging.directories.ignored")}
</Row>
<Row>
<LabelColumn name="File Snapshotted" help="Log verbosity when a file, symbolic link, etc. is snapshotted." />
<WideValueColumn>
{LogDetailSelector(this, "policy.logging.entries.snapshotted")}
</WideValueColumn>
{EffectiveValue(this, "logging.entries.snapshotted")}
</Row>
<Row>
<LabelColumn name="File Ignored" help="Log verbosity when a file, symbolic link, etc. is ignored." />
<WideValueColumn>
{LogDetailSelector(this, "policy.logging.entries.ignored")}
</WideValueColumn>
{EffectiveValue(this, "logging.entries.ignored")}
</Row>
<Row>
<LabelColumn name="Cache Hit" help="Log verbosity when an cache is used instead of uploading the file." />
<WideValueColumn>
{LogDetailSelector(this, "policy.logging.entries.cacheHit")}
</WideValueColumn>
{EffectiveValue(this, "logging.entries.cacheHit")}
</Row>
<Row>
<LabelColumn name="Cache Miss" help="Log verbosity when an cache cannot be used and a file must be hashed." />
<WideValueColumn>
{LogDetailSelector(this, "policy.logging.entries.cacheHit")}
</WideValueColumn>
{EffectiveValue(this, "logging.entries.cacheMiss")}
</Row>
</Accordion.Body>
</Accordion.Item>
@@ -387,7 +538,7 @@ export class PolicyEditor extends Component {
</Accordion.Item>
</Accordion>
<Button size="sm" variant="success" type="submit" onClick={this.saveChanges} disabled={this.state.saving}>Save Policy</Button>
<Button size="sm" variant="success" type="submit" onClick={this.saveChanges} data-testid="button-save" disabled={this.state.saving}>Save Policy</Button>
{!this.state.isNew && <>&nbsp;
<Button size="sm" variant="danger" disabled={this.isGlobal() || this.state.saving} onClick={this.deletePolicy}>Delete Policy</Button>
</>}
@@ -395,7 +546,8 @@ export class PolicyEditor extends Component {
&nbsp;
<Spinner animation="border" variant="primary" size="sm" />
</>}
{/* <pre className="debug-json">{JSON.stringify(this.state, null, 4)}
</pre> */}
</Form>
</div>;
}
+144
View File
@@ -0,0 +1,144 @@
import { render, waitFor, logDOM } from '@testing-library/react';
import React from 'react';
import { PolicyEditor } from './PolicyEditor';
import { MemoryRouter } from 'react-router-dom';
import { setupAPIMock } from './tests/api_mocks';
import moment from 'moment';
import { changeControlValue, simulateClick } from './tests/testutils';
it('e2e', async () => {
let ref = React.createRef();
let serverMock = setupAPIMock();
const ust1 = "2021-01-01T12:00:00Z";
const ust2 = "2021-01-01T13:00:00Z";
// first request from client - will fetch current policy
serverMock.onGet('/api/v1/policy?userName=some-user&host=some-host&path=some-path').reply(200,
{
"retention": {
"keepLatest": 33,
}
}
);
// second request to resolve the policy.
serverMock.onPost('/api/v1/policy/resolve?userName=some-user&host=some-host&path=some-path', {
"updates": {
"retention": {
"keepLatest": 33
}
},
"numUpcomingSnapshotTimes": 5
}).reply(200,
{
"effective": {
"retention": {
"keepLatest": 33,
"keepHourly": 45,
}
},
"definition": {
"retention": {
"keepLatest": { "host": "h1", "userName": "some-user", "path": "some-path" },
"keepMonthly": { "host": "h1", "userName": "u1", "path": "p1" }
}
},
"upcomingSnapshotTimes": [
ust1,
ust2,
]
}
);
// second request to resolve the policy.
serverMock.onPost('/api/v1/policy/resolve?userName=some-user&host=some-host&path=some-path', {
"updates": {
"retention": {
"keepLatest": 44
}
},
"numUpcomingSnapshotTimes": 5
}).reply(200,
{
"effective": {
"retention": {
"keepLatest": 44,
"keepHourly": 45,
}
},
"definition": {
"retention": {
"keepLatest": { "host": "some-host", "userName": "some-user", "path": "some-path" },
"keepMonthly": { "host": "h1", "userName": "u1", "path": "p1" }
}
},
"upcomingSnapshotTimes": [
ust1,
ust2,
]
}
);
// request to save updated policy.
serverMock.onPut('/api/v1/policy?userName=some-user&host=some-host&path=some-path', {
"retention": {
"keepLatest": 44
}
}).reply(200,
{
"effective": {
"retention": {
"keepLatest": 33,
"keepHourly": 45,
}
},
"definition": {
"retention": {
"keepLatest": { "host": "h1", "userName": "some-user", "path": "some-path" },
"keepMonthly": { "host": "h1", "userName": "u1", "path": "p1" }
}
},
"upcomingSnapshotTimes": [
ust1,
ust2,
]
}
);
let closeCalled = 0;
function closeFunc() {
closeCalled++;
}
const { getByTestId } = render(<MemoryRouter>
<PolicyEditor ref={ref} host="some-host" userName="some-user" path="some-path" close={closeFunc} />
</MemoryRouter>)
await waitFor(() => expect(getByTestId("effective-retention.keepHourly").value).toBe("45"));
expect(getByTestId("effective-retention.keepLatest").value).toEqual("33");
if (false) {
logDOM(getByTestId("definition-retention.keepHourly"));
}
await waitFor(() => expect(getByTestId("definition-retention.keepLatest").innerHTML).toEqual("Defined by <b>Directory: some-user@h1:some-path</b>"));
await waitFor(() => expect(getByTestId("definition-retention.keepMonthly").innerHTML).toEqual("Defined by <b>Directory: u1@h1:p1</b>"));
const expectedUpcomingSnapshotTimes = [
`<li>${moment(ust1).format('L LT')} (${moment(ust1).fromNow()})</li>`,
`<li>${moment(ust2).format('L LT')} (${moment(ust2).fromNow()})</li>`,
].join("");
await waitFor(() => expect(getByTestId("upcoming-snapshot-times").innerHTML).toEqual(expectedUpcomingSnapshotTimes));
// change a field
changeControlValue(getByTestId('control-policy.retention.keepLatest'), "44");
// this will trigger resolve and will update effective field: "(Defined by this policy)"
await waitFor(() => expect(getByTestId("effective-retention.keepLatest").value).toBe("44"));
await waitFor(() => expect(getByTestId("definition-retention.keepLatest").innerHTML).toEqual("(Defined by this policy)"));
simulateClick(getByTestId('button-save'));
await waitFor(() => expect(serverMock.history.put.length).toEqual(1));
await waitFor(() => expect(closeCalled).toEqual(1));
});
+44
View File
@@ -5,9 +5,11 @@ import (
"encoding/json"
"net/http"
"net/url"
"time"
"github.com/pkg/errors"
"github.com/kopia/kopia/internal/clock"
"github.com/kopia/kopia/internal/serverapi"
"github.com/kopia/kopia/repo"
"github.com/kopia/kopia/snapshot"
@@ -61,6 +63,48 @@ func (s *Server) handlePolicyGet(ctx context.Context, r *http.Request, body []by
return pol, nil
}
func (s *Server) handlePolicyResolve(ctx context.Context, r *http.Request, body []byte) (interface{}, *apiError) {
var req serverapi.ResolvePolicyRequest
if err := json.Unmarshal(body, &req); err != nil {
return nil, requestError(serverapi.ErrorMalformedRequest, "unable to decode request: "+err.Error())
}
target := getPolicyTargetFromURL(r.URL)
// build a list of parents
policies, err := policy.GetPolicyHierarchy(ctx, s.rep, target)
if err != nil {
return nil, internalServerError(err)
}
resp := &serverapi.ResolvePolicyResponse{
Defined: policies[0],
}
if req.Updates != nil {
policies[0] = req.Updates
policies[0].Labels = policy.LabelsForSource(target)
}
resp.Effective, resp.Definition = policy.MergePolicies(policies, target)
resp.UpcomingSnapshotTimes = []time.Time{}
now := clock.Now().Local()
for i := 0; i < req.NumUpcomingSnapshotTimes; i++ {
st, ok := resp.Effective.SchedulingPolicy.NextSnapshotTime(now, now)
if !ok {
break
}
resp.UpcomingSnapshotTimes = append(resp.UpcomingSnapshotTimes, st)
now = st.Add(1 * time.Second)
}
return resp, nil
}
func (s *Server) handlePolicyDelete(ctx context.Context, r *http.Request, body []byte) (interface{}, *apiError) {
w, ok := s.rep.(repo.RepositoryWriter)
if !ok {
+164
View File
@@ -0,0 +1,164 @@
package server_test
import (
"fmt"
"path/filepath"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/kopia/kopia/internal/apiclient"
"github.com/kopia/kopia/internal/repotesting"
"github.com/kopia/kopia/internal/serverapi"
"github.com/kopia/kopia/internal/testutil"
"github.com/kopia/kopia/repo/compression"
"github.com/kopia/kopia/snapshot"
"github.com/kopia/kopia/snapshot/policy"
)
func TestPolicies(t *testing.T) {
ctx, env := repotesting.NewEnvironment(t, repotesting.FormatNotImportant)
srvInfo := startServer(t, env, false)
cli, err := apiclient.NewKopiaAPIClient(apiclient.Options{
BaseURL: srvInfo.BaseURL,
TrustedServerCertificateFingerprint: srvInfo.TrustedServerCertificateFingerprint,
Username: testUIUsername,
Password: testUIPassword,
})
require.NoError(t, err)
dir0 := testutil.TempDirectory(t)
si0 := localSource(env, dir0)
dir1 := testutil.TempDirectory(t)
si1 := localSource(env, dir1)
dir2 := filepath.Join(dir1, "subdir1")
si2 := localSource(env, dir2)
dir3 := filepath.Join(dir2, "subdir2")
si3 := localSource(env, dir3)
dir4 := filepath.Join(dir3, "subdir3")
si4 := localSource(env, dir4)
mustSetPolicy(t, cli, si1, &policy.Policy{
CompressionPolicy: policy.CompressionPolicy{
CompressorName: "some-compressor",
},
})
mustSetPolicy(t, cli, si2, &policy.Policy{
CompressionPolicy: policy.CompressionPolicy{
NeverCompress: []string{"a", "b"},
},
SchedulingPolicy: policy.SchedulingPolicy{
IntervalSeconds: 60,
},
})
mustSetPolicy(t, cli, si3, &policy.Policy{
CompressionPolicy: policy.CompressionPolicy{
CompressorName: "some-compressor3",
},
})
cases := []struct {
si snapshot.SourceInfo
updates *policy.Policy
wantCompressorName compression.Name
wantCompressorNameSource snapshot.SourceInfo
wantNeverCompress []string
wantNeverCompressSource snapshot.SourceInfo
wantUpcomingSnapshotTimesLength int
}{
{
si: si0,
wantCompressorName: compression.Name("none"),
wantNeverCompress: nil,
wantCompressorNameSource: policy.GlobalPolicySourceInfo,
wantNeverCompressSource: policy.GlobalPolicySourceInfo,
},
{
si: si1,
wantCompressorName: compression.Name("some-compressor"),
wantNeverCompress: nil,
wantCompressorNameSource: si1,
wantNeverCompressSource: policy.GlobalPolicySourceInfo,
},
{
si: si1,
wantCompressorName: compression.Name("some-compressor-updated"),
wantNeverCompress: nil,
wantCompressorNameSource: si1,
wantNeverCompressSource: policy.GlobalPolicySourceInfo,
updates: &policy.Policy{
CompressionPolicy: policy.CompressionPolicy{
CompressorName: "some-compressor-updated",
},
},
},
{
si: si2,
wantCompressorName: compression.Name("some-compressor"), // inherited
wantNeverCompress: []string{"a", "b"},
wantCompressorNameSource: si1,
wantNeverCompressSource: si2,
wantUpcomingSnapshotTimesLength: 3,
},
{
si: si3,
wantCompressorName: compression.Name("some-compressor3"),
wantNeverCompress: []string{"a", "b"},
wantCompressorNameSource: si3,
wantNeverCompressSource: si2,
wantUpcomingSnapshotTimesLength: 3,
},
{
si: si4,
wantCompressorName: compression.Name("some-compressor3"),
wantNeverCompress: []string{"a", "b"},
wantCompressorNameSource: si3,
wantNeverCompressSource: si2,
wantUpcomingSnapshotTimesLength: 3,
},
{
si: si4,
wantCompressorName: compression.Name("some-compressor-updated"),
wantNeverCompress: []string{"a", "b"},
wantCompressorNameSource: si4,
wantNeverCompressSource: si2,
updates: &policy.Policy{
CompressionPolicy: policy.CompressionPolicy{
CompressorName: "some-compressor-updated",
},
},
wantUpcomingSnapshotTimesLength: 3,
},
}
for i, tc := range cases {
t.Run(fmt.Sprintf("case-%v", i), func(t *testing.T) {
res, err2 := serverapi.ResolvePolicy(ctx, cli, tc.si, &serverapi.ResolvePolicyRequest{
Updates: tc.updates,
NumUpcomingSnapshotTimes: 3,
})
require.NoError(t, err2)
require.Equal(t, tc.wantCompressorName, res.Effective.CompressionPolicy.CompressorName)
require.Equal(t, tc.wantNeverCompress, res.Effective.CompressionPolicy.NeverCompress)
require.Equal(t, tc.wantCompressorNameSource, res.Definition.CompressionPolicy.CompressorName)
require.Equal(t, tc.wantNeverCompressSource, res.Definition.CompressionPolicy.NeverCompress)
require.Len(t, res.UpcomingSnapshotTimes, tc.wantUpcomingSnapshotTimesLength)
for j, ust := range res.UpcomingSnapshotTimes {
require.Equal(t, ust.Truncate(60*time.Second), ust)
if j > 0 {
require.Equal(t, 60*time.Second, ust.Sub(res.UpcomingSnapshotTimes[j-1]))
}
}
})
}
}
+1
View File
@@ -85,6 +85,7 @@ func (s *Server) APIHandlers(legacyAPI bool) http.Handler {
m.HandleFunc("/api/v1/policy", s.handleAPI(requireUIUser, s.handlePolicyGet)).Methods(http.MethodGet)
m.HandleFunc("/api/v1/policy", s.handleAPI(requireUIUser, s.handlePolicyPut)).Methods(http.MethodPut)
m.HandleFunc("/api/v1/policy", s.handleAPI(requireUIUser, s.handlePolicyDelete)).Methods(http.MethodDelete)
m.HandleFunc("/api/v1/policy/resolve", s.handleAPI(requireUIUser, s.handlePolicyResolve)).Methods(http.MethodPost)
m.HandleFunc("/api/v1/policies", s.handleAPI(requireUIUser, s.handlePolicyList)).Methods(http.MethodGet)
+11
View File
@@ -162,6 +162,17 @@ func SetPolicy(ctx context.Context, c *apiclient.KopiaAPIClient, si snapshot.Sou
return nil
}
// ResolvePolicy resolves the policy.
func ResolvePolicy(ctx context.Context, c *apiclient.KopiaAPIClient, si snapshot.SourceInfo, req *ResolvePolicyRequest) (*ResolvePolicyResponse, error) {
resp := &ResolvePolicyResponse{}
if err := c.Post(ctx, "policy/resolve?"+policyTargetURLParamters(si), req, resp); err != nil {
return nil, errors.Wrap(err, "ResolvePolicy")
}
return resp, nil
}
// ListTasks lists the tasks.
func ListTasks(ctx context.Context, c *apiclient.KopiaAPIClient) (*TaskListResponse, error) {
resp := &TaskListResponse{}
+14
View File
@@ -220,3 +220,17 @@ type EstimateRequest struct {
Root string `json:"root"`
MaxExamplesPerBucket int `json:"maxExamplesPerBucket"`
}
// ResolvePolicyRequest contains request structure to ResolvePolicy.
type ResolvePolicyRequest struct {
Updates *policy.Policy `json:"updates"`
NumUpcomingSnapshotTimes int `json:"numUpcomingSnapshotTimes"` // if > 0, return N UpcomingSnapshotTimes
}
// ResolvePolicyResponse returns the resolved details about a single policy.
type ResolvePolicyResponse struct {
Effective *policy.Policy `json:"effective"`
Definition *policy.Definition `json:"definition"`
Defined *policy.Policy `json:"defined"`
UpcomingSnapshotTimes []time.Time `json:"upcomingSnapshotTimes"`
}
+32 -17
View File
@@ -41,13 +41,25 @@ var log = logging.Module("kopia/snapshot/policy")
// with parent policies. The source must contain a path.
// Returns the effective policies and all source policies that contributed to that (most specific first).
func GetEffectivePolicy(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) (effective *Policy, definition *Definition, sources []*Policy, e error) {
policies, err := GetPolicyHierarchy(ctx, rep, si)
if err != nil {
return nil, nil, nil, errors.Wrap(err, "unable to get parent policies")
}
merged, def := MergePolicies(policies, si)
return merged, def, policies, nil
}
// GetPolicyHierarchy returns the set of parent policies that apply to the path in most-specific-to-most-general order.
func GetPolicyHierarchy(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) ([]*Policy, error) {
var md []*manifest.EntryMetadata
// Find policies applying to paths all the way up to the root.
for tmp := si; len(si.Path) > 0; {
manifests, err := rep.FindManifests(ctx, labelsForSource(tmp))
manifests, err := rep.FindManifests(ctx, LabelsForSource(tmp))
if err != nil {
return nil, nil, nil, errors.Wrapf(err, "unable to find manifest for source %v", tmp)
return nil, errors.Wrapf(err, "unable to find manifest for source %v", tmp)
}
md = append(md, manifests...)
@@ -61,25 +73,25 @@ func GetEffectivePolicy(ctx context.Context, rep repo.Repository, si snapshot.So
}
// Try user@host policy
userHostManifests, err := rep.FindManifests(ctx, labelsForSource(snapshot.SourceInfo{Host: si.Host, UserName: si.UserName}))
userHostManifests, err := rep.FindManifests(ctx, LabelsForSource(snapshot.SourceInfo{Host: si.Host, UserName: si.UserName}))
if err != nil {
return nil, nil, nil, errors.Wrap(err, "unable to find user@host manifest")
return nil, errors.Wrap(err, "unable to find user@host manifest")
}
md = append(md, userHostManifests...)
// Try host-level policy.
hostManifests, err := rep.FindManifests(ctx, labelsForSource(snapshot.SourceInfo{Host: si.Host}))
hostManifests, err := rep.FindManifests(ctx, LabelsForSource(snapshot.SourceInfo{Host: si.Host}))
if err != nil {
return nil, nil, nil, errors.Wrap(err, "unable to find host-level manifest")
return nil, errors.Wrap(err, "unable to find host-level manifest")
}
md = append(md, hostManifests...)
// Global policy.
globalManifests, err := rep.FindManifests(ctx, labelsForSource(GlobalPolicySourceInfo))
globalManifests, err := rep.FindManifests(ctx, LabelsForSource(GlobalPolicySourceInfo))
if err != nil {
return nil, nil, nil, errors.Wrap(err, "unable to find global manifest")
return nil, errors.Wrap(err, "unable to find global manifest")
}
md = append(md, globalManifests...)
@@ -89,21 +101,23 @@ func GetEffectivePolicy(ctx context.Context, rep repo.Repository, si snapshot.So
for _, em := range md {
p := &Policy{}
if err := loadPolicyFromManifest(ctx, rep, em.ID, p); err != nil {
return nil, nil, nil, errors.Wrapf(err, "got unexpected error when loading policy item %v", em.ID)
return nil, errors.Wrapf(err, "got unexpected error when loading policy item %v", em.ID)
}
policies = append(policies, p)
}
merged, def := MergePolicies(policies)
merged.Labels = labelsForSource(si)
// add artificial empty policy for the source.
if len(policies) == 0 || policies[0].Target() != si {
policies = append([]*Policy{{Labels: LabelsForSource(si)}}, policies...)
}
return merged, def, policies, nil
return policies, nil
}
// GetDefinedPolicy returns the policy defined on the provided snapshot.SourceInfo or ErrPolicyNotFound if not present.
func GetDefinedPolicy(ctx context.Context, rep repo.Repository, si snapshot.SourceInfo) (*Policy, error) {
md, err := rep.FindManifests(ctx, labelsForSource(si))
md, err := rep.FindManifests(ctx, LabelsForSource(si))
if err != nil {
return nil, errors.Wrap(err, "unable to find policy for source")
}
@@ -140,12 +154,12 @@ func SetPolicy(ctx context.Context, rep repo.RepositoryWriter, si snapshot.Sourc
}
}
md, err := rep.FindManifests(ctx, labelsForSource(si))
md, err := rep.FindManifests(ctx, LabelsForSource(si))
if err != nil {
return errors.Wrapf(err, "unable to load manifests for %v", si)
}
if _, err := rep.PutManifest(ctx, labelsForSource(si), pol); err != nil {
if _, err := rep.PutManifest(ctx, LabelsForSource(si), pol); err != nil {
return errors.Wrap(err, "error writing policy manifest")
}
@@ -160,7 +174,7 @@ func SetPolicy(ctx context.Context, rep repo.RepositoryWriter, si snapshot.Sourc
// RemovePolicy removes the policy for a given source.
func RemovePolicy(ctx context.Context, rep repo.RepositoryWriter, si snapshot.SourceInfo) error {
md, err := rep.FindManifests(ctx, labelsForSource(si))
md, err := rep.FindManifests(ctx, LabelsForSource(si))
if err != nil {
return errors.Wrapf(err, "unable to load manifests for %v", si)
}
@@ -290,7 +304,8 @@ func loadPolicyFromManifest(ctx context.Context, rep repo.Repository, id manifes
return nil
}
func labelsForSource(si snapshot.SourceInfo) map[string]string {
// LabelsForSource returns the set of labels that a given source will have on a policy.
func LabelsForSource(si snapshot.SourceInfo) map[string]string {
switch {
case si.Path != "":
return map[string]string{
+12
View File
@@ -55,6 +55,9 @@ func TestPolicyManagerInheritanceTest(t *testing.T) {
{
sourceInfo: GlobalPolicySourceInfo,
wantEffective: defaultPolicyWithLabels,
wantSources: []snapshot.SourceInfo{
GlobalPolicySourceInfo,
},
},
{
sourceInfo: snapshot.SourceInfo{
@@ -69,6 +72,13 @@ func TestPolicyManagerInheritanceTest(t *testing.T) {
"path": "/some/path",
"username": "myuser",
}),
wantSources: []snapshot.SourceInfo{
{
UserName: "myuser",
Host: "host-c",
Path: "/some/path",
},
},
},
{
sourceInfo: snapshot.SourceInfo{
@@ -84,6 +94,7 @@ func TestPolicyManagerInheritanceTest(t *testing.T) {
"username": "myuser",
}),
wantSources: []snapshot.SourceInfo{
{UserName: "myuser", Host: "host-a", Path: "/some/path"},
{Host: "host-a"},
},
wantDef: Definition{
@@ -106,6 +117,7 @@ func TestPolicyManagerInheritanceTest(t *testing.T) {
"username": "myuser",
}),
wantSources: []snapshot.SourceInfo{
{UserName: "myuser", Path: "/some/path2/nested", Host: "host-a"},
{UserName: "myuser", Path: "/some/path2", Host: "host-a"},
{Host: "host-a"},
},
+3 -1
View File
@@ -9,12 +9,14 @@ import (
// MergePolicies computes the policy by applying the specified list of policies in order from most
// specific to most general.
func MergePolicies(policies []*Policy) (*Policy, *Definition) {
func MergePolicies(policies []*Policy, si snapshot.SourceInfo) (*Policy, *Definition) {
var (
merged Policy
def Definition
)
merged.Labels = LabelsForSource(si)
for _, p := range policies {
merged.RetentionPolicy.Merge(p.RetentionPolicy, &def.RetentionPolicy, p.Target())
merged.FilesPolicy.Merge(p.FilesPolicy, &def.FilesPolicy, p.Target())
+12 -12
View File
@@ -168,16 +168,16 @@ func testPolicyMergeSingleField(t *testing.T, fieldName string, typ reflect.Type
disableParentMerging(defaultPolicyMod)
// merging default policy with no-op policy results in default policy
result, _ := policy.MergePolicies([]*policy.Policy{pol0, defaultPolicyMod})
result, _ := policy.MergePolicies([]*policy.Policy{pol0, defaultPolicyMod}, pol0.Target())
require.Equal(t, defaultPolicyMod.String(), result.String())
result, _ = policy.MergePolicies([]*policy.Policy{pol1, pol0, defaultPolicyMod})
result, _ = policy.MergePolicies([]*policy.Policy{pol1, pol0, defaultPolicyMod}, pol1.Target())
require.Equal(t, pol1.String(), result.String())
result, _ = policy.MergePolicies([]*policy.Policy{pol2, pol1, pol0, defaultPolicyMod})
result, _ = policy.MergePolicies([]*policy.Policy{pol2, pol1, pol0, defaultPolicyMod}, pol2.Target())
require.Equal(t, pol2.String(), result.String())
result, _ = policy.MergePolicies([]*policy.Policy{pol2, pol0, defaultPolicyMod})
result, _ = policy.MergePolicies([]*policy.Policy{pol2, pol0, defaultPolicyMod}, pol2.Target())
require.Equal(t, pol2.String(), result.String())
}
@@ -225,7 +225,7 @@ func TestPolicyMergeOnlyCompressIncludingParents(t *testing.T) {
},
}
result, _ := policy.MergePolicies([]*policy.Policy{p0, p1, p2})
result, _ := policy.MergePolicies([]*policy.Policy{p0, p1, p2}, p0.Target())
want := *policy.DefaultPolicy
want.CompressionPolicy.OnlyCompress = []string{"a", "b", "c", "d", "e", "f", "g"}
@@ -233,7 +233,7 @@ func TestPolicyMergeOnlyCompressIncludingParents(t *testing.T) {
require.Equal(t, want.String(), result.String())
p2.NoParent = true
result, _ = policy.MergePolicies([]*policy.Policy{p0, p1, p2})
result, _ = policy.MergePolicies([]*policy.Policy{p0, p1, p2}, p0.Target())
want = policy.Policy{}
want.CompressionPolicy.OnlyCompress = []string{"a", "b", "c", "d", "e", "f", "g"}
@@ -241,7 +241,7 @@ func TestPolicyMergeOnlyCompressIncludingParents(t *testing.T) {
require.Equal(t, want.String(), result.String())
p1.NoParent = true
result, _ = policy.MergePolicies([]*policy.Policy{p0, p1, p2})
result, _ = policy.MergePolicies([]*policy.Policy{p0, p1, p2}, p0.Target())
want = policy.Policy{}
want.CompressionPolicy.OnlyCompress = []string{"a", "b", "c", "d", "e"}
@@ -249,7 +249,7 @@ func TestPolicyMergeOnlyCompressIncludingParents(t *testing.T) {
require.Equal(t, want.String(), result.String())
p0.NoParent = true
result, _ = policy.MergePolicies([]*policy.Policy{p0, p1, p2})
result, _ = policy.MergePolicies([]*policy.Policy{p0, p1, p2}, p0.Target())
want = policy.Policy{}
want.CompressionPolicy.OnlyCompress = []string{"a", "c", "e"}
@@ -284,7 +284,7 @@ func TestPolicyMergeTimesOfDayIncludingParents(t *testing.T) {
},
}
result, _ := policy.MergePolicies([]*policy.Policy{p0, p1, p2})
result, _ := policy.MergePolicies([]*policy.Policy{p0, p1, p2}, p0.Target())
want := *policy.DefaultPolicy
want.SchedulingPolicy.TimesOfDay = []policy.TimeOfDay{tod0, tod1, tod2, tod3, tod4, tod5, tod6}
@@ -292,7 +292,7 @@ func TestPolicyMergeTimesOfDayIncludingParents(t *testing.T) {
require.Equal(t, want.String(), result.String())
p2.NoParent = true
result, _ = policy.MergePolicies([]*policy.Policy{p0, p1, p2})
result, _ = policy.MergePolicies([]*policy.Policy{p0, p1, p2}, p0.Target())
want = policy.Policy{}
want.SchedulingPolicy.TimesOfDay = []policy.TimeOfDay{tod0, tod1, tod2, tod3, tod4, tod5, tod6}
@@ -300,7 +300,7 @@ func TestPolicyMergeTimesOfDayIncludingParents(t *testing.T) {
require.Equal(t, want.String(), result.String())
p1.NoParent = true
result, _ = policy.MergePolicies([]*policy.Policy{p0, p1, p2})
result, _ = policy.MergePolicies([]*policy.Policy{p0, p1, p2}, p0.Target())
want = policy.Policy{}
want.SchedulingPolicy.TimesOfDay = []policy.TimeOfDay{tod0, tod1, tod2, tod3, tod4}
@@ -308,7 +308,7 @@ func TestPolicyMergeTimesOfDayIncludingParents(t *testing.T) {
require.Equal(t, want.String(), result.String())
p0.NoParent = true
result, _ = policy.MergePolicies([]*policy.Policy{p0, p1, p2})
result, _ = policy.MergePolicies([]*policy.Policy{p0, p1, p2}, p0.Target())
want = policy.Policy{}
want.SchedulingPolicy.TimesOfDay = []policy.TimeOfDay{tod0, tod2, tod4}
+3
View File
@@ -88,6 +88,9 @@ func (p *SchedulingPolicy) NextSnapshotTime(previousSnapshotTime, now time.Time)
ok bool
)
now = now.Local()
previousSnapshotTime = previousSnapshotTime.Local()
// compute next snapshot time based on interval
if interval := p.IntervalSeconds; interval != 0 {
interval := time.Duration(interval) * time.Second
+17 -4
View File
@@ -1,6 +1,7 @@
package policy_test
import (
"fmt"
"testing"
"time"
@@ -151,12 +152,24 @@ func TestNextSnapshotTime(t *testing.T) {
wantTime: time.Date(2020, time.January, 2, 11, 54, 0, 0, time.Local),
wantOK: true,
},
{
pol: policy.SchedulingPolicy{
IntervalSeconds: 43200,
TimesOfDay: []policy.TimeOfDay{{19, 0}, {20, 0}},
},
previousSnapshotTime: time.Date(2020, time.January, 1, 19, 0, 0, 0, time.Local),
now: time.Date(2020, time.January, 1, 10, 0, 0, 0, time.Local),
wantTime: time.Date(2020, time.January, 1, 19, 0, 0, 0, time.Local),
wantOK: true,
},
}
for _, tc := range cases {
gotTime, gotOK := tc.pol.NextSnapshotTime(tc.previousSnapshotTime, tc.now)
for i, tc := range cases {
t.Run(fmt.Sprintf("case-%v", i), func(t *testing.T) {
gotTime, gotOK := tc.pol.NextSnapshotTime(tc.previousSnapshotTime, tc.now)
require.Equal(t, tc.wantTime, gotTime)
require.Equal(t, tc.wantOK, gotOK)
require.Equal(t, tc.wantTime, gotTime)
require.Equal(t, tc.wantOK, gotOK)
})
}
}