feat(cli): improved snapshot reports (#4244)

This commit is contained in:
Jarek Kowalski
2024-11-15 22:09:07 -08:00
committed by GitHub
parent 91d00e8256
commit f73887f4d6
10 changed files with 471 additions and 53 deletions

View File

@@ -280,7 +280,7 @@ func (c *commandSnapshotCreate) snapshotSingleSource(
var mwe notifydata.ManifestWithError
mwe.Source = sourceInfo
mwe.Manifest.Source = sourceInfo
st.Snapshots = append(st.Snapshots, &mwe)
@@ -297,6 +297,10 @@ func (c *commandSnapshotCreate) snapshotSingleSource(
return finalErr
}
if len(previous) > 0 {
mwe.Previous = previous[0]
}
policyTree, finalErr := policy.TreeForSource(ctx, rep, sourceInfo)
if finalErr != nil {
return errors.Wrap(finalErr, "unable to get policy tree")

View File

@@ -6,44 +6,94 @@
"github.com/kopia/kopia/snapshot"
)
const durationPrecision = 100 * time.Millisecond
// ManifestWithError represents information about the snapshot manifest with optional error.
type ManifestWithError struct {
snapshot.Manifest `json:"manifest"` // may not be filled out if there was an error, Manifst.Source is always set.
Manifest snapshot.Manifest `json:"manifest"` // may not be filled out if there was an error, Manifst.Source is always set.
Previous *snapshot.Manifest `json:"previous"` // may not be filled out
Error string `json:"error"` // will be present if there was an error
}
// StartTimestamp returns the start time of the snapshot.
func (m *ManifestWithError) StartTimestamp() time.Time {
return m.StartTime.ToTime().UTC().Truncate(time.Second)
return m.Manifest.StartTime.ToTime().UTC().Truncate(time.Second)
}
// EndTimestamp returns the end time of the snapshot.
func (m *ManifestWithError) EndTimestamp() time.Time {
return m.EndTime.ToTime().UTC().Truncate(time.Second)
return m.Manifest.EndTime.ToTime().UTC().Truncate(time.Second)
}
// TotalSize returns the total size of the snapshot in bytes.
func (m *ManifestWithError) TotalSize() int64 {
if m.RootEntry == nil {
if m.Manifest.RootEntry == nil {
return 0
}
if m.RootEntry.DirSummary != nil {
return m.RootEntry.DirSummary.TotalFileSize
if m.Manifest.RootEntry.DirSummary != nil {
return m.Manifest.RootEntry.DirSummary.TotalFileSize
}
return m.RootEntry.FileSize
return m.Manifest.RootEntry.FileSize
}
// TotalSizeDelta returns the total size of the snapshot in bytes.
func (m *ManifestWithError) TotalSizeDelta() int64 {
if m.Previous == nil {
return 0
}
if m.Manifest.RootEntry == nil {
return 0
}
if m.Manifest.RootEntry.DirSummary != nil && m.Previous.RootEntry.DirSummary != nil {
return m.Manifest.RootEntry.DirSummary.TotalFileSize - m.Previous.RootEntry.DirSummary.TotalFileSize
}
return m.Manifest.RootEntry.FileSize
}
// New returns the total size of the snapshot in bytes.
func (m *ManifestWithError) New() int64 {
if m.Manifest.RootEntry == nil {
return 0
}
if m.Manifest.RootEntry.DirSummary != nil {
return m.Manifest.RootEntry.DirSummary.TotalFileSize
}
return m.Manifest.RootEntry.FileSize
}
// TotalFiles returns the total number of files in the snapshot.
func (m *ManifestWithError) TotalFiles() int64 {
if m.RootEntry == nil {
if m.Manifest.RootEntry == nil {
return 0
}
if m.RootEntry.DirSummary != nil {
return m.RootEntry.DirSummary.TotalFileCount
if m.Manifest.RootEntry.DirSummary != nil {
return m.Manifest.RootEntry.DirSummary.TotalFileCount
}
return 1
}
// TotalFilesDelta returns the total number of new files in the snapshot.
func (m *ManifestWithError) TotalFilesDelta() int64 {
if m.Previous == nil {
return 0
}
if m.Manifest.RootEntry == nil || m.Previous.RootEntry == nil {
return 0
}
if m.Manifest.RootEntry.DirSummary != nil && m.Previous.RootEntry.DirSummary != nil {
return m.Manifest.RootEntry.DirSummary.TotalFileCount - m.Previous.RootEntry.DirSummary.TotalFileCount
}
return 1
@@ -51,12 +101,29 @@ func (m *ManifestWithError) TotalFiles() int64 {
// TotalDirs returns the total number of directories in the snapshot.
func (m *ManifestWithError) TotalDirs() int64 {
if m.RootEntry == nil {
if m.Manifest.RootEntry == nil {
return 0
}
if m.RootEntry.DirSummary != nil {
return m.RootEntry.DirSummary.TotalDirCount
if m.Manifest.RootEntry.DirSummary != nil {
return m.Manifest.RootEntry.DirSummary.TotalDirCount
}
return 0
}
// TotalDirsDelta returns the total number of new directories in the snapshot.
func (m *ManifestWithError) TotalDirsDelta() int64 {
if m.Previous == nil {
return 0
}
if m.Manifest.RootEntry == nil || m.Previous.RootEntry == nil {
return 0
}
if m.Manifest.RootEntry.DirSummary != nil && m.Previous.RootEntry.DirSummary != nil {
return m.Manifest.RootEntry.DirSummary.TotalDirCount - m.Previous.RootEntry.DirSummary.TotalDirCount
}
return 0
@@ -64,7 +131,7 @@ func (m *ManifestWithError) TotalDirs() int64 {
// Duration returns the duration of the snapshot.
func (m *ManifestWithError) Duration() time.Duration {
return time.Duration(m.EndTime - m.StartTime)
return time.Duration(m.Manifest.EndTime - m.Manifest.StartTime).Round(durationPrecision)
}
// StatusCode returns the status code of the manifest.

View File

@@ -3,8 +3,10 @@
import (
"embed"
"fmt"
"slices"
"sort"
"strconv"
"text/template"
"time"
@@ -29,6 +31,10 @@ type Options struct {
TimeFormat string
}
func formatCount(v int64) string {
return strconv.FormatInt(v, 10)
}
// functions is a map of functions that can be used in templates.
func functions(opt Options) template.FuncMap {
if opt.Timezone == nil {
@@ -40,11 +46,52 @@ func functions(opt Options) template.FuncMap {
}
return template.FuncMap{
"bytes": units.BytesString[int64],
"bytes": units.BytesString[int64],
"formatCount": formatCount,
"bytesDelta": func(v int64) string {
switch {
case v == 0:
return ""
case v > 0:
return " (+" + units.BytesString(v) + ")"
default:
return " (-" + units.BytesString(-v) + ")"
}
},
"bytesDeltaHTML": func(v int64) string {
switch {
case v == 0:
return ""
case v > 0:
return " <span class='increase'>(&#x2191; " + units.BytesString(v) + ")</span>"
default:
return " <span class='decrease'>(&#x2193; " + units.BytesString(-v) + ")</span>"
}
},
"countDelta": func(v int64) string {
switch {
case v == 0:
return ""
case v > 0:
return fmt.Sprintf(" (+%v)", formatCount(v))
default:
return fmt.Sprintf(" (-%v)", formatCount(-v))
}
},
"countDeltaHTML": func(v int64) string {
switch {
case v == 0:
return ""
case v > 0:
return fmt.Sprintf(" <span class='increase'>(&#x2191; %v)</span>", formatCount(v))
default:
return fmt.Sprintf(" <span class='decrease'>(&#x2193; %v)</span>", formatCount(-v))
}
},
"sortSnapshotManifestsByName": func(man []*notifydata.ManifestWithError) []*notifydata.ManifestWithError {
res := slices.Clone(man)
sort.Slice(res, func(i, j int) bool {
return res[i].Source.String() < res[j].Source.String()
return res[i].Manifest.Source.String() < res[j].Manifest.Source.String()
})
return res
},

View File

@@ -48,10 +48,85 @@ func TestNotifyTemplate_snapshot_report(t *testing.T) {
args := notification.MakeTemplateArgs(&notifydata.MultiSnapshotStatus{
Snapshots: []*notifydata.ManifestWithError{
{
// normal snapshot with positive deltas
Manifest: snapshot.Manifest{
Source: snapshot.SourceInfo{Host: "some-host", UserName: "some-user", Path: "/some/path"},
StartTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC).UnixNano()),
EndTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 6, 7, time.UTC).UnixNano()),
EndTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 6, 120000000, time.UTC).UnixNano()),
RootEntry: &snapshot.DirEntry{
DirSummary: &fs.DirectorySummary{
TotalFileCount: 123,
TotalFileSize: 456,
TotalDirCount: 33,
FailedEntries: []*fs.EntryWithError{
{
EntryPath: "/some/path",
Error: "some error",
},
{
EntryPath: "/some/path2",
Error: "some error",
},
},
},
},
},
Previous: &snapshot.Manifest{
Source: snapshot.SourceInfo{Host: "some-host", UserName: "some-user", Path: "/some/path"},
StartTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC).UnixNano()),
EndTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 6, 120000000, time.UTC).UnixNano()),
RootEntry: &snapshot.DirEntry{
DirSummary: &fs.DirectorySummary{
TotalFileCount: 100,
TotalFileSize: 400,
TotalDirCount: 30,
},
},
},
},
{
// normal snapshot with positive deltas
Manifest: snapshot.Manifest{
Source: snapshot.SourceInfo{Host: "some-host", UserName: "some-user", Path: "/some/path"},
StartTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC).UnixNano()),
EndTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 6, 120000000, time.UTC).UnixNano()),
RootEntry: &snapshot.DirEntry{
DirSummary: &fs.DirectorySummary{
TotalFileCount: 123,
TotalFileSize: 456,
TotalDirCount: 33,
FailedEntries: []*fs.EntryWithError{
{
EntryPath: "/some/path",
Error: "some error",
},
{
EntryPath: "/some/path2",
Error: "some error",
},
},
},
},
},
Previous: &snapshot.Manifest{
Source: snapshot.SourceInfo{Host: "some-host", UserName: "some-user", Path: "/some/path"},
StartTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC).UnixNano()),
EndTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 6, 120000000, time.UTC).UnixNano()),
RootEntry: &snapshot.DirEntry{
DirSummary: &fs.DirectorySummary{
TotalFileCount: 200,
TotalFileSize: 500,
TotalDirCount: 40,
},
},
},
},
{
// no previous snapshot
Manifest: snapshot.Manifest{
Source: snapshot.SourceInfo{Host: "some-host", UserName: "some-user", Path: "/some/path2"},
StartTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC).UnixNano()),
EndTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 6, 120000000, time.UTC).UnixNano()),
RootEntry: &snapshot.DirEntry{
DirSummary: &fs.DirectorySummary{
TotalFileCount: 123,

View File

@@ -34,6 +34,16 @@ Subject: Kopia created {{ len .EventArgs.Snapshots }} snapshot{{ if gt (len .Eve
font-weight: bold;
}
span.increase {
color: green;
font-style: italic;
}
span.decrease {
color: red;
font-style: italic;
}
tr.snapshotstatus-fatal {
background-color: #fde9e4;
}
@@ -61,12 +71,12 @@ Subject: Kopia created {{ len .EventArgs.Snapshots }} snapshot{{ if gt (len .Eve
</thead>
{{ range .EventArgs.Snapshots | sortSnapshotManifestsByName}}
<tr class="snapshotstatus-{{ .StatusCode }}">
<td><span class="path">{{ .Source.Path }}</span></td>
<td><span class="path">{{ .Manifest.Source.Path }}</span></td>
<td>{{ .StartTimestamp | formatTime }}</td>
<td>{{ .Duration }}</td>
<td>{{ .TotalSize | bytes }}</td>
<td>{{ .TotalFiles }}</td>
<td>{{ .TotalDirs }}</td>
<td>{{ .TotalSize | bytes }}{{ .TotalSizeDelta | bytesDeltaHTML }}</td>
<td>{{ .TotalFiles | formatCount }}{{ .TotalFilesDelta | countDeltaHTML }}</td>
<td>{{ .TotalDirs | formatCount }}{{ .TotalDirsDelta | countDeltaHTML }}</td>
</tr>
{{ if .Error }}
@@ -77,14 +87,14 @@ Subject: Kopia created {{ len .EventArgs.Snapshots }} snapshot{{ if gt (len .Eve
</tr>
{{ end }}
{{ if .RootEntry }}
{{ if .RootEntry.DirSummary }}
{{ if .RootEntry.DirSummary.FailedEntries }}
{{ if .Manifest.RootEntry }}
{{ if .Manifest.RootEntry.DirSummary }}
{{ if .Manifest.RootEntry.DirSummary.FailedEntries }}
<tr class="snapshotstatus-{{ .StatusCode }}">
<td colspan="6">
<b style="color:red">Failed Entries:</b>
<ul>
{{ range .RootEntry.DirSummary.FailedEntries }}
{{ range .Manifest.RootEntry.DirSummary.FailedEntries }}
<li><span class="path">{{.EntryPath}}</span>: {{.Error}}</li>
{{ end }}
</ul>

View File

@@ -1,14 +1,17 @@
Subject: Kopia created {{ len .EventArgs.Snapshots }} snapshot{{ if gt (len .EventArgs.Snapshots) 1 }}s{{end}} on {{.Hostname}}
{{ range .EventArgs.Snapshots | sortSnapshotManifestsByName}}Path: {{ .Source.Path }}
{{ range .EventArgs.Snapshots | sortSnapshotManifestsByName}}Path: {{ .Manifest.Source.Path }}
Status: {{ .StatusCode }}
Start: {{ .StartTimestamp | formatTime }} Duration: {{ .Duration }}
Size: {{ .TotalSize | bytes }}, {{ .TotalFiles }} files, {{ .TotalDirs }} directories.
{{ if .Error }} Error: {{ .Error }}
{{ end }}{{ if .RootEntry }}{{ if .RootEntry.DirSummary }}{{ if .RootEntry.DirSummary.FailedEntries }}
Status: {{ .StatusCode }}
Start: {{ .StartTimestamp | formatTime }}
Duration: {{ .Duration }}
Size: {{ .TotalSize | bytes }}{{ .TotalSizeDelta | bytesDelta }}
Files: {{ .TotalFiles | formatCount }}{{ .TotalFilesDelta | countDelta }}
Directories: {{ .TotalDirs | formatCount }}{{ .TotalDirsDelta | countDelta }}
{{ if .Error }} Error: {{ .Error }}
{{ end }}{{ if .Manifest.RootEntry }}{{ if .Manifest.RootEntry.DirSummary }}{{ if .Manifest.RootEntry.DirSummary.FailedEntries }}
Failed Entries:
{{ range .RootEntry.DirSummary.FailedEntries }}
{{ range .Manifest.RootEntry.DirSummary.FailedEntries }}
- {{.EntryPath}}: {{.Error}}{{ end }}{{ end }}{{ end }}
{{ end }}
{{ end }}Generated at {{ .EventTime | formatTime }} by Kopia {{ .KopiaBuildVersion }}.

View File

@@ -1,4 +1,4 @@
Subject: Kopia created 2 snapshots on some-host
Subject: Kopia created 4 snapshots on some-host
<!doctype html>
<html>
@@ -34,6 +34,16 @@ Subject: Kopia created 2 snapshots on some-host
font-weight: bold;
}
span.increase {
color: green;
font-style: italic;
}
span.decrease {
color: red;
font-style: italic;
}
tr.snapshotstatus-fatal {
background-color: #fde9e4;
}
@@ -83,7 +93,69 @@ Subject: Kopia created 2 snapshots on some-host
<tr class="snapshotstatus-ok">
<td><span class="path">/some/path</span></td>
<td>Wed, 01 Jan 2020 19:04:05 PST</td>
<td>1.000000001s</td>
<td>1.1s</td>
<td>456 B <span class='increase'>(&#x2191; 56 B)</span></td>
<td>123 <span class='increase'>(&#x2191; 23)</span></td>
<td>33 <span class='increase'>(&#x2191; 3)</span></td>
</tr>
<tr class="snapshotstatus-ok">
<td colspan="6">
<b style="color:red">Failed Entries:</b>
<ul>
<li><span class="path">/some/path</span>: some error</li>
<li><span class="path">/some/path2</span>: some error</li>
</ul>
</td>
</tr>
<tr class="snapshotstatus-ok">
<td><span class="path">/some/path</span></td>
<td>Wed, 01 Jan 2020 19:04:05 PST</td>
<td>1.1s</td>
<td>456 B <span class='decrease'>(&#x2193; 44 B)</span></td>
<td>123 <span class='decrease'>(&#x2193; 77)</span></td>
<td>33 <span class='decrease'>(&#x2193; 7)</span></td>
</tr>
<tr class="snapshotstatus-ok">
<td colspan="6">
<b style="color:red">Failed Entries:</b>
<ul>
<li><span class="path">/some/path</span>: some error</li>
<li><span class="path">/some/path2</span>: some error</li>
</ul>
</td>
</tr>
<tr class="snapshotstatus-ok">
<td><span class="path">/some/path2</span></td>
<td>Wed, 01 Jan 2020 19:04:05 PST</td>
<td>1.1s</td>
<td>456 B</td>
<td>123</td>
<td>33</td>

View File

@@ -1,4 +1,4 @@
Subject: Kopia created 2 snapshots on some-host
Subject: Kopia created 4 snapshots on some-host
<!doctype html>
<html>
@@ -34,6 +34,16 @@ Subject: Kopia created 2 snapshots on some-host
font-weight: bold;
}
span.increase {
color: green;
font-style: italic;
}
span.decrease {
color: red;
font-style: italic;
}
tr.snapshotstatus-fatal {
background-color: #fde9e4;
}
@@ -83,7 +93,69 @@ Subject: Kopia created 2 snapshots on some-host
<tr class="snapshotstatus-ok">
<td><span class="path">/some/path</span></td>
<td>Thu, 02 Jan 2020 03:04:05 +0000</td>
<td>1.000000001s</td>
<td>1.1s</td>
<td>456 B <span class='increase'>(&#x2191; 56 B)</span></td>
<td>123 <span class='increase'>(&#x2191; 23)</span></td>
<td>33 <span class='increase'>(&#x2191; 3)</span></td>
</tr>
<tr class="snapshotstatus-ok">
<td colspan="6">
<b style="color:red">Failed Entries:</b>
<ul>
<li><span class="path">/some/path</span>: some error</li>
<li><span class="path">/some/path2</span>: some error</li>
</ul>
</td>
</tr>
<tr class="snapshotstatus-ok">
<td><span class="path">/some/path</span></td>
<td>Thu, 02 Jan 2020 03:04:05 +0000</td>
<td>1.1s</td>
<td>456 B <span class='decrease'>(&#x2193; 44 B)</span></td>
<td>123 <span class='decrease'>(&#x2193; 77)</span></td>
<td>33 <span class='decrease'>(&#x2193; 7)</span></td>
</tr>
<tr class="snapshotstatus-ok">
<td colspan="6">
<b style="color:red">Failed Entries:</b>
<ul>
<li><span class="path">/some/path</span>: some error</li>
<li><span class="path">/some/path2</span>: some error</li>
</ul>
</td>
</tr>
<tr class="snapshotstatus-ok">
<td><span class="path">/some/path2</span></td>
<td>Thu, 02 Jan 2020 03:04:05 +0000</td>
<td>1.1s</td>
<td>456 B</td>
<td>123</td>
<td>33</td>

View File

@@ -1,17 +1,51 @@
Subject: Kopia created 2 snapshots on some-host
Subject: Kopia created 4 snapshots on some-host
Path: /some/other/path
Status: fatal
Start: Wed, 31 Dec 1969 16:00:00 PST Duration: 0s
Size: 0 B, 0 files, 0 directories.
Error: some top-level error
Status: fatal
Start: Wed, 31 Dec 1969 16:00:00 PST
Duration: 0s
Size: 0 B
Files: 0
Directories: 0
Error: some top-level error
Path: /some/path
Status: ok
Start: Wed, 01 Jan 2020 19:04:05 PST Duration: 1.000000001s
Size: 456 B, 123 files, 33 directories.
Status: ok
Start: Wed, 01 Jan 2020 19:04:05 PST
Duration: 1.1s
Size: 456 B (+56 B)
Files: 123 (+23)
Directories: 33 (+3)
Failed Entries:
- /some/path: some error
- /some/path2: some error
Path: /some/path
Status: ok
Start: Wed, 01 Jan 2020 19:04:05 PST
Duration: 1.1s
Size: 456 B (-44 B)
Files: 123 (-77)
Directories: 33 (-7)
Failed Entries:
- /some/path: some error
- /some/path2: some error
Path: /some/path2
Status: ok
Start: Wed, 01 Jan 2020 19:04:05 PST
Duration: 1.1s
Size: 456 B
Files: 123
Directories: 33
Failed Entries:

View File

@@ -1,17 +1,51 @@
Subject: Kopia created 2 snapshots on some-host
Subject: Kopia created 4 snapshots on some-host
Path: /some/other/path
Status: fatal
Start: Thu, 01 Jan 1970 00:00:00 +0000 Duration: 0s
Size: 0 B, 0 files, 0 directories.
Error: some top-level error
Status: fatal
Start: Thu, 01 Jan 1970 00:00:00 +0000
Duration: 0s
Size: 0 B
Files: 0
Directories: 0
Error: some top-level error
Path: /some/path
Status: ok
Start: Thu, 02 Jan 2020 03:04:05 +0000 Duration: 1.000000001s
Size: 456 B, 123 files, 33 directories.
Status: ok
Start: Thu, 02 Jan 2020 03:04:05 +0000
Duration: 1.1s
Size: 456 B (+56 B)
Files: 123 (+23)
Directories: 33 (+3)
Failed Entries:
- /some/path: some error
- /some/path2: some error
Path: /some/path
Status: ok
Start: Thu, 02 Jan 2020 03:04:05 +0000
Duration: 1.1s
Size: 456 B (-44 B)
Files: 123 (-77)
Directories: 33 (-7)
Failed Entries:
- /some/path: some error
- /some/path2: some error
Path: /some/path2
Status: ok
Start: Thu, 02 Jan 2020 03:04:05 +0000
Duration: 1.1s
Size: 456 B
Files: 123
Directories: 33
Failed Entries: