diff --git a/cli/command_snapshot_create.go b/cli/command_snapshot_create.go index 7f34127ef..7d671c7cd 100644 --- a/cli/command_snapshot_create.go +++ b/cli/command_snapshot_create.go @@ -280,7 +280,7 @@ func (c *commandSnapshotCreate) snapshotSingleSource( var mwe notifydata.ManifestWithError - mwe.Source = sourceInfo + mwe.Manifest.Source = sourceInfo st.Snapshots = append(st.Snapshots, &mwe) @@ -297,6 +297,10 @@ func (c *commandSnapshotCreate) snapshotSingleSource( return finalErr } + if len(previous) > 0 { + mwe.Previous = previous[0] + } + policyTree, finalErr := policy.TreeForSource(ctx, rep, sourceInfo) if finalErr != nil { return errors.Wrap(finalErr, "unable to get policy tree") diff --git a/notification/notifydata/multi_snapshot_status.go b/notification/notifydata/multi_snapshot_status.go index 4543ea89b..fba5ad1c9 100644 --- a/notification/notifydata/multi_snapshot_status.go +++ b/notification/notifydata/multi_snapshot_status.go @@ -6,44 +6,94 @@ "github.com/kopia/kopia/snapshot" ) +const durationPrecision = 100 * time.Millisecond + // ManifestWithError represents information about the snapshot manifest with optional error. type ManifestWithError struct { - snapshot.Manifest `json:"manifest"` // may not be filled out if there was an error, Manifst.Source is always set. + Manifest snapshot.Manifest `json:"manifest"` // may not be filled out if there was an error, Manifst.Source is always set. + Previous *snapshot.Manifest `json:"previous"` // may not be filled out Error string `json:"error"` // will be present if there was an error } // StartTimestamp returns the start time of the snapshot. func (m *ManifestWithError) StartTimestamp() time.Time { - return m.StartTime.ToTime().UTC().Truncate(time.Second) + return m.Manifest.StartTime.ToTime().UTC().Truncate(time.Second) } // EndTimestamp returns the end time of the snapshot. func (m *ManifestWithError) EndTimestamp() time.Time { - return m.EndTime.ToTime().UTC().Truncate(time.Second) + return m.Manifest.EndTime.ToTime().UTC().Truncate(time.Second) } // TotalSize returns the total size of the snapshot in bytes. func (m *ManifestWithError) TotalSize() int64 { - if m.RootEntry == nil { + if m.Manifest.RootEntry == nil { return 0 } - if m.RootEntry.DirSummary != nil { - return m.RootEntry.DirSummary.TotalFileSize + if m.Manifest.RootEntry.DirSummary != nil { + return m.Manifest.RootEntry.DirSummary.TotalFileSize } - return m.RootEntry.FileSize + return m.Manifest.RootEntry.FileSize +} + +// TotalSizeDelta returns the total size of the snapshot in bytes. +func (m *ManifestWithError) TotalSizeDelta() int64 { + if m.Previous == nil { + return 0 + } + + if m.Manifest.RootEntry == nil { + return 0 + } + + if m.Manifest.RootEntry.DirSummary != nil && m.Previous.RootEntry.DirSummary != nil { + return m.Manifest.RootEntry.DirSummary.TotalFileSize - m.Previous.RootEntry.DirSummary.TotalFileSize + } + + return m.Manifest.RootEntry.FileSize +} + +// New returns the total size of the snapshot in bytes. +func (m *ManifestWithError) New() int64 { + if m.Manifest.RootEntry == nil { + return 0 + } + + if m.Manifest.RootEntry.DirSummary != nil { + return m.Manifest.RootEntry.DirSummary.TotalFileSize + } + + return m.Manifest.RootEntry.FileSize } // TotalFiles returns the total number of files in the snapshot. func (m *ManifestWithError) TotalFiles() int64 { - if m.RootEntry == nil { + if m.Manifest.RootEntry == nil { return 0 } - if m.RootEntry.DirSummary != nil { - return m.RootEntry.DirSummary.TotalFileCount + if m.Manifest.RootEntry.DirSummary != nil { + return m.Manifest.RootEntry.DirSummary.TotalFileCount + } + + return 1 +} + +// TotalFilesDelta returns the total number of new files in the snapshot. +func (m *ManifestWithError) TotalFilesDelta() int64 { + if m.Previous == nil { + return 0 + } + + if m.Manifest.RootEntry == nil || m.Previous.RootEntry == nil { + return 0 + } + + if m.Manifest.RootEntry.DirSummary != nil && m.Previous.RootEntry.DirSummary != nil { + return m.Manifest.RootEntry.DirSummary.TotalFileCount - m.Previous.RootEntry.DirSummary.TotalFileCount } return 1 @@ -51,12 +101,29 @@ func (m *ManifestWithError) TotalFiles() int64 { // TotalDirs returns the total number of directories in the snapshot. func (m *ManifestWithError) TotalDirs() int64 { - if m.RootEntry == nil { + if m.Manifest.RootEntry == nil { return 0 } - if m.RootEntry.DirSummary != nil { - return m.RootEntry.DirSummary.TotalDirCount + if m.Manifest.RootEntry.DirSummary != nil { + return m.Manifest.RootEntry.DirSummary.TotalDirCount + } + + return 0 +} + +// TotalDirsDelta returns the total number of new directories in the snapshot. +func (m *ManifestWithError) TotalDirsDelta() int64 { + if m.Previous == nil { + return 0 + } + + if m.Manifest.RootEntry == nil || m.Previous.RootEntry == nil { + return 0 + } + + if m.Manifest.RootEntry.DirSummary != nil && m.Previous.RootEntry.DirSummary != nil { + return m.Manifest.RootEntry.DirSummary.TotalDirCount - m.Previous.RootEntry.DirSummary.TotalDirCount } return 0 @@ -64,7 +131,7 @@ func (m *ManifestWithError) TotalDirs() int64 { // Duration returns the duration of the snapshot. func (m *ManifestWithError) Duration() time.Duration { - return time.Duration(m.EndTime - m.StartTime) + return time.Duration(m.Manifest.EndTime - m.Manifest.StartTime).Round(durationPrecision) } // StatusCode returns the status code of the manifest. diff --git a/notification/notifytemplate/embeddedtemplate.go b/notification/notifytemplate/embeddedtemplate.go index 0c24cb128..a975a0bd6 100644 --- a/notification/notifytemplate/embeddedtemplate.go +++ b/notification/notifytemplate/embeddedtemplate.go @@ -3,8 +3,10 @@ import ( "embed" + "fmt" "slices" "sort" + "strconv" "text/template" "time" @@ -29,6 +31,10 @@ type Options struct { TimeFormat string } +func formatCount(v int64) string { + return strconv.FormatInt(v, 10) +} + // functions is a map of functions that can be used in templates. func functions(opt Options) template.FuncMap { if opt.Timezone == nil { @@ -40,11 +46,52 @@ func functions(opt Options) template.FuncMap { } return template.FuncMap{ - "bytes": units.BytesString[int64], + "bytes": units.BytesString[int64], + "formatCount": formatCount, + "bytesDelta": func(v int64) string { + switch { + case v == 0: + return "" + case v > 0: + return " (+" + units.BytesString(v) + ")" + default: + return " (-" + units.BytesString(-v) + ")" + } + }, + "bytesDeltaHTML": func(v int64) string { + switch { + case v == 0: + return "" + case v > 0: + return " (↑ " + units.BytesString(v) + ")" + default: + return " (↓ " + units.BytesString(-v) + ")" + } + }, + "countDelta": func(v int64) string { + switch { + case v == 0: + return "" + case v > 0: + return fmt.Sprintf(" (+%v)", formatCount(v)) + default: + return fmt.Sprintf(" (-%v)", formatCount(-v)) + } + }, + "countDeltaHTML": func(v int64) string { + switch { + case v == 0: + return "" + case v > 0: + return fmt.Sprintf(" (↑ %v)", formatCount(v)) + default: + return fmt.Sprintf(" (↓ %v)", formatCount(-v)) + } + }, "sortSnapshotManifestsByName": func(man []*notifydata.ManifestWithError) []*notifydata.ManifestWithError { res := slices.Clone(man) sort.Slice(res, func(i, j int) bool { - return res[i].Source.String() < res[j].Source.String() + return res[i].Manifest.Source.String() < res[j].Manifest.Source.String() }) return res }, diff --git a/notification/notifytemplate/notifytemplate_test.go b/notification/notifytemplate/notifytemplate_test.go index 182c98fbc..8a05d3c20 100644 --- a/notification/notifytemplate/notifytemplate_test.go +++ b/notification/notifytemplate/notifytemplate_test.go @@ -48,10 +48,85 @@ func TestNotifyTemplate_snapshot_report(t *testing.T) { args := notification.MakeTemplateArgs(¬ifydata.MultiSnapshotStatus{ Snapshots: []*notifydata.ManifestWithError{ { + // normal snapshot with positive deltas Manifest: snapshot.Manifest{ Source: snapshot.SourceInfo{Host: "some-host", UserName: "some-user", Path: "/some/path"}, StartTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC).UnixNano()), - EndTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 6, 7, time.UTC).UnixNano()), + EndTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 6, 120000000, time.UTC).UnixNano()), + RootEntry: &snapshot.DirEntry{ + DirSummary: &fs.DirectorySummary{ + TotalFileCount: 123, + TotalFileSize: 456, + TotalDirCount: 33, + FailedEntries: []*fs.EntryWithError{ + { + EntryPath: "/some/path", + Error: "some error", + }, + { + EntryPath: "/some/path2", + Error: "some error", + }, + }, + }, + }, + }, + Previous: &snapshot.Manifest{ + Source: snapshot.SourceInfo{Host: "some-host", UserName: "some-user", Path: "/some/path"}, + StartTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC).UnixNano()), + EndTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 6, 120000000, time.UTC).UnixNano()), + RootEntry: &snapshot.DirEntry{ + DirSummary: &fs.DirectorySummary{ + TotalFileCount: 100, + TotalFileSize: 400, + TotalDirCount: 30, + }, + }, + }, + }, + { + // normal snapshot with positive deltas + Manifest: snapshot.Manifest{ + Source: snapshot.SourceInfo{Host: "some-host", UserName: "some-user", Path: "/some/path"}, + StartTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC).UnixNano()), + EndTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 6, 120000000, time.UTC).UnixNano()), + RootEntry: &snapshot.DirEntry{ + DirSummary: &fs.DirectorySummary{ + TotalFileCount: 123, + TotalFileSize: 456, + TotalDirCount: 33, + FailedEntries: []*fs.EntryWithError{ + { + EntryPath: "/some/path", + Error: "some error", + }, + { + EntryPath: "/some/path2", + Error: "some error", + }, + }, + }, + }, + }, + Previous: &snapshot.Manifest{ + Source: snapshot.SourceInfo{Host: "some-host", UserName: "some-user", Path: "/some/path"}, + StartTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC).UnixNano()), + EndTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 6, 120000000, time.UTC).UnixNano()), + RootEntry: &snapshot.DirEntry{ + DirSummary: &fs.DirectorySummary{ + TotalFileCount: 200, + TotalFileSize: 500, + TotalDirCount: 40, + }, + }, + }, + }, + { + // no previous snapshot + Manifest: snapshot.Manifest{ + Source: snapshot.SourceInfo{Host: "some-host", UserName: "some-user", Path: "/some/path2"}, + StartTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC).UnixNano()), + EndTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 6, 120000000, time.UTC).UnixNano()), RootEntry: &snapshot.DirEntry{ DirSummary: &fs.DirectorySummary{ TotalFileCount: 123, diff --git a/notification/notifytemplate/snapshot-report.html b/notification/notifytemplate/snapshot-report.html index 272d43301..4a279d118 100644 --- a/notification/notifytemplate/snapshot-report.html +++ b/notification/notifytemplate/snapshot-report.html @@ -34,6 +34,16 @@ Subject: Kopia created {{ len .EventArgs.Snapshots }} snapshot{{ if gt (len .Eve font-weight: bold; } + span.increase { + color: green; + font-style: italic; + } + + span.decrease { + color: red; + font-style: italic; + } + tr.snapshotstatus-fatal { background-color: #fde9e4; } @@ -61,12 +71,12 @@ Subject: Kopia created {{ len .EventArgs.Snapshots }} snapshot{{ if gt (len .Eve {{ range .EventArgs.Snapshots | sortSnapshotManifestsByName}} -{{ .Source.Path }} +{{ .Manifest.Source.Path }} {{ .StartTimestamp | formatTime }} {{ .Duration }} -{{ .TotalSize | bytes }} -{{ .TotalFiles }} -{{ .TotalDirs }} +{{ .TotalSize | bytes }}{{ .TotalSizeDelta | bytesDeltaHTML }} +{{ .TotalFiles | formatCount }}{{ .TotalFilesDelta | countDeltaHTML }} +{{ .TotalDirs | formatCount }}{{ .TotalDirsDelta | countDeltaHTML }} {{ if .Error }} @@ -77,14 +87,14 @@ Subject: Kopia created {{ len .EventArgs.Snapshots }} snapshot{{ if gt (len .Eve {{ end }} -{{ if .RootEntry }} -{{ if .RootEntry.DirSummary }} -{{ if .RootEntry.DirSummary.FailedEntries }} +{{ if .Manifest.RootEntry }} +{{ if .Manifest.RootEntry.DirSummary }} +{{ if .Manifest.RootEntry.DirSummary.FailedEntries }} Failed Entries: diff --git a/notification/notifytemplate/snapshot-report.txt b/notification/notifytemplate/snapshot-report.txt index 212c85d0f..2c689c645 100644 --- a/notification/notifytemplate/snapshot-report.txt +++ b/notification/notifytemplate/snapshot-report.txt @@ -1,14 +1,17 @@ Subject: Kopia created {{ len .EventArgs.Snapshots }} snapshot{{ if gt (len .EventArgs.Snapshots) 1 }}s{{end}} on {{.Hostname}} -{{ range .EventArgs.Snapshots | sortSnapshotManifestsByName}}Path: {{ .Source.Path }} +{{ range .EventArgs.Snapshots | sortSnapshotManifestsByName}}Path: {{ .Manifest.Source.Path }} - Status: {{ .StatusCode }} - Start: {{ .StartTimestamp | formatTime }} Duration: {{ .Duration }} - Size: {{ .TotalSize | bytes }}, {{ .TotalFiles }} files, {{ .TotalDirs }} directories. -{{ if .Error }} Error: {{ .Error }} -{{ end }}{{ if .RootEntry }}{{ if .RootEntry.DirSummary }}{{ if .RootEntry.DirSummary.FailedEntries }} + Status: {{ .StatusCode }} + Start: {{ .StartTimestamp | formatTime }} + Duration: {{ .Duration }} + Size: {{ .TotalSize | bytes }}{{ .TotalSizeDelta | bytesDelta }} + Files: {{ .TotalFiles | formatCount }}{{ .TotalFilesDelta | countDelta }} + Directories: {{ .TotalDirs | formatCount }}{{ .TotalDirsDelta | countDelta }} +{{ if .Error }} Error: {{ .Error }} +{{ end }}{{ if .Manifest.RootEntry }}{{ if .Manifest.RootEntry.DirSummary }}{{ if .Manifest.RootEntry.DirSummary.FailedEntries }} Failed Entries: -{{ range .RootEntry.DirSummary.FailedEntries }} +{{ range .Manifest.RootEntry.DirSummary.FailedEntries }} - {{.EntryPath}}: {{.Error}}{{ end }}{{ end }}{{ end }} {{ end }} {{ end }}Generated at {{ .EventTime | formatTime }} by Kopia {{ .KopiaBuildVersion }}. diff --git a/notification/notifytemplate/testdata/snapshot-report.html.alt.expected b/notification/notifytemplate/testdata/snapshot-report.html.alt.expected index 11647e19f..459279826 100644 --- a/notification/notifytemplate/testdata/snapshot-report.html.alt.expected +++ b/notification/notifytemplate/testdata/snapshot-report.html.alt.expected @@ -1,4 +1,4 @@ -Subject: Kopia created 2 snapshots on some-host +Subject: Kopia created 4 snapshots on some-host @@ -34,6 +34,16 @@ Subject: Kopia created 2 snapshots on some-host font-weight: bold; } + span.increase { + color: green; + font-style: italic; + } + + span.decrease { + color: red; + font-style: italic; + } + tr.snapshotstatus-fatal { background-color: #fde9e4; } @@ -83,7 +93,69 @@ Subject: Kopia created 2 snapshots on some-host /some/path Wed, 01 Jan 2020 19:04:05 PST -1.000000001s +1.1s +456 B (↑ 56 B) +123 (↑ 23) +33 (↑ 3) + + + + + + + + + + Failed Entries: + + + + + + + + + +/some/path +Wed, 01 Jan 2020 19:04:05 PST +1.1s +456 B (↓ 44 B) +123 (↓ 77) +33 (↓ 7) + + + + + + + + + + Failed Entries: + + + + + + + + + +/some/path2 +Wed, 01 Jan 2020 19:04:05 PST +1.1s 456 B 123 33 diff --git a/notification/notifytemplate/testdata/snapshot-report.html.default.expected b/notification/notifytemplate/testdata/snapshot-report.html.default.expected index 5cf75fb2d..a34aa84bf 100644 --- a/notification/notifytemplate/testdata/snapshot-report.html.default.expected +++ b/notification/notifytemplate/testdata/snapshot-report.html.default.expected @@ -1,4 +1,4 @@ -Subject: Kopia created 2 snapshots on some-host +Subject: Kopia created 4 snapshots on some-host @@ -34,6 +34,16 @@ Subject: Kopia created 2 snapshots on some-host font-weight: bold; } + span.increase { + color: green; + font-style: italic; + } + + span.decrease { + color: red; + font-style: italic; + } + tr.snapshotstatus-fatal { background-color: #fde9e4; } @@ -83,7 +93,69 @@ Subject: Kopia created 2 snapshots on some-host /some/path Thu, 02 Jan 2020 03:04:05 +0000 -1.000000001s +1.1s +456 B (↑ 56 B) +123 (↑ 23) +33 (↑ 3) + + + + + + + + + + Failed Entries: + + + + + + + + + +/some/path +Thu, 02 Jan 2020 03:04:05 +0000 +1.1s +456 B (↓ 44 B) +123 (↓ 77) +33 (↓ 7) + + + + + + + + + + Failed Entries: + + + + + + + + + +/some/path2 +Thu, 02 Jan 2020 03:04:05 +0000 +1.1s 456 B 123 33 diff --git a/notification/notifytemplate/testdata/snapshot-report.txt.alt.expected b/notification/notifytemplate/testdata/snapshot-report.txt.alt.expected index 46de1240c..7a28db844 100644 --- a/notification/notifytemplate/testdata/snapshot-report.txt.alt.expected +++ b/notification/notifytemplate/testdata/snapshot-report.txt.alt.expected @@ -1,17 +1,51 @@ -Subject: Kopia created 2 snapshots on some-host +Subject: Kopia created 4 snapshots on some-host Path: /some/other/path - Status: fatal - Start: Wed, 31 Dec 1969 16:00:00 PST Duration: 0s - Size: 0 B, 0 files, 0 directories. - Error: some top-level error + Status: fatal + Start: Wed, 31 Dec 1969 16:00:00 PST + Duration: 0s + Size: 0 B + Files: 0 + Directories: 0 + Error: some top-level error Path: /some/path - Status: ok - Start: Wed, 01 Jan 2020 19:04:05 PST Duration: 1.000000001s - Size: 456 B, 123 files, 33 directories. + Status: ok + Start: Wed, 01 Jan 2020 19:04:05 PST + Duration: 1.1s + Size: 456 B (+56 B) + Files: 123 (+23) + Directories: 33 (+3) + + Failed Entries: + + - /some/path: some error + - /some/path2: some error + +Path: /some/path + + Status: ok + Start: Wed, 01 Jan 2020 19:04:05 PST + Duration: 1.1s + Size: 456 B (-44 B) + Files: 123 (-77) + Directories: 33 (-7) + + Failed Entries: + + - /some/path: some error + - /some/path2: some error + +Path: /some/path2 + + Status: ok + Start: Wed, 01 Jan 2020 19:04:05 PST + Duration: 1.1s + Size: 456 B + Files: 123 + Directories: 33 Failed Entries: diff --git a/notification/notifytemplate/testdata/snapshot-report.txt.default.expected b/notification/notifytemplate/testdata/snapshot-report.txt.default.expected index b11a4e06d..0b7331580 100644 --- a/notification/notifytemplate/testdata/snapshot-report.txt.default.expected +++ b/notification/notifytemplate/testdata/snapshot-report.txt.default.expected @@ -1,17 +1,51 @@ -Subject: Kopia created 2 snapshots on some-host +Subject: Kopia created 4 snapshots on some-host Path: /some/other/path - Status: fatal - Start: Thu, 01 Jan 1970 00:00:00 +0000 Duration: 0s - Size: 0 B, 0 files, 0 directories. - Error: some top-level error + Status: fatal + Start: Thu, 01 Jan 1970 00:00:00 +0000 + Duration: 0s + Size: 0 B + Files: 0 + Directories: 0 + Error: some top-level error Path: /some/path - Status: ok - Start: Thu, 02 Jan 2020 03:04:05 +0000 Duration: 1.000000001s - Size: 456 B, 123 files, 33 directories. + Status: ok + Start: Thu, 02 Jan 2020 03:04:05 +0000 + Duration: 1.1s + Size: 456 B (+56 B) + Files: 123 (+23) + Directories: 33 (+3) + + Failed Entries: + + - /some/path: some error + - /some/path2: some error + +Path: /some/path + + Status: ok + Start: Thu, 02 Jan 2020 03:04:05 +0000 + Duration: 1.1s + Size: 456 B (-44 B) + Files: 123 (-77) + Directories: 33 (-7) + + Failed Entries: + + - /some/path: some error + - /some/path2: some error + +Path: /some/path2 + + Status: ok + Start: Thu, 02 Jan 2020 03:04:05 +0000 + Duration: 1.1s + Size: 456 B + Files: 123 + Directories: 33 Failed Entries: