feat(cli): send error notifications and snapshot reports (#4233)

* feat(cli): send error notifications and snapshot reports

Notifications will be sent to all configured notification profiles
according to their severity levels.

The following events will trigger notifications:

- Snapshot is created (CLI only, severity >= report)
- Server Maintenance error occurs (CLI, server and UI, severity >= error)
- Any other CLI error occurs (CLI only, severity >= error).

A flag `--no-error-notifications` can be used to disable error notifications.

* added template tests

* improved time formatting in templates

* plumb through notifytemplate.Options

* more testing for formatting options

* fixed default date format to RFC1123

cli/app.go

@@ -16,9 +16,13 @@
 	"go.opentelemetry.io/otel/trace"
 
 	"github.com/kopia/kopia/internal/apiclient"
+	"github.com/kopia/kopia/internal/clock"
 	"github.com/kopia/kopia/internal/gather"
 	"github.com/kopia/kopia/internal/passwordpersist"
 	"github.com/kopia/kopia/internal/releasable"
+	"github.com/kopia/kopia/notification"
+	"github.com/kopia/kopia/notification/notifydata"
+	"github.com/kopia/kopia/notification/notifytemplate"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/blob"
 	"github.com/kopia/kopia/repo/logging"
@@ -88,6 +92,7 @@ type appServices interface {
 	repositoryConfigFileName() string
 	getProgress() *cliProgress
 	getRestoreProgress() RestoreProgress
+	notificationTemplateOptions() notifytemplate.Options
 
 	stdout() io.Writer
 	Stderr() io.Writer
@@ -112,6 +117,7 @@ type advancedAppServices interface {
 	getPasswordFromFlags(ctx context.Context, isCreate, allowPersistent bool) (string, error)
 	optionsFromFlags(ctx context.Context) *repo.Options
 	runAppWithContext(command *kingpin.CmdClause, callback func(ctx context.Context) error) error
+	enableErrorNotifications() bool
 }
 
 // App contains per-invocation flags and state of Kopia CLI.
@@ -139,6 +145,8 @@ type App struct {
 	upgradeOwnerID      string
 	doNotWaitForUpgrade bool
 
+	errorNotifications bool
+
 	currentAction         string
 	onExitCallbacks       []func()
 	onFatalErrorCallbacks []func(err error)
@@ -240,6 +248,10 @@ func (c *App) passwordPersistenceStrategy() passwordpersist.Strategy {
 	return passwordpersist.File()
 }
 
+func (c *App) enableErrorNotifications() bool {
+	return c.errorNotifications
+}
+
 func (c *App) setup(app *kingpin.Application) {
 	app.PreAction(func(pc *kingpin.ParseContext) error {
 		if sc := pc.SelectedCommand; sc != nil {
@@ -274,6 +286,7 @@ func (c *App) setup(app *kingpin.Application) {
 	app.Flag("dump-allocator-stats", "Dump allocator stats at the end of execution.").Hidden().Envar(c.EnvName("KOPIA_DUMP_ALLOCATOR_STATS")).BoolVar(&c.dumpAllocatorStats)
 	app.Flag("upgrade-owner-id", "Repository format upgrade owner-id.").Hidden().Envar(c.EnvName("KOPIA_REPO_UPGRADE_OWNER_ID")).StringVar(&c.upgradeOwnerID)
 	app.Flag("upgrade-no-block", "Do not block when repository format upgrade is in progress, instead exit with a message.").Hidden().Default("false").Envar(c.EnvName("KOPIA_REPO_UPGRADE_NO_BLOCK")).BoolVar(&c.doNotWaitForUpgrade)
+	app.Flag("error-notifications", "Send notification on errors").Hidden().Envar(c.EnvName("KOPIA_SEND_ERROR_NOTIFICATIONS")).Default("true").BoolVar(&c.errorNotifications)
 
 	if c.enableTestOnlyFlags() {
 		app.Flag("ignore-missing-required-features", "Open repository despite missing features (VERY DANGEROUS, ONLY FOR TESTING)").Hidden().BoolVar(&c.testonlyIgnoreMissingRequiredFeatures)
@@ -562,6 +575,8 @@ func (c *App) maybeRepositoryAction(act func(ctx context.Context, rep repo.Repos
 			return errors.Wrap(err, "open repository")
 		}
 
+		t0 := clock.Now()
+
 		err = act(ctx, rep)
 
 		if rep != nil && err == nil && !mode.disableMaintenance {
@@ -570,6 +585,17 @@ func (c *App) maybeRepositoryAction(act func(ctx context.Context, rep repo.Repos
 			}
 		}
 
+		if err != nil && rep != nil && c.errorNotifications {
+			notification.Send(ctx, rep, "generic-error", notifydata.NewErrorInfo(
+				c.currentActionName(),
+				c.currentActionName(),
+				t0,
+				clock.Now(),
+				err), notification.SeverityError,
+				c.notificationTemplateOptions(),
+			)
+		}
+
 		if rep != nil && mode.mustBeConnected {
 			if cerr := rep.Close(ctx); cerr != nil {
 				return errors.Wrap(cerr, "unable to close repository")
@@ -647,6 +673,11 @@ func (c *App) advancedCommand(ctx context.Context) {
 	}
 }
 
+func (c *App) notificationTemplateOptions() notifytemplate.Options {
+	// perhaps make this configurable in the future
+	return notifytemplate.DefaultOptions
+}
+
 func init() {
 	kingpin.EnableFileExpansion = false
 }

cli/command_notification_template_set.go

@@ -77,7 +77,7 @@ func (c *commandNotificationTemplateSet) launchEditor(ctx context.Context, rep r
 	var lastUpdated string
 
 	if err := editor.EditLoop(ctx, "template.md", s, false, func(updated string) error {
-		_, err := notifytemplate.ParseTemplate(updated)
+		_, err := notifytemplate.ParseTemplate(updated, notifytemplate.DefaultOptions)
 		if err == nil {
 			lastUpdated = updated
 			return nil

cli/command_server_start.go

@@ -161,6 +161,9 @@ func (c *commandServerStart) serverStartOptions(ctx context.Context) (*server.Op
 		DebugScheduler:         c.debugScheduler,
 		MinMaintenanceInterval: c.minMaintenanceInterval,
 		DisableCSRFTokenChecks: c.disableCSRFTokenChecks,
+
+		EnableErrorNotifications: c.svc.enableErrorNotifications(),
+		NotifyTemplateOptions:    c.svc.notificationTemplateOptions(),
 	}, nil
 }
 

cli/command_snapshot_create.go

@@ -12,6 +12,8 @@
 
 	"github.com/kopia/kopia/fs"
 	"github.com/kopia/kopia/fs/virtualfs"
+	"github.com/kopia/kopia/notification"
+	"github.com/kopia/kopia/notification/notifydata"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/snapshot"
 	"github.com/kopia/kopia/snapshot/policy"
@@ -127,6 +129,8 @@ func (c *commandSnapshotCreate) run(ctx context.Context, rep repo.RepositoryWrit
 		return err
 	}
 
+	var st notifydata.MultiSnapshotStatus
+
 	for _, snapshotDir := range sources {
 		if u.IsCanceled() {
 			log(ctx).Info("Upload canceled")
@@ -138,11 +142,13 @@ func (c *commandSnapshotCreate) run(ctx context.Context, rep repo.RepositoryWrit
 			finalErrors = append(finalErrors, fmt.Sprintf("failed to prepare source: %s", err))
 		}
 
-		if err := c.snapshotSingleSource(ctx, fsEntry, setManual, rep, u, sourceInfo, tags); err != nil {
+		if err := c.snapshotSingleSource(ctx, fsEntry, setManual, rep, u, sourceInfo, tags, &st); err != nil {
 			finalErrors = append(finalErrors, err.Error())
 		}
 	}
 
+	notification.Send(ctx, rep, "snapshot-report", st, notification.SeverityReport, c.svc.notificationTemplateOptions())
+
 	// ensure we flush at least once in the session to properly close all pending buffers,
 	// otherwise the session will be reported as memory leak.
 	// by default the wrapper function does not flush on errors, which is what we want to do always.
@@ -259,27 +265,48 @@ func startTimeAfterEndTime(startTime, endTime time.Time) bool {
 		startTime.After(endTime)
 }
 
-//nolint:gocyclo
-func (c *commandSnapshotCreate) snapshotSingleSource(ctx context.Context, fsEntry fs.Entry, setManual bool, rep repo.RepositoryWriter, u *snapshotfs.Uploader, sourceInfo snapshot.SourceInfo, tags map[string]string) error {
+//nolint:gocyclo,funlen
+func (c *commandSnapshotCreate) snapshotSingleSource(
+	ctx context.Context,
+	fsEntry fs.Entry,
+	setManual bool,
+	rep repo.RepositoryWriter,
+	u *snapshotfs.Uploader,
+	sourceInfo snapshot.SourceInfo,
+	tags map[string]string,
+	st *notifydata.MultiSnapshotStatus,
+) (finalErr error) {
 	log(ctx).Infof("Snapshotting %v ...", sourceInfo)
 
-	var err error
+	var mwe notifydata.ManifestWithError
 
-	previous, err := findPreviousSnapshotManifest(ctx, rep, sourceInfo, nil)
-	if err != nil {
-		return err
+	mwe.Source = sourceInfo
+
+	st.Snapshots = append(st.Snapshots, &mwe)
+
+	defer func() {
+		if finalErr != nil {
+			mwe.Error = finalErr.Error()
+		}
+	}()
+
+	var previous []*snapshot.Manifest
+
+	previous, finalErr = findPreviousSnapshotManifest(ctx, rep, sourceInfo, nil)
+	if finalErr != nil {
+		return finalErr
 	}
 
-	policyTree, err := policy.TreeForSource(ctx, rep, sourceInfo)
-	if err != nil {
-		return errors.Wrap(err, "unable to get policy tree")
+	policyTree, finalErr := policy.TreeForSource(ctx, rep, sourceInfo)
+	if finalErr != nil {
+		return errors.Wrap(finalErr, "unable to get policy tree")
 	}
 
-	manifest, err := u.Upload(ctx, fsEntry, policyTree, sourceInfo, previous...)
-	if err != nil {
+	manifest, finalErr := u.Upload(ctx, fsEntry, policyTree, sourceInfo, previous...)
+	if finalErr != nil {
 		// fail-fast uploads will fail here without recording a manifest, other uploads will
 		// possibly fail later.
-		return errors.Wrap(err, "upload error")
+		return errors.Wrap(finalErr, "upload error")
 	}
 
 	manifest.Description = c.snapshotCreateDescription
@@ -308,6 +335,8 @@ func (c *commandSnapshotCreate) snapshotSingleSource(ctx context.Context, fsEntr
 		manifest.EndTime = fs.UTCTimestampFromTime(endTimeOverride)
 	}
 
+	mwe.Manifest = *manifest
+
 	ignoreIdenticalSnapshot := policyTree.EffectivePolicy().RetentionPolicy.IgnoreIdenticalSnapshots.OrDefault(false)
 	if ignoreIdenticalSnapshot && len(previous) > 0 {
 		if previous[0].RootObjectID() == manifest.RootObjectID() {
@@ -316,17 +345,17 @@ func (c *commandSnapshotCreate) snapshotSingleSource(ctx context.Context, fsEntr
 		}
 	}
 
-	if _, err = snapshot.SaveSnapshot(ctx, rep, manifest); err != nil {
-		return errors.Wrap(err, "cannot save manifest")
+	if _, finalErr = snapshot.SaveSnapshot(ctx, rep, manifest); finalErr != nil {
+		return errors.Wrap(finalErr, "cannot save manifest")
 	}
 
-	if _, err = policy.ApplyRetentionPolicy(ctx, rep, sourceInfo, true); err != nil {
-		return errors.Wrap(err, "unable to apply retention policy")
+	if _, finalErr = policy.ApplyRetentionPolicy(ctx, rep, sourceInfo, true); finalErr != nil {
+		return errors.Wrap(finalErr, "unable to apply retention policy")
 	}
 
 	if setManual {
-		if err = policy.SetManual(ctx, rep, sourceInfo); err != nil {
-			return errors.Wrap(err, "unable to set manual field in scheduling policy for source")
+		if finalErr = policy.SetManual(ctx, rep, sourceInfo); finalErr != nil {
+			return errors.Wrap(finalErr, "unable to set manual field in scheduling policy for source")
 		}
 	}
 

internal/server/grpc_session.go

@@ -132,7 +132,7 @@ func (s *Server) Session(srv grpcapi.KopiaRepository_SessionServer) error {
 			go func() {
 				defer s.grpcServerState.sem.Release(1)
 
-				handleSessionRequest(ctx, dw, authz, usernameAtHostname, req, func(resp *grpcapi.SessionResponse) {
+				s.handleSessionRequest(ctx, dw, authz, usernameAtHostname, req, func(resp *grpcapi.SessionResponse) {
 					if err := s.send(srv, req.GetRequestId(), resp); err != nil {
 						select {
 						case lastErr <- err:
@@ -149,7 +149,7 @@ func (s *Server) Session(srv grpcapi.KopiaRepository_SessionServer) error {
 
 var tracer = otel.Tracer("kopia/grpc")
 
-func handleSessionRequest(ctx context.Context, dw repo.DirectRepositoryWriter, authz auth.AuthorizationInfo, usernameAtHostname string, req *grpcapi.SessionRequest, respond func(*grpcapi.SessionResponse)) {
+func (s *Server) handleSessionRequest(ctx context.Context, dw repo.DirectRepositoryWriter, authz auth.AuthorizationInfo, usernameAtHostname string, req *grpcapi.SessionRequest, respond func(*grpcapi.SessionResponse)) {
 	if req.GetTraceContext() != nil {
 		var tc propagation.TraceContext
 		ctx = tc.Extract(ctx, propagation.MapCarrier(req.GetTraceContext()))
@@ -187,7 +187,7 @@ func handleSessionRequest(ctx context.Context, dw repo.DirectRepositoryWriter, a
 		respond(handleApplyRetentionPolicyRequest(ctx, dw, authz, usernameAtHostname, inner.ApplyRetentionPolicy))
 
 	case *grpcapi.SessionRequest_SendNotification:
-		respond(handleSendNotificationRequest(ctx, dw, authz, inner.SendNotification))
+		respond(s.handleSendNotificationRequest(ctx, dw, authz, inner.SendNotification))
 
 	case *grpcapi.SessionRequest_InitializeSession:
 		respond(errorResponse(errors.New("InitializeSession must be the first request in a session")))
@@ -488,7 +488,7 @@ func handleApplyRetentionPolicyRequest(ctx context.Context, rep repo.RepositoryW
 	}
 }
 
-func handleSendNotificationRequest(ctx context.Context, rep repo.RepositoryWriter, authz auth.AuthorizationInfo, req *grpcapi.SendNotificationRequest) *grpcapi.SessionResponse {
+func (s *Server) handleSendNotificationRequest(ctx context.Context, rep repo.RepositoryWriter, authz auth.AuthorizationInfo, req *grpcapi.SendNotificationRequest) *grpcapi.SessionResponse {
 	ctx, span := tracer.Start(ctx, "GRPCSession.SendNotification")
 	defer span.End()
 
@@ -499,7 +499,8 @@ func handleSendNotificationRequest(ctx context.Context, rep repo.RepositoryWrite
 	if err := notification.SendInternal(ctx, rep,
 		req.GetTemplateName(),
 		json.RawMessage(req.GetEventArgs()),
-		notification.Severity(req.GetSeverity())); err != nil {
+		notification.Severity(req.GetSeverity()),
+		s.options.NotifyTemplateOptions); err != nil {
 		return errorResponse(err)
 	}
 

internal/server/server.go

@@ -26,6 +26,7 @@
 	"github.com/kopia/kopia/internal/scheduler"
 	"github.com/kopia/kopia/internal/serverapi"
 	"github.com/kopia/kopia/internal/uitask"
+	"github.com/kopia/kopia/notification/notifytemplate"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/logging"
 	"github.com/kopia/kopia/repo/maintenance"
@@ -535,6 +536,14 @@ func (s *Server) endUpload(ctx context.Context, src snapshot.SourceInfo) {
 	s.parallelSnapshotsChanged.Signal()
 }
 
+func (s *Server) enableErrorNotifications() bool {
+	return s.options.EnableErrorNotifications
+}
+
+func (s *Server) notificationTemplateOptions() notifytemplate.Options {
+	return s.options.NotifyTemplateOptions
+}
+
 // SetRepository sets the repository (nil is allowed and indicates server that is not
 // connected to the repository).
 func (s *Server) SetRepository(ctx context.Context, rep repo.Repository) error {
@@ -785,23 +794,25 @@ func (s *Server) ServeStaticFiles(m *mux.Router, fs http.FileSystem) {
 
 // Options encompasses all API server options.
 type Options struct {
-	ConfigFile             string
-	ConnectOptions         *repo.ConnectOptions
-	RefreshInterval        time.Duration
-	MaxConcurrency         int
-	Authenticator          auth.Authenticator
-	Authorizer             auth.Authorizer
-	PasswordPersist        passwordpersist.Strategy
-	AuthCookieSigningKey   string
-	LogRequests            bool
-	UIUser                 string // name of the user allowed to access the UI API
-	UIPreferencesFile      string // name of the JSON file storing UI preferences
-	ServerControlUser      string // name of the user allowed to access the server control API
-	DisableCSRFTokenChecks bool
-	PersistentLogs         bool
-	UITitlePrefix          string
-	DebugScheduler         bool
-	MinMaintenanceInterval time.Duration
+	ConfigFile               string
+	ConnectOptions           *repo.ConnectOptions
+	RefreshInterval          time.Duration
+	MaxConcurrency           int
+	Authenticator            auth.Authenticator
+	Authorizer               auth.Authorizer
+	PasswordPersist          passwordpersist.Strategy
+	AuthCookieSigningKey     string
+	LogRequests              bool
+	UIUser                   string // name of the user allowed to access the UI API
+	UIPreferencesFile        string // name of the JSON file storing UI preferences
+	ServerControlUser        string // name of the user allowed to access the server control API
+	DisableCSRFTokenChecks   bool
+	PersistentLogs           bool
+	UITitlePrefix            string
+	DebugScheduler           bool
+	MinMaintenanceInterval   time.Duration
+	EnableErrorNotifications bool
+	NotifyTemplateOptions    notifytemplate.Options
 }
 
 // InitRepositoryFunc is a function that attempts to connect to/open repository.

internal/server/server_maintenance.go

@@ -8,6 +8,9 @@
 	"github.com/pkg/errors"
 
 	"github.com/kopia/kopia/internal/clock"
+	"github.com/kopia/kopia/notification"
+	"github.com/kopia/kopia/notification/notifydata"
+	"github.com/kopia/kopia/notification/notifytemplate"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/maintenance"
 )
@@ -32,6 +35,8 @@ type srvMaintenance struct {
 type maintenanceManagerServerInterface interface {
 	runMaintenanceTask(ctx context.Context, dr repo.DirectRepository) error
 	refreshScheduler(reason string)
+	enableErrorNotifications() bool
+	notificationTemplateOptions() notifytemplate.Options
 }
 
 func (s *srvMaintenance) trigger() {
@@ -138,9 +143,21 @@ func startMaintenanceManager(
 
 				m.beforeRun()
 
+				t0 := clock.Now()
+
 				if err := srv.runMaintenanceTask(mctx, rep); err != nil {
 					log(ctx).Debugw("maintenance task failed", "err", err)
 					m.afterFailedRun()
+
+					if srv.enableErrorNotifications() {
+						notification.Send(ctx,
+							rep,
+							"generic-error",
+							notifydata.NewErrorInfo("Maintenance", "Scheduled Maintenance", t0, clock.Now(), err),
+							notification.SeverityError,
+							srv.notificationTemplateOptions(),
+						)
+					}
 				}
 
 				m.refresh(mctx, true)

internal/server/server_maintenance_test.go

@@ -12,6 +12,7 @@
 
 	"github.com/kopia/kopia/internal/clock"
 	"github.com/kopia/kopia/internal/repotesting"
+	"github.com/kopia/kopia/notification/notifytemplate"
 	"github.com/kopia/kopia/repo"
 	"github.com/kopia/kopia/repo/maintenance"
 )
@@ -47,6 +48,14 @@ func (s *testServer) refreshScheduler(reason string) {
 	s.refreshSchedulerCount.Add(1)
 }
 
+func (s *testServer) enableErrorNotifications() bool {
+	return false
+}
+
+func (s *testServer) notificationTemplateOptions() notifytemplate.Options {
+	return notifytemplate.DefaultOptions
+}
+
 func TestServerMaintenance(t *testing.T) {
 	ctx, env := repotesting.NewEnvironment(t, repotesting.FormatNotImportant)
 

internal/server/server_test.go

@@ -290,7 +290,7 @@ func remoteRepositoryNotificationTest(t *testing.T, ctx context.Context, rep rep
 	}))
 	require.NoError(t, rw.Flush(ctx))
 
-	notification.Send(ctx, rep, notifytemplate.TestNotification, nil, notification.SeverityError)
+	notification.Send(ctx, rep, notifytemplate.TestNotification, nil, notification.SeverityError, notifytemplate.DefaultOptions)
 	require.Equal(t, int32(1), numRequestsReceived.Load())
 
 	// another webhook which fails
@@ -307,7 +307,7 @@ func remoteRepositoryNotificationTest(t *testing.T, ctx context.Context, rep rep
 	}))
 
 	require.NoError(t, rw.Flush(ctx))
-	notification.Send(ctx, rep, notifytemplate.TestNotification, nil, notification.SeverityError)
+	notification.Send(ctx, rep, notifytemplate.TestNotification, nil, notification.SeverityError, notifytemplate.DefaultOptions)
 	require.Equal(t, int32(1), numRequestsReceived.Load())
 }
 

notification/notification_send.go

@@ -105,7 +105,7 @@ func notificationSendersFromRepo(ctx context.Context, rep repo.Repository, sever
 
 // Send sends a notification for the given event.
 // Any errors encountered during the process are logged.
-func Send(ctx context.Context, rep repo.Repository, templateName string, eventArgs any, sev Severity) {
+func Send(ctx context.Context, rep repo.Repository, templateName string, eventArgs any, sev Severity, opt notifytemplate.Options) {
 	// if we're connected to a repository server, send the notification there.
 	if rem, ok := rep.(repo.RemoteNotifications); ok {
 		jsonData, err := json.Marshal(eventArgs)
@@ -122,13 +122,13 @@ func Send(ctx context.Context, rep repo.Repository, templateName string, eventAr
 		return
 	}
 
-	if err := SendInternal(ctx, rep, templateName, eventArgs, sev); err != nil {
+	if err := SendInternal(ctx, rep, templateName, eventArgs, sev, opt); err != nil {
 		log(ctx).Warnw("unable to send notification", "err", err)
 	}
 }
 
 // SendInternal sends a notification for the given event and returns an error.
-func SendInternal(ctx context.Context, rep repo.Repository, templateName string, eventArgs any, sev Severity) error {
+func SendInternal(ctx context.Context, rep repo.Repository, templateName string, eventArgs any, sev Severity, opt notifytemplate.Options) error {
 	senders, err := notificationSendersFromRepo(ctx, rep, sev)
 	if err != nil {
 		return errors.Wrap(err, "unable to get notification senders")
@@ -137,7 +137,7 @@ func SendInternal(ctx context.Context, rep repo.Repository, templateName string,
 	var resultErr error
 
 	for _, s := range senders {
-		if err := SendTo(ctx, rep, s, templateName, eventArgs, sev); err != nil {
+		if err := SendTo(ctx, rep, s, templateName, eventArgs, sev, opt); err != nil {
 			resultErr = multierr.Append(resultErr, err)
 		}
 	}
@@ -166,7 +166,7 @@ func MakeTemplateArgs(eventArgs any) TemplateArgs {
 }
 
 // SendTo sends a notification to the given sender.
-func SendTo(ctx context.Context, rep repo.Repository, s sender.Sender, templateName string, eventArgs any, sev Severity) error {
+func SendTo(ctx context.Context, rep repo.Repository, s sender.Sender, templateName string, eventArgs any, sev Severity, opt notifytemplate.Options) error {
 	// execute template
 	var bodyBuf bytes.Buffer
 
@@ -175,7 +175,7 @@ func SendTo(ctx context.Context, rep repo.Repository, s sender.Sender, templateN
 		return errors.Wrap(err, "unable to resolve notification template")
 	}
 
-	t, err := notifytemplate.ParseTemplate(tmpl)
+	t, err := notifytemplate.ParseTemplate(tmpl, opt)
 	if err != nil {
 		return errors.Wrap(err, "unable to parse notification template")
 	}
@@ -205,5 +205,5 @@ func SendTo(ctx context.Context, rep repo.Repository, s sender.Sender, templateN
 func SendTestNotification(ctx context.Context, rep repo.Repository, s sender.Sender) error {
 	log(ctx).Infof("Sending test notification to %v", s.Summary())
 
-	return SendTo(ctx, rep, s, notifytemplate.TestNotification, struct{}{}, SeveritySuccess)
+	return SendTo(ctx, rep, s, notifytemplate.TestNotification, struct{}{}, SeveritySuccess, notifytemplate.DefaultOptions)
 }

notification/notifydata/doc.go

@@ -0,0 +1,2 @@
+// Package notifydata contains the data structures used by the notification package.
+package notifydata

notification/notifydata/error_info.go

@@ -0,0 +1,43 @@
+package notifydata
+
+import (
+	"fmt"
+	"time"
+)
+
+// ErrorInfo represents information about errors.
+type ErrorInfo struct {
+	Operation        string    `json:"operation"`
+	OperationDetails string    `json:"operationDetails"`
+	StartTime        time.Time `json:"start"`
+	EndTime          time.Time `json:"end"`
+	ErrorMessage     string    `json:"error"`
+	ErrorDetails     string    `json:"errorDetails"`
+}
+
+// StartTimestamp returns the start time of the operation that caused the error.
+func (e *ErrorInfo) StartTimestamp() time.Time {
+	return e.StartTime.Truncate(time.Second)
+}
+
+// EndTimestamp returns the end time of the operation that caused the error.
+func (e *ErrorInfo) EndTimestamp() time.Time {
+	return e.EndTime.Truncate(time.Second)
+}
+
+// Duration returns the duration of the operation.
+func (e *ErrorInfo) Duration() time.Duration {
+	return e.EndTimestamp().Sub(e.StartTimestamp())
+}
+
+// NewErrorInfo creates a new ErrorInfo.
+func NewErrorInfo(operation, operationDetails string, startTime, endTime time.Time, err error) *ErrorInfo {
+	return &ErrorInfo{
+		Operation:        operation,
+		OperationDetails: operationDetails,
+		StartTime:        startTime,
+		EndTime:          endTime,
+		ErrorMessage:     fmt.Sprintf("%v", err),
+		ErrorDetails:     fmt.Sprintf("%+v", err),
+	}
+}

notification/notifydata/multi_snapshot_status.go

@@ -0,0 +1,96 @@
+package notifydata
+
+import (
+	"time"
+
+	"github.com/kopia/kopia/snapshot"
+)
+
+// ManifestWithError represents information about the snapshot manifest with optional error.
+type ManifestWithError struct {
+	snapshot.Manifest `json:"manifest"` // may not be filled out if there was an error, Manifst.Source is always set.
+
+	Error string `json:"error"` // will be present if there was an error
+}
+
+// StartTimestamp returns the start time of the snapshot.
+func (m *ManifestWithError) StartTimestamp() time.Time {
+	return m.StartTime.ToTime().UTC().Truncate(time.Second)
+}
+
+// EndTimestamp returns the end time of the snapshot.
+func (m *ManifestWithError) EndTimestamp() time.Time {
+	return m.EndTime.ToTime().UTC().Truncate(time.Second)
+}
+
+// TotalSize returns the total size of the snapshot in bytes.
+func (m *ManifestWithError) TotalSize() int64 {
+	if m.RootEntry == nil {
+		return 0
+	}
+
+	if m.RootEntry.DirSummary != nil {
+		return m.RootEntry.DirSummary.TotalFileSize
+	}
+
+	return m.RootEntry.FileSize
+}
+
+// TotalFiles returns the total number of files in the snapshot.
+func (m *ManifestWithError) TotalFiles() int64 {
+	if m.RootEntry == nil {
+		return 0
+	}
+
+	if m.RootEntry.DirSummary != nil {
+		return m.RootEntry.DirSummary.TotalFileCount
+	}
+
+	return 1
+}
+
+// TotalDirs returns the total number of directories in the snapshot.
+func (m *ManifestWithError) TotalDirs() int64 {
+	if m.RootEntry == nil {
+		return 0
+	}
+
+	if m.RootEntry.DirSummary != nil {
+		return m.RootEntry.DirSummary.TotalDirCount
+	}
+
+	return 0
+}
+
+// Duration returns the duration of the snapshot.
+func (m *ManifestWithError) Duration() time.Duration {
+	return time.Duration(m.EndTime - m.StartTime)
+}
+
+// StatusCode returns the status code of the manifest.
+func (m *ManifestWithError) StatusCode() string {
+	if m.Error != "" {
+		return "fatal"
+	}
+
+	if m.Manifest.IncompleteReason != "" {
+		return "incomplete"
+	}
+
+	if m.Manifest.RootEntry != nil && m.Manifest.RootEntry.DirSummary != nil {
+		if m.Manifest.RootEntry.DirSummary.FatalErrorCount > 0 {
+			return "fatal"
+		}
+
+		if m.Manifest.RootEntry.DirSummary.IgnoredErrorCount > 0 {
+			return "error"
+		}
+	}
+
+	return "ok"
+}
+
+// MultiSnapshotStatus represents the status of multiple snapshots.
+type MultiSnapshotStatus struct {
+	Snapshots []*ManifestWithError `json:"snapshots"`
+}

notification/notifytemplate/embeddedtemplate.go

@@ -3,12 +3,15 @@
 
 import (
 	"embed"
+	"slices"
+	"sort"
 	"text/template"
 	"time"
 
 	"github.com/pkg/errors"
 
-	"github.com/kopia/kopia/fs"
+	"github.com/kopia/kopia/internal/units"
+	"github.com/kopia/kopia/notification/notifydata"
 )
 
 //go:embed "*.html"
@@ -20,22 +23,41 @@
 	TestNotification = "test-notification"
 )
 
-// Functions is a map of functions that can be used in templates.
+// Options provides options for template rendering.
+type Options struct {
+	Timezone   *time.Location
+	TimeFormat string
+}
+
+// functions is a map of functions that can be used in templates.
+func functions(opt Options) template.FuncMap {
+	if opt.Timezone == nil {
+		opt.Timezone = time.Local
+	}
+
+	if opt.TimeFormat == "" {
+		opt.TimeFormat = time.RFC1123Z
+	}
+
+	return template.FuncMap{
+		"bytes": units.BytesString[int64],
+		"sortSnapshotManifestsByName": func(man []*notifydata.ManifestWithError) []*notifydata.ManifestWithError {
+			res := slices.Clone(man)
+			sort.Slice(res, func(i, j int) bool {
+				return res[i].Source.String() < res[j].Source.String()
+			})
+			return res
+		},
+		"formatTime": func(t time.Time) string {
+			return t.In(opt.Timezone).Format(opt.TimeFormat)
+		},
+	}
+}
+
+// DefaultOptions is the default set of options.
 //
 //nolint:gochecknoglobals
-var Functions = template.FuncMap{
-	"toTime": func(t any) time.Time {
-		if t, ok := t.(time.Time); ok {
-			return t
-		}
-
-		if t, ok := t.(fs.UTCTimestamp); ok {
-			return t.ToTime()
-		}
-
-		return time.Time{}
-	},
-}
+var DefaultOptions = Options{}
 
 // GetEmbeddedTemplate returns embedded template by name.
 func GetEmbeddedTemplate(templateName string) (string, error) {
@@ -61,7 +83,7 @@ func SupportedTemplates() []string {
 }
 
 // ParseTemplate parses a named template.
-func ParseTemplate(tmpl string) (*template.Template, error) {
+func ParseTemplate(tmpl string, opt Options) (*template.Template, error) {
 	//nolint:wrapcheck
-	return template.New("template").Funcs(Functions).Parse(tmpl)
+	return template.New("template").Funcs(functions(opt)).Parse(tmpl)
 }

notification/notifytemplate/generic-error.html

@@ -0,0 +1,20 @@
+Subject: Kopia has encountered an error during {{ .EventArgs.Operation }} on {{.Hostname}}
+
+<!doctype html>
+<html>
+<head>
+</head>
+<body>
+
+<p><b>Operation:</b> {{ .EventArgs.OperationDetails }}</p>
+<p><b>Started:</b> {{ .EventArgs.StartTimestamp | formatTime }}</p>
+<p><b>Finished:</b> {{ .EventArgs.EndTimestamp | formatTime }} ({{ .EventArgs.Duration }})</p>
+
+<p><b>Message:</b> {{ .EventArgs.ErrorMessage }}</p>
+
+<pre>{{ .EventArgs.ErrorDetails }}</pre>
+
+<p>Generated at {{ .EventTime | formatTime }} by <a href="https://kopia.io">Kopia {{ .KopiaBuildVersion }}</a>.</p>
+
+</body>
+</html>

notification/notifytemplate/generic-error.txt

@@ -0,0 +1,11 @@
+Subject: Kopia has encountered an error during {{ .EventArgs.Operation }} on {{.Hostname}}
+
+Operation: {{ .EventArgs.OperationDetails }}
+Started:   {{ .EventArgs.StartTimestamp | formatTime }}
+Finished:  {{ .EventArgs.EndTimestamp | formatTime }} ({{ .EventArgs.Duration }})
+
+{{ .EventArgs.ErrorDetails }}
+
+Generated at {{ .EventTime | formatTime }} by Kopia {{ .KopiaBuildVersion }}.
+
+https://kopia.io/

notification/notifytemplate/notifytemplate_test.go

@@ -0,0 +1,118 @@
+package notifytemplate_test
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/kopia/kopia/fs"
+	"github.com/kopia/kopia/notification"
+	"github.com/kopia/kopia/notification/notifydata"
+	"github.com/kopia/kopia/notification/notifytemplate"
+	"github.com/kopia/kopia/snapshot"
+)
+
+var defaultTestOptions = notifytemplate.Options{
+	Timezone: time.UTC,
+}
+
+var altTestOptions = notifytemplate.Options{
+	Timezone:   time.FixedZone("PST", -8*60*60),
+	TimeFormat: time.RFC1123,
+}
+
+func TestNotifyTemplate_generic_error(t *testing.T) {
+	args := notification.MakeTemplateArgs(&notifydata.ErrorInfo{
+		Operation:        "Some Operation",
+		OperationDetails: "Some Operation Details",
+		ErrorMessage:     "error message",
+		ErrorDetails:     "error details",
+		StartTime:        time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC),
+		EndTime:          time.Date(2020, 1, 2, 3, 4, 6, 7, time.UTC),
+	})
+
+	args.EventTime = time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC)
+	args.Hostname = "some-host"
+
+	verifyTemplate(t, "generic-error.txt", ".default", args, defaultTestOptions)
+	verifyTemplate(t, "generic-error.html", ".default", args, defaultTestOptions)
+	verifyTemplate(t, "generic-error.txt", ".alt", args, altTestOptions)
+	verifyTemplate(t, "generic-error.html", ".alt", args, altTestOptions)
+}
+
+func TestNotifyTemplate_snapshot_report(t *testing.T) {
+	args := notification.MakeTemplateArgs(&notifydata.MultiSnapshotStatus{
+		Snapshots: []*notifydata.ManifestWithError{
+			{
+				Manifest: snapshot.Manifest{
+					Source:    snapshot.SourceInfo{Host: "some-host", UserName: "some-user", Path: "/some/path"},
+					StartTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC).UnixNano()),
+					EndTime:   fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 6, 7, time.UTC).UnixNano()),
+					RootEntry: &snapshot.DirEntry{
+						DirSummary: &fs.DirectorySummary{
+							TotalFileCount: 123,
+							TotalFileSize:  456,
+							TotalDirCount:  33,
+							FailedEntries: []*fs.EntryWithError{
+								{
+									EntryPath: "/some/path",
+									Error:     "some error",
+								},
+								{
+									EntryPath: "/some/path2",
+									Error:     "some error",
+								},
+							},
+						},
+					},
+				},
+			},
+			{
+				Error: "some top-level error",
+				Manifest: snapshot.Manifest{
+					Source: snapshot.SourceInfo{Host: "some-host", UserName: "some-user", Path: "/some/other/path"},
+				},
+			},
+		},
+	})
+
+	args.EventTime = time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC)
+	args.Hostname = "some-host"
+
+	verifyTemplate(t, "snapshot-report.txt", ".default", args, defaultTestOptions)
+	verifyTemplate(t, "snapshot-report.html", ".default", args, defaultTestOptions)
+	verifyTemplate(t, "snapshot-report.txt", ".alt", args, altTestOptions)
+	verifyTemplate(t, "snapshot-report.html", ".alt", args, altTestOptions)
+}
+
+func verifyTemplate(t *testing.T, embeddedTemplateName, expectedSuffix string, args interface{}, opt notifytemplate.Options) {
+	t.Helper()
+
+	tmpl, err := notifytemplate.GetEmbeddedTemplate(embeddedTemplateName)
+	require.NoError(t, err)
+
+	tt, err := notifytemplate.ParseTemplate(tmpl, opt)
+	require.NoError(t, err)
+
+	var buf bytes.Buffer
+
+	require.NoError(t, tt.Execute(&buf, args))
+
+	actualFileName := filepath.Join("testdata", embeddedTemplateName+expectedSuffix+".actual")
+	require.NoError(t, os.WriteFile(actualFileName, buf.Bytes(), 0o644))
+
+	expectedFileName := filepath.Join("testdata", embeddedTemplateName+expectedSuffix+".expected")
+
+	wantBytes, err := os.ReadFile(expectedFileName)
+	require.NoError(t, err)
+
+	want := string(wantBytes)
+
+	require.Equal(t, want, buf.String())
+
+	require.NoError(t, os.Remove(actualFileName))
+}

notification/notifytemplate/snapshot-report.html

@@ -0,0 +1,103 @@
+Subject: Kopia created {{ len .EventArgs.Snapshots }} snapshot{{ if gt (len .EventArgs.Snapshots) 1 }}s{{end}} on {{.Hostname}}
+
+<!doctype html>
+<html>
+<head>
+<style type="text/css">
+    table {
+        width: 100%;
+        border-collapse: collapse;
+    }
+
+    thead tr {
+        background-color: #f2f2f2;
+    }
+
+    th, td {
+        border: 1px solid black;
+        padding: 2px;
+        text-align: left;
+    }
+
+    th {
+        background-color: #f2f2f2;
+    }
+
+    code {
+        background-color: #f2f2f2;
+        padding: 2px;
+    }
+
+    span.path {
+        font-family: monospace;
+        color: #344652;
+        font-weight: bold;
+    }
+
+    tr.snapshotstatus-fatal {
+        background-color: #fde9e4;
+    }
+
+    tr.snapshotstatus-error {
+        background-color: #fcffba;
+    }
+
+    tr.snapshotstatus-incomplete {
+        background-color: #8a8c7e;
+    }
+</style>
+</head>
+<body>
+<table border="1">
+<thead>
+    <tr>
+        <th>Source</th>
+        <th>Started</th>
+        <th>Duration</th>
+        <th>Total Size</th>
+        <th>Total Files</th>
+        <th>Total Directories</th>
+    </tr>
+</thead>
+{{ range .EventArgs.Snapshots | sortSnapshotManifestsByName}}
+<tr class="snapshotstatus-{{ .StatusCode }}">
+<td><span class="path">{{ .Source.Path }}</span></td>
+<td>{{ .StartTimestamp | formatTime }}</td>
+<td>{{ .Duration }}</td>
+<td>{{ .TotalSize | bytes }}</td>
+<td>{{ .TotalFiles }}</td>
+<td>{{ .TotalDirs }}</td>
+</tr>
+
+{{ if .Error }}
+<tr class="snapshotstatus-{{ .StatusCode }}">
+    <td colspan="6">
+        <b style="color:red">Error:</b> {{ .Error }}
+    </td>
+</tr>
+{{ end }}
+
+{{ if .RootEntry }}
+{{ if .RootEntry.DirSummary }}
+{{ if .RootEntry.DirSummary.FailedEntries }}
+<tr class="snapshotstatus-{{ .StatusCode }}">
+    <td colspan="6">
+        <b style="color:red">Failed Entries:</b>
+        <ul>
+        {{ range .RootEntry.DirSummary.FailedEntries }}
+        <li><span class="path">{{.EntryPath}}</span>: {{.Error}}</li>
+        {{ end }}
+        </ul>
+    </td>
+</tr>
+{{ end }}
+{{ end }}
+{{ end }}
+
+{{ end }}
+</table>
+
+<p>Generated at {{ .EventTime | formatTime }} by <a href="https://kopia.io">Kopia {{ .KopiaBuildVersion }}</a>.</p>
+
+</body>
+</html>

notification/notifytemplate/snapshot-report.txt

@@ -0,0 +1,16 @@
+Subject: Kopia created {{ len .EventArgs.Snapshots }} snapshot{{ if gt (len .EventArgs.Snapshots) 1 }}s{{end}} on {{.Hostname}}
+
+{{ range .EventArgs.Snapshots | sortSnapshotManifestsByName}}Path: {{ .Source.Path }}
+
+  Status: {{ .StatusCode }}
+  Start: {{ .StartTimestamp | formatTime  }} Duration: {{ .Duration }}
+  Size: {{ .TotalSize | bytes }}, {{ .TotalFiles }} files, {{ .TotalDirs }} directories.
+{{ if .Error }}  Error: {{ .Error }}
+{{ end }}{{ if .RootEntry }}{{ if .RootEntry.DirSummary }}{{ if .RootEntry.DirSummary.FailedEntries }}
+  Failed Entries:
+{{ range .RootEntry.DirSummary.FailedEntries }}
+  - {{.EntryPath}}: {{.Error}}{{ end }}{{ end }}{{ end }}
+{{ end }}
+{{ end }}Generated at {{ .EventTime | formatTime }} by Kopia {{ .KopiaBuildVersion }}.
+
+https://kopia.io/

notification/notifytemplate/test-notification.html

@@ -1,4 +1,4 @@
-Subject: Test notification from Kopia at {{ .EventTime }}
+Subject: Test notification from Kopia at {{ .EventTime | formatTime }}
 
 <p>This is a test notification from Kopia.</p>
 

notification/notifytemplate/test-notification.txt

@@ -1,4 +1,4 @@
-Subject: Test notification from Kopia at {{ .EventTime }}
+Subject: Test notification from Kopia at {{ .EventTime | formatTime }}
 
 This is a test notification from Kopia.
 

notification/notifytemplate/testdata/generic-error.html.alt.expected

@@ -0,0 +1,20 @@
+Subject: Kopia has encountered an error during Some Operation on some-host
+
+<!doctype html>
+<html>
+<head>
+</head>
+<body>
+
+<p><b>Operation:</b> Some Operation Details</p>
+<p><b>Started:</b> Wed, 01 Jan 2020 19:04:05 PST</p>
+<p><b>Finished:</b> Wed, 01 Jan 2020 19:04:06 PST (1s)</p>
+
+<p><b>Message:</b> error message</p>
+
+<pre>error details</pre>
+
+<p>Generated at Wed, 01 Jan 2020 19:04:05 PST by <a href="https://kopia.io">Kopia v0-unofficial</a>.</p>
+
+</body>
+</html>

notification/notifytemplate/testdata/generic-error.html.default.expected

@@ -0,0 +1,20 @@
+Subject: Kopia has encountered an error during Some Operation on some-host
+
+<!doctype html>
+<html>
+<head>
+</head>
+<body>
+
+<p><b>Operation:</b> Some Operation Details</p>
+<p><b>Started:</b> Thu, 02 Jan 2020 03:04:05 +0000</p>
+<p><b>Finished:</b> Thu, 02 Jan 2020 03:04:06 +0000 (1s)</p>
+
+<p><b>Message:</b> error message</p>
+
+<pre>error details</pre>
+
+<p>Generated at Thu, 02 Jan 2020 03:04:05 +0000 by <a href="https://kopia.io">Kopia v0-unofficial</a>.</p>
+
+</body>
+</html>

notification/notifytemplate/testdata/generic-error.txt.alt.expected

@@ -0,0 +1,11 @@
+Subject: Kopia has encountered an error during Some Operation on some-host
+
+Operation: Some Operation Details
+Started:   Wed, 01 Jan 2020 19:04:05 PST
+Finished:  Wed, 01 Jan 2020 19:04:06 PST (1s)
+
+error details
+
+Generated at Wed, 01 Jan 2020 19:04:05 PST by Kopia v0-unofficial.
+
+https://kopia.io/

notification/notifytemplate/testdata/generic-error.txt.default.expected

@@ -0,0 +1,11 @@
+Subject: Kopia has encountered an error during Some Operation on some-host
+
+Operation: Some Operation Details
+Started:   Thu, 02 Jan 2020 03:04:05 +0000
+Finished:  Thu, 02 Jan 2020 03:04:06 +0000 (1s)
+
+error details
+
+Generated at Thu, 02 Jan 2020 03:04:05 +0000 by Kopia v0-unofficial.
+
+https://kopia.io/

notification/notifytemplate/testdata/snapshot-report.html.alt.expected

@@ -0,0 +1,119 @@
+Subject: Kopia created 2 snapshots on some-host
+
+<!doctype html>
+<html>
+<head>
+<style type="text/css">
+    table {
+        width: 100%;
+        border-collapse: collapse;
+    }
+
+    thead tr {
+        background-color: #f2f2f2;
+    }
+
+    th, td {
+        border: 1px solid black;
+        padding: 2px;
+        text-align: left;
+    }
+
+    th {
+        background-color: #f2f2f2;
+    }
+
+    code {
+        background-color: #f2f2f2;
+        padding: 2px;
+    }
+
+    span.path {
+        font-family: monospace;
+        color: #344652;
+        font-weight: bold;
+    }
+
+    tr.snapshotstatus-fatal {
+        background-color: #fde9e4;
+    }
+
+    tr.snapshotstatus-error {
+        background-color: #fcffba;
+    }
+
+    tr.snapshotstatus-incomplete {
+        background-color: #8a8c7e;
+    }
+</style>
+</head>
+<body>
+<table border="1">
+<thead>
+    <tr>
+        <th>Source</th>
+        <th>Started</th>
+        <th>Duration</th>
+        <th>Total Size</th>
+        <th>Total Files</th>
+        <th>Total Directories</th>
+    </tr>
+</thead>
+
+<tr class="snapshotstatus-fatal">
+<td><span class="path">/some/other/path</span></td>
+<td>Wed, 31 Dec 1969 16:00:00 PST</td>
+<td>0s</td>
+<td>0 B</td>
+<td>0</td>
+<td>0</td>
+</tr>
+
+
+<tr class="snapshotstatus-fatal">
+    <td colspan="6">
+        <b style="color:red">Error:</b> some top-level error
+    </td>
+</tr>
+
+
+
+
+
+<tr class="snapshotstatus-ok">
+<td><span class="path">/some/path</span></td>
+<td>Wed, 01 Jan 2020 19:04:05 PST</td>
+<td>1.000000001s</td>
+<td>456 B</td>
+<td>123</td>
+<td>33</td>
+</tr>
+
+
+
+
+
+
+<tr class="snapshotstatus-ok">
+    <td colspan="6">
+        <b style="color:red">Failed Entries:</b>
+        <ul>
+        
+        <li><span class="path">/some/path</span>: some error</li>
+        
+        <li><span class="path">/some/path2</span>: some error</li>
+        
+        </ul>
+    </td>
+</tr>
+
+
+
+
+
+</table>
+
+<p>Generated at Wed, 01 Jan 2020 19:04:05 PST by <a href="https://kopia.io">Kopia v0-unofficial</a>.</p>
+
+</body>
+</html>

notification/notifytemplate/testdata/snapshot-report.html.default.expected

@@ -0,0 +1,119 @@
+Subject: Kopia created 2 snapshots on some-host
+
+<!doctype html>
+<html>
+<head>
+<style type="text/css">
+    table {
+        width: 100%;
+        border-collapse: collapse;
+    }
+
+    thead tr {
+        background-color: #f2f2f2;
+    }
+
+    th, td {
+        border: 1px solid black;
+        padding: 2px;
+        text-align: left;
+    }
+
+    th {
+        background-color: #f2f2f2;
+    }
+
+    code {
+        background-color: #f2f2f2;
+        padding: 2px;
+    }
+
+    span.path {
+        font-family: monospace;
+        color: #344652;
+        font-weight: bold;
+    }
+
+    tr.snapshotstatus-fatal {
+        background-color: #fde9e4;
+    }
+
+    tr.snapshotstatus-error {
+        background-color: #fcffba;
+    }
+
+    tr.snapshotstatus-incomplete {
+        background-color: #8a8c7e;
+    }
+</style>
+</head>
+<body>
+<table border="1">
+<thead>
+    <tr>
+        <th>Source</th>
+        <th>Started</th>
+        <th>Duration</th>
+        <th>Total Size</th>
+        <th>Total Files</th>
+        <th>Total Directories</th>
+    </tr>
+</thead>
+
+<tr class="snapshotstatus-fatal">
+<td><span class="path">/some/other/path</span></td>
+<td>Thu, 01 Jan 1970 00:00:00 +0000</td>
+<td>0s</td>
+<td>0 B</td>
+<td>0</td>
+<td>0</td>
+</tr>
+
+
+<tr class="snapshotstatus-fatal">
+    <td colspan="6">
+        <b style="color:red">Error:</b> some top-level error
+    </td>
+</tr>
+
+
+
+
+
+<tr class="snapshotstatus-ok">
+<td><span class="path">/some/path</span></td>
+<td>Thu, 02 Jan 2020 03:04:05 +0000</td>
+<td>1.000000001s</td>
+<td>456 B</td>
+<td>123</td>
+<td>33</td>
+</tr>
+
+
+
+
+
+
+<tr class="snapshotstatus-ok">
+    <td colspan="6">
+        <b style="color:red">Failed Entries:</b>
+        <ul>
+        
+        <li><span class="path">/some/path</span>: some error</li>
+        
+        <li><span class="path">/some/path2</span>: some error</li>
+        
+        </ul>
+    </td>
+</tr>
+
+
+
+
+
+</table>
+
+<p>Generated at Thu, 02 Jan 2020 03:04:05 +0000 by <a href="https://kopia.io">Kopia v0-unofficial</a>.</p>
+
+</body>
+</html>

notification/notifytemplate/testdata/snapshot-report.txt.alt.expected

@@ -0,0 +1,23 @@
+Subject: Kopia created 2 snapshots on some-host
+
+Path: /some/other/path
+
+  Status: fatal
+  Start: Wed, 31 Dec 1969 16:00:00 PST Duration: 0s
+  Size: 0 B, 0 files, 0 directories.
+  Error: some top-level error
+
+Path: /some/path
+
+  Status: ok
+  Start: Wed, 01 Jan 2020 19:04:05 PST Duration: 1.000000001s
+  Size: 456 B, 123 files, 33 directories.
+
+  Failed Entries:
+
+  - /some/path: some error
+  - /some/path2: some error
+
+Generated at Wed, 01 Jan 2020 19:04:05 PST by Kopia v0-unofficial.
+
+https://kopia.io/

notification/notifytemplate/testdata/snapshot-report.txt.default.expected

@@ -0,0 +1,23 @@
+Subject: Kopia created 2 snapshots on some-host
+
+Path: /some/other/path
+
+  Status: fatal
+  Start: Thu, 01 Jan 1970 00:00:00 +0000 Duration: 0s
+  Size: 0 B, 0 files, 0 directories.
+  Error: some top-level error
+
+Path: /some/path
+
+  Status: ok
+  Start: Thu, 02 Jan 2020 03:04:05 +0000 Duration: 1.000000001s
+  Size: 456 B, 123 files, 33 directories.
+
+  Failed Entries:
+
+  - /some/path: some error
+  - /some/path2: some error
+
+Generated at Thu, 02 Jan 2020 03:04:05 +0000 by Kopia v0-unofficial.
+
+https://kopia.io/