diff --git a/cli/app.go b/cli/app.go index 430835752..b9bf1fe1c 100644 --- a/cli/app.go +++ b/cli/app.go @@ -16,9 +16,13 @@ "go.opentelemetry.io/otel/trace" "github.com/kopia/kopia/internal/apiclient" + "github.com/kopia/kopia/internal/clock" "github.com/kopia/kopia/internal/gather" "github.com/kopia/kopia/internal/passwordpersist" "github.com/kopia/kopia/internal/releasable" + "github.com/kopia/kopia/notification" + "github.com/kopia/kopia/notification/notifydata" + "github.com/kopia/kopia/notification/notifytemplate" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/blob" "github.com/kopia/kopia/repo/logging" @@ -88,6 +92,7 @@ type appServices interface { repositoryConfigFileName() string getProgress() *cliProgress getRestoreProgress() RestoreProgress + notificationTemplateOptions() notifytemplate.Options stdout() io.Writer Stderr() io.Writer @@ -112,6 +117,7 @@ type advancedAppServices interface { getPasswordFromFlags(ctx context.Context, isCreate, allowPersistent bool) (string, error) optionsFromFlags(ctx context.Context) *repo.Options runAppWithContext(command *kingpin.CmdClause, callback func(ctx context.Context) error) error + enableErrorNotifications() bool } // App contains per-invocation flags and state of Kopia CLI. @@ -139,6 +145,8 @@ type App struct { upgradeOwnerID string doNotWaitForUpgrade bool + errorNotifications bool + currentAction string onExitCallbacks []func() onFatalErrorCallbacks []func(err error) @@ -240,6 +248,10 @@ func (c *App) passwordPersistenceStrategy() passwordpersist.Strategy { return passwordpersist.File() } +func (c *App) enableErrorNotifications() bool { + return c.errorNotifications +} + func (c *App) setup(app *kingpin.Application) { app.PreAction(func(pc *kingpin.ParseContext) error { if sc := pc.SelectedCommand; sc != nil { @@ -274,6 +286,7 @@ func (c *App) setup(app *kingpin.Application) { app.Flag("dump-allocator-stats", "Dump allocator stats at the end of execution.").Hidden().Envar(c.EnvName("KOPIA_DUMP_ALLOCATOR_STATS")).BoolVar(&c.dumpAllocatorStats) app.Flag("upgrade-owner-id", "Repository format upgrade owner-id.").Hidden().Envar(c.EnvName("KOPIA_REPO_UPGRADE_OWNER_ID")).StringVar(&c.upgradeOwnerID) app.Flag("upgrade-no-block", "Do not block when repository format upgrade is in progress, instead exit with a message.").Hidden().Default("false").Envar(c.EnvName("KOPIA_REPO_UPGRADE_NO_BLOCK")).BoolVar(&c.doNotWaitForUpgrade) + app.Flag("error-notifications", "Send notification on errors").Hidden().Envar(c.EnvName("KOPIA_SEND_ERROR_NOTIFICATIONS")).Default("true").BoolVar(&c.errorNotifications) if c.enableTestOnlyFlags() { app.Flag("ignore-missing-required-features", "Open repository despite missing features (VERY DANGEROUS, ONLY FOR TESTING)").Hidden().BoolVar(&c.testonlyIgnoreMissingRequiredFeatures) @@ -562,6 +575,8 @@ func (c *App) maybeRepositoryAction(act func(ctx context.Context, rep repo.Repos return errors.Wrap(err, "open repository") } + t0 := clock.Now() + err = act(ctx, rep) if rep != nil && err == nil && !mode.disableMaintenance { @@ -570,6 +585,17 @@ func (c *App) maybeRepositoryAction(act func(ctx context.Context, rep repo.Repos } } + if err != nil && rep != nil && c.errorNotifications { + notification.Send(ctx, rep, "generic-error", notifydata.NewErrorInfo( + c.currentActionName(), + c.currentActionName(), + t0, + clock.Now(), + err), notification.SeverityError, + c.notificationTemplateOptions(), + ) + } + if rep != nil && mode.mustBeConnected { if cerr := rep.Close(ctx); cerr != nil { return errors.Wrap(cerr, "unable to close repository") @@ -647,6 +673,11 @@ func (c *App) advancedCommand(ctx context.Context) { } } +func (c *App) notificationTemplateOptions() notifytemplate.Options { + // perhaps make this configurable in the future + return notifytemplate.DefaultOptions +} + func init() { kingpin.EnableFileExpansion = false } diff --git a/cli/command_notification_template_set.go b/cli/command_notification_template_set.go index 82d5b6b17..5c1ba85be 100644 --- a/cli/command_notification_template_set.go +++ b/cli/command_notification_template_set.go @@ -77,7 +77,7 @@ func (c *commandNotificationTemplateSet) launchEditor(ctx context.Context, rep r var lastUpdated string if err := editor.EditLoop(ctx, "template.md", s, false, func(updated string) error { - _, err := notifytemplate.ParseTemplate(updated) + _, err := notifytemplate.ParseTemplate(updated, notifytemplate.DefaultOptions) if err == nil { lastUpdated = updated return nil diff --git a/cli/command_server_start.go b/cli/command_server_start.go index 98ff726a2..cb5094905 100644 --- a/cli/command_server_start.go +++ b/cli/command_server_start.go @@ -161,6 +161,9 @@ func (c *commandServerStart) serverStartOptions(ctx context.Context) (*server.Op DebugScheduler: c.debugScheduler, MinMaintenanceInterval: c.minMaintenanceInterval, DisableCSRFTokenChecks: c.disableCSRFTokenChecks, + + EnableErrorNotifications: c.svc.enableErrorNotifications(), + NotifyTemplateOptions: c.svc.notificationTemplateOptions(), }, nil } diff --git a/cli/command_snapshot_create.go b/cli/command_snapshot_create.go index 26700cbe1..7f34127ef 100644 --- a/cli/command_snapshot_create.go +++ b/cli/command_snapshot_create.go @@ -12,6 +12,8 @@ "github.com/kopia/kopia/fs" "github.com/kopia/kopia/fs/virtualfs" + "github.com/kopia/kopia/notification" + "github.com/kopia/kopia/notification/notifydata" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/snapshot" "github.com/kopia/kopia/snapshot/policy" @@ -127,6 +129,8 @@ func (c *commandSnapshotCreate) run(ctx context.Context, rep repo.RepositoryWrit return err } + var st notifydata.MultiSnapshotStatus + for _, snapshotDir := range sources { if u.IsCanceled() { log(ctx).Info("Upload canceled") @@ -138,11 +142,13 @@ func (c *commandSnapshotCreate) run(ctx context.Context, rep repo.RepositoryWrit finalErrors = append(finalErrors, fmt.Sprintf("failed to prepare source: %s", err)) } - if err := c.snapshotSingleSource(ctx, fsEntry, setManual, rep, u, sourceInfo, tags); err != nil { + if err := c.snapshotSingleSource(ctx, fsEntry, setManual, rep, u, sourceInfo, tags, &st); err != nil { finalErrors = append(finalErrors, err.Error()) } } + notification.Send(ctx, rep, "snapshot-report", st, notification.SeverityReport, c.svc.notificationTemplateOptions()) + // ensure we flush at least once in the session to properly close all pending buffers, // otherwise the session will be reported as memory leak. // by default the wrapper function does not flush on errors, which is what we want to do always. @@ -259,27 +265,48 @@ func startTimeAfterEndTime(startTime, endTime time.Time) bool { startTime.After(endTime) } -//nolint:gocyclo -func (c *commandSnapshotCreate) snapshotSingleSource(ctx context.Context, fsEntry fs.Entry, setManual bool, rep repo.RepositoryWriter, u *snapshotfs.Uploader, sourceInfo snapshot.SourceInfo, tags map[string]string) error { +//nolint:gocyclo,funlen +func (c *commandSnapshotCreate) snapshotSingleSource( + ctx context.Context, + fsEntry fs.Entry, + setManual bool, + rep repo.RepositoryWriter, + u *snapshotfs.Uploader, + sourceInfo snapshot.SourceInfo, + tags map[string]string, + st *notifydata.MultiSnapshotStatus, +) (finalErr error) { log(ctx).Infof("Snapshotting %v ...", sourceInfo) - var err error + var mwe notifydata.ManifestWithError - previous, err := findPreviousSnapshotManifest(ctx, rep, sourceInfo, nil) - if err != nil { - return err + mwe.Source = sourceInfo + + st.Snapshots = append(st.Snapshots, &mwe) + + defer func() { + if finalErr != nil { + mwe.Error = finalErr.Error() + } + }() + + var previous []*snapshot.Manifest + + previous, finalErr = findPreviousSnapshotManifest(ctx, rep, sourceInfo, nil) + if finalErr != nil { + return finalErr } - policyTree, err := policy.TreeForSource(ctx, rep, sourceInfo) - if err != nil { - return errors.Wrap(err, "unable to get policy tree") + policyTree, finalErr := policy.TreeForSource(ctx, rep, sourceInfo) + if finalErr != nil { + return errors.Wrap(finalErr, "unable to get policy tree") } - manifest, err := u.Upload(ctx, fsEntry, policyTree, sourceInfo, previous...) - if err != nil { + manifest, finalErr := u.Upload(ctx, fsEntry, policyTree, sourceInfo, previous...) + if finalErr != nil { // fail-fast uploads will fail here without recording a manifest, other uploads will // possibly fail later. - return errors.Wrap(err, "upload error") + return errors.Wrap(finalErr, "upload error") } manifest.Description = c.snapshotCreateDescription @@ -308,6 +335,8 @@ func (c *commandSnapshotCreate) snapshotSingleSource(ctx context.Context, fsEntr manifest.EndTime = fs.UTCTimestampFromTime(endTimeOverride) } + mwe.Manifest = *manifest + ignoreIdenticalSnapshot := policyTree.EffectivePolicy().RetentionPolicy.IgnoreIdenticalSnapshots.OrDefault(false) if ignoreIdenticalSnapshot && len(previous) > 0 { if previous[0].RootObjectID() == manifest.RootObjectID() { @@ -316,17 +345,17 @@ func (c *commandSnapshotCreate) snapshotSingleSource(ctx context.Context, fsEntr } } - if _, err = snapshot.SaveSnapshot(ctx, rep, manifest); err != nil { - return errors.Wrap(err, "cannot save manifest") + if _, finalErr = snapshot.SaveSnapshot(ctx, rep, manifest); finalErr != nil { + return errors.Wrap(finalErr, "cannot save manifest") } - if _, err = policy.ApplyRetentionPolicy(ctx, rep, sourceInfo, true); err != nil { - return errors.Wrap(err, "unable to apply retention policy") + if _, finalErr = policy.ApplyRetentionPolicy(ctx, rep, sourceInfo, true); finalErr != nil { + return errors.Wrap(finalErr, "unable to apply retention policy") } if setManual { - if err = policy.SetManual(ctx, rep, sourceInfo); err != nil { - return errors.Wrap(err, "unable to set manual field in scheduling policy for source") + if finalErr = policy.SetManual(ctx, rep, sourceInfo); finalErr != nil { + return errors.Wrap(finalErr, "unable to set manual field in scheduling policy for source") } } diff --git a/internal/server/grpc_session.go b/internal/server/grpc_session.go index 3779ea26b..e532b2886 100644 --- a/internal/server/grpc_session.go +++ b/internal/server/grpc_session.go @@ -132,7 +132,7 @@ func (s *Server) Session(srv grpcapi.KopiaRepository_SessionServer) error { go func() { defer s.grpcServerState.sem.Release(1) - handleSessionRequest(ctx, dw, authz, usernameAtHostname, req, func(resp *grpcapi.SessionResponse) { + s.handleSessionRequest(ctx, dw, authz, usernameAtHostname, req, func(resp *grpcapi.SessionResponse) { if err := s.send(srv, req.GetRequestId(), resp); err != nil { select { case lastErr <- err: @@ -149,7 +149,7 @@ func (s *Server) Session(srv grpcapi.KopiaRepository_SessionServer) error { var tracer = otel.Tracer("kopia/grpc") -func handleSessionRequest(ctx context.Context, dw repo.DirectRepositoryWriter, authz auth.AuthorizationInfo, usernameAtHostname string, req *grpcapi.SessionRequest, respond func(*grpcapi.SessionResponse)) { +func (s *Server) handleSessionRequest(ctx context.Context, dw repo.DirectRepositoryWriter, authz auth.AuthorizationInfo, usernameAtHostname string, req *grpcapi.SessionRequest, respond func(*grpcapi.SessionResponse)) { if req.GetTraceContext() != nil { var tc propagation.TraceContext ctx = tc.Extract(ctx, propagation.MapCarrier(req.GetTraceContext())) @@ -187,7 +187,7 @@ func handleSessionRequest(ctx context.Context, dw repo.DirectRepositoryWriter, a respond(handleApplyRetentionPolicyRequest(ctx, dw, authz, usernameAtHostname, inner.ApplyRetentionPolicy)) case *grpcapi.SessionRequest_SendNotification: - respond(handleSendNotificationRequest(ctx, dw, authz, inner.SendNotification)) + respond(s.handleSendNotificationRequest(ctx, dw, authz, inner.SendNotification)) case *grpcapi.SessionRequest_InitializeSession: respond(errorResponse(errors.New("InitializeSession must be the first request in a session"))) @@ -488,7 +488,7 @@ func handleApplyRetentionPolicyRequest(ctx context.Context, rep repo.RepositoryW } } -func handleSendNotificationRequest(ctx context.Context, rep repo.RepositoryWriter, authz auth.AuthorizationInfo, req *grpcapi.SendNotificationRequest) *grpcapi.SessionResponse { +func (s *Server) handleSendNotificationRequest(ctx context.Context, rep repo.RepositoryWriter, authz auth.AuthorizationInfo, req *grpcapi.SendNotificationRequest) *grpcapi.SessionResponse { ctx, span := tracer.Start(ctx, "GRPCSession.SendNotification") defer span.End() @@ -499,7 +499,8 @@ func handleSendNotificationRequest(ctx context.Context, rep repo.RepositoryWrite if err := notification.SendInternal(ctx, rep, req.GetTemplateName(), json.RawMessage(req.GetEventArgs()), - notification.Severity(req.GetSeverity())); err != nil { + notification.Severity(req.GetSeverity()), + s.options.NotifyTemplateOptions); err != nil { return errorResponse(err) } diff --git a/internal/server/server.go b/internal/server/server.go index a0e4e5169..8a6423b86 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -26,6 +26,7 @@ "github.com/kopia/kopia/internal/scheduler" "github.com/kopia/kopia/internal/serverapi" "github.com/kopia/kopia/internal/uitask" + "github.com/kopia/kopia/notification/notifytemplate" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/logging" "github.com/kopia/kopia/repo/maintenance" @@ -535,6 +536,14 @@ func (s *Server) endUpload(ctx context.Context, src snapshot.SourceInfo) { s.parallelSnapshotsChanged.Signal() } +func (s *Server) enableErrorNotifications() bool { + return s.options.EnableErrorNotifications +} + +func (s *Server) notificationTemplateOptions() notifytemplate.Options { + return s.options.NotifyTemplateOptions +} + // SetRepository sets the repository (nil is allowed and indicates server that is not // connected to the repository). func (s *Server) SetRepository(ctx context.Context, rep repo.Repository) error { @@ -785,23 +794,25 @@ func (s *Server) ServeStaticFiles(m *mux.Router, fs http.FileSystem) { // Options encompasses all API server options. type Options struct { - ConfigFile string - ConnectOptions *repo.ConnectOptions - RefreshInterval time.Duration - MaxConcurrency int - Authenticator auth.Authenticator - Authorizer auth.Authorizer - PasswordPersist passwordpersist.Strategy - AuthCookieSigningKey string - LogRequests bool - UIUser string // name of the user allowed to access the UI API - UIPreferencesFile string // name of the JSON file storing UI preferences - ServerControlUser string // name of the user allowed to access the server control API - DisableCSRFTokenChecks bool - PersistentLogs bool - UITitlePrefix string - DebugScheduler bool - MinMaintenanceInterval time.Duration + ConfigFile string + ConnectOptions *repo.ConnectOptions + RefreshInterval time.Duration + MaxConcurrency int + Authenticator auth.Authenticator + Authorizer auth.Authorizer + PasswordPersist passwordpersist.Strategy + AuthCookieSigningKey string + LogRequests bool + UIUser string // name of the user allowed to access the UI API + UIPreferencesFile string // name of the JSON file storing UI preferences + ServerControlUser string // name of the user allowed to access the server control API + DisableCSRFTokenChecks bool + PersistentLogs bool + UITitlePrefix string + DebugScheduler bool + MinMaintenanceInterval time.Duration + EnableErrorNotifications bool + NotifyTemplateOptions notifytemplate.Options } // InitRepositoryFunc is a function that attempts to connect to/open repository. diff --git a/internal/server/server_maintenance.go b/internal/server/server_maintenance.go index 8faedbcda..1703d1e8e 100644 --- a/internal/server/server_maintenance.go +++ b/internal/server/server_maintenance.go @@ -8,6 +8,9 @@ "github.com/pkg/errors" "github.com/kopia/kopia/internal/clock" + "github.com/kopia/kopia/notification" + "github.com/kopia/kopia/notification/notifydata" + "github.com/kopia/kopia/notification/notifytemplate" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/maintenance" ) @@ -32,6 +35,8 @@ type srvMaintenance struct { type maintenanceManagerServerInterface interface { runMaintenanceTask(ctx context.Context, dr repo.DirectRepository) error refreshScheduler(reason string) + enableErrorNotifications() bool + notificationTemplateOptions() notifytemplate.Options } func (s *srvMaintenance) trigger() { @@ -138,9 +143,21 @@ func startMaintenanceManager( m.beforeRun() + t0 := clock.Now() + if err := srv.runMaintenanceTask(mctx, rep); err != nil { log(ctx).Debugw("maintenance task failed", "err", err) m.afterFailedRun() + + if srv.enableErrorNotifications() { + notification.Send(ctx, + rep, + "generic-error", + notifydata.NewErrorInfo("Maintenance", "Scheduled Maintenance", t0, clock.Now(), err), + notification.SeverityError, + srv.notificationTemplateOptions(), + ) + } } m.refresh(mctx, true) diff --git a/internal/server/server_maintenance_test.go b/internal/server/server_maintenance_test.go index cade06394..1d4785493 100644 --- a/internal/server/server_maintenance_test.go +++ b/internal/server/server_maintenance_test.go @@ -12,6 +12,7 @@ "github.com/kopia/kopia/internal/clock" "github.com/kopia/kopia/internal/repotesting" + "github.com/kopia/kopia/notification/notifytemplate" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/maintenance" ) @@ -47,6 +48,14 @@ func (s *testServer) refreshScheduler(reason string) { s.refreshSchedulerCount.Add(1) } +func (s *testServer) enableErrorNotifications() bool { + return false +} + +func (s *testServer) notificationTemplateOptions() notifytemplate.Options { + return notifytemplate.DefaultOptions +} + func TestServerMaintenance(t *testing.T) { ctx, env := repotesting.NewEnvironment(t, repotesting.FormatNotImportant) diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 36182e990..4a6a84025 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -290,7 +290,7 @@ func remoteRepositoryNotificationTest(t *testing.T, ctx context.Context, rep rep })) require.NoError(t, rw.Flush(ctx)) - notification.Send(ctx, rep, notifytemplate.TestNotification, nil, notification.SeverityError) + notification.Send(ctx, rep, notifytemplate.TestNotification, nil, notification.SeverityError, notifytemplate.DefaultOptions) require.Equal(t, int32(1), numRequestsReceived.Load()) // another webhook which fails @@ -307,7 +307,7 @@ func remoteRepositoryNotificationTest(t *testing.T, ctx context.Context, rep rep })) require.NoError(t, rw.Flush(ctx)) - notification.Send(ctx, rep, notifytemplate.TestNotification, nil, notification.SeverityError) + notification.Send(ctx, rep, notifytemplate.TestNotification, nil, notification.SeverityError, notifytemplate.DefaultOptions) require.Equal(t, int32(1), numRequestsReceived.Load()) } diff --git a/notification/notification_send.go b/notification/notification_send.go index 1a4c2b920..200f34d19 100644 --- a/notification/notification_send.go +++ b/notification/notification_send.go @@ -105,7 +105,7 @@ func notificationSendersFromRepo(ctx context.Context, rep repo.Repository, sever // Send sends a notification for the given event. // Any errors encountered during the process are logged. -func Send(ctx context.Context, rep repo.Repository, templateName string, eventArgs any, sev Severity) { +func Send(ctx context.Context, rep repo.Repository, templateName string, eventArgs any, sev Severity, opt notifytemplate.Options) { // if we're connected to a repository server, send the notification there. if rem, ok := rep.(repo.RemoteNotifications); ok { jsonData, err := json.Marshal(eventArgs) @@ -122,13 +122,13 @@ func Send(ctx context.Context, rep repo.Repository, templateName string, eventAr return } - if err := SendInternal(ctx, rep, templateName, eventArgs, sev); err != nil { + if err := SendInternal(ctx, rep, templateName, eventArgs, sev, opt); err != nil { log(ctx).Warnw("unable to send notification", "err", err) } } // SendInternal sends a notification for the given event and returns an error. -func SendInternal(ctx context.Context, rep repo.Repository, templateName string, eventArgs any, sev Severity) error { +func SendInternal(ctx context.Context, rep repo.Repository, templateName string, eventArgs any, sev Severity, opt notifytemplate.Options) error { senders, err := notificationSendersFromRepo(ctx, rep, sev) if err != nil { return errors.Wrap(err, "unable to get notification senders") @@ -137,7 +137,7 @@ func SendInternal(ctx context.Context, rep repo.Repository, templateName string, var resultErr error for _, s := range senders { - if err := SendTo(ctx, rep, s, templateName, eventArgs, sev); err != nil { + if err := SendTo(ctx, rep, s, templateName, eventArgs, sev, opt); err != nil { resultErr = multierr.Append(resultErr, err) } } @@ -166,7 +166,7 @@ func MakeTemplateArgs(eventArgs any) TemplateArgs { } // SendTo sends a notification to the given sender. -func SendTo(ctx context.Context, rep repo.Repository, s sender.Sender, templateName string, eventArgs any, sev Severity) error { +func SendTo(ctx context.Context, rep repo.Repository, s sender.Sender, templateName string, eventArgs any, sev Severity, opt notifytemplate.Options) error { // execute template var bodyBuf bytes.Buffer @@ -175,7 +175,7 @@ func SendTo(ctx context.Context, rep repo.Repository, s sender.Sender, templateN return errors.Wrap(err, "unable to resolve notification template") } - t, err := notifytemplate.ParseTemplate(tmpl) + t, err := notifytemplate.ParseTemplate(tmpl, opt) if err != nil { return errors.Wrap(err, "unable to parse notification template") } @@ -205,5 +205,5 @@ func SendTo(ctx context.Context, rep repo.Repository, s sender.Sender, templateN func SendTestNotification(ctx context.Context, rep repo.Repository, s sender.Sender) error { log(ctx).Infof("Sending test notification to %v", s.Summary()) - return SendTo(ctx, rep, s, notifytemplate.TestNotification, struct{}{}, SeveritySuccess) + return SendTo(ctx, rep, s, notifytemplate.TestNotification, struct{}{}, SeveritySuccess, notifytemplate.DefaultOptions) } diff --git a/notification/notifydata/doc.go b/notification/notifydata/doc.go new file mode 100644 index 000000000..26449422e --- /dev/null +++ b/notification/notifydata/doc.go @@ -0,0 +1,2 @@ +// Package notifydata contains the data structures used by the notification package. +package notifydata diff --git a/notification/notifydata/error_info.go b/notification/notifydata/error_info.go new file mode 100644 index 000000000..4002a0614 --- /dev/null +++ b/notification/notifydata/error_info.go @@ -0,0 +1,43 @@ +package notifydata + +import ( + "fmt" + "time" +) + +// ErrorInfo represents information about errors. +type ErrorInfo struct { + Operation string `json:"operation"` + OperationDetails string `json:"operationDetails"` + StartTime time.Time `json:"start"` + EndTime time.Time `json:"end"` + ErrorMessage string `json:"error"` + ErrorDetails string `json:"errorDetails"` +} + +// StartTimestamp returns the start time of the operation that caused the error. +func (e *ErrorInfo) StartTimestamp() time.Time { + return e.StartTime.Truncate(time.Second) +} + +// EndTimestamp returns the end time of the operation that caused the error. +func (e *ErrorInfo) EndTimestamp() time.Time { + return e.EndTime.Truncate(time.Second) +} + +// Duration returns the duration of the operation. +func (e *ErrorInfo) Duration() time.Duration { + return e.EndTimestamp().Sub(e.StartTimestamp()) +} + +// NewErrorInfo creates a new ErrorInfo. +func NewErrorInfo(operation, operationDetails string, startTime, endTime time.Time, err error) *ErrorInfo { + return &ErrorInfo{ + Operation: operation, + OperationDetails: operationDetails, + StartTime: startTime, + EndTime: endTime, + ErrorMessage: fmt.Sprintf("%v", err), + ErrorDetails: fmt.Sprintf("%+v", err), + } +} diff --git a/notification/notifydata/multi_snapshot_status.go b/notification/notifydata/multi_snapshot_status.go new file mode 100644 index 000000000..4543ea89b --- /dev/null +++ b/notification/notifydata/multi_snapshot_status.go @@ -0,0 +1,96 @@ +package notifydata + +import ( + "time" + + "github.com/kopia/kopia/snapshot" +) + +// ManifestWithError represents information about the snapshot manifest with optional error. +type ManifestWithError struct { + snapshot.Manifest `json:"manifest"` // may not be filled out if there was an error, Manifst.Source is always set. + + Error string `json:"error"` // will be present if there was an error +} + +// StartTimestamp returns the start time of the snapshot. +func (m *ManifestWithError) StartTimestamp() time.Time { + return m.StartTime.ToTime().UTC().Truncate(time.Second) +} + +// EndTimestamp returns the end time of the snapshot. +func (m *ManifestWithError) EndTimestamp() time.Time { + return m.EndTime.ToTime().UTC().Truncate(time.Second) +} + +// TotalSize returns the total size of the snapshot in bytes. +func (m *ManifestWithError) TotalSize() int64 { + if m.RootEntry == nil { + return 0 + } + + if m.RootEntry.DirSummary != nil { + return m.RootEntry.DirSummary.TotalFileSize + } + + return m.RootEntry.FileSize +} + +// TotalFiles returns the total number of files in the snapshot. +func (m *ManifestWithError) TotalFiles() int64 { + if m.RootEntry == nil { + return 0 + } + + if m.RootEntry.DirSummary != nil { + return m.RootEntry.DirSummary.TotalFileCount + } + + return 1 +} + +// TotalDirs returns the total number of directories in the snapshot. +func (m *ManifestWithError) TotalDirs() int64 { + if m.RootEntry == nil { + return 0 + } + + if m.RootEntry.DirSummary != nil { + return m.RootEntry.DirSummary.TotalDirCount + } + + return 0 +} + +// Duration returns the duration of the snapshot. +func (m *ManifestWithError) Duration() time.Duration { + return time.Duration(m.EndTime - m.StartTime) +} + +// StatusCode returns the status code of the manifest. +func (m *ManifestWithError) StatusCode() string { + if m.Error != "" { + return "fatal" + } + + if m.Manifest.IncompleteReason != "" { + return "incomplete" + } + + if m.Manifest.RootEntry != nil && m.Manifest.RootEntry.DirSummary != nil { + if m.Manifest.RootEntry.DirSummary.FatalErrorCount > 0 { + return "fatal" + } + + if m.Manifest.RootEntry.DirSummary.IgnoredErrorCount > 0 { + return "error" + } + } + + return "ok" +} + +// MultiSnapshotStatus represents the status of multiple snapshots. +type MultiSnapshotStatus struct { + Snapshots []*ManifestWithError `json:"snapshots"` +} diff --git a/notification/notifytemplate/embeddedtemplate.go b/notification/notifytemplate/embeddedtemplate.go index 9e3841f83..0c24cb128 100644 --- a/notification/notifytemplate/embeddedtemplate.go +++ b/notification/notifytemplate/embeddedtemplate.go @@ -3,12 +3,15 @@ import ( "embed" + "slices" + "sort" "text/template" "time" "github.com/pkg/errors" - "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/internal/units" + "github.com/kopia/kopia/notification/notifydata" ) //go:embed "*.html" @@ -20,22 +23,41 @@ TestNotification = "test-notification" ) -// Functions is a map of functions that can be used in templates. +// Options provides options for template rendering. +type Options struct { + Timezone *time.Location + TimeFormat string +} + +// functions is a map of functions that can be used in templates. +func functions(opt Options) template.FuncMap { + if opt.Timezone == nil { + opt.Timezone = time.Local + } + + if opt.TimeFormat == "" { + opt.TimeFormat = time.RFC1123Z + } + + return template.FuncMap{ + "bytes": units.BytesString[int64], + "sortSnapshotManifestsByName": func(man []*notifydata.ManifestWithError) []*notifydata.ManifestWithError { + res := slices.Clone(man) + sort.Slice(res, func(i, j int) bool { + return res[i].Source.String() < res[j].Source.String() + }) + return res + }, + "formatTime": func(t time.Time) string { + return t.In(opt.Timezone).Format(opt.TimeFormat) + }, + } +} + +// DefaultOptions is the default set of options. // //nolint:gochecknoglobals -var Functions = template.FuncMap{ - "toTime": func(t any) time.Time { - if t, ok := t.(time.Time); ok { - return t - } - - if t, ok := t.(fs.UTCTimestamp); ok { - return t.ToTime() - } - - return time.Time{} - }, -} +var DefaultOptions = Options{} // GetEmbeddedTemplate returns embedded template by name. func GetEmbeddedTemplate(templateName string) (string, error) { @@ -61,7 +83,7 @@ func SupportedTemplates() []string { } // ParseTemplate parses a named template. -func ParseTemplate(tmpl string) (*template.Template, error) { +func ParseTemplate(tmpl string, opt Options) (*template.Template, error) { //nolint:wrapcheck - return template.New("template").Funcs(Functions).Parse(tmpl) + return template.New("template").Funcs(functions(opt)).Parse(tmpl) } diff --git a/notification/notifytemplate/generic-error.html b/notification/notifytemplate/generic-error.html new file mode 100644 index 000000000..7287c11aa --- /dev/null +++ b/notification/notifytemplate/generic-error.html @@ -0,0 +1,20 @@ +Subject: Kopia has encountered an error during {{ .EventArgs.Operation }} on {{.Hostname}} + + + + + + + +

Operation: {{ .EventArgs.OperationDetails }}

+

Started: {{ .EventArgs.StartTimestamp | formatTime }}

+

Finished: {{ .EventArgs.EndTimestamp | formatTime }} ({{ .EventArgs.Duration }})

+ +

Message: {{ .EventArgs.ErrorMessage }}

+ +
{{ .EventArgs.ErrorDetails }}
+ +

Generated at {{ .EventTime | formatTime }} by Kopia {{ .KopiaBuildVersion }}.

+ + + diff --git a/notification/notifytemplate/generic-error.txt b/notification/notifytemplate/generic-error.txt new file mode 100644 index 000000000..eea8ace72 --- /dev/null +++ b/notification/notifytemplate/generic-error.txt @@ -0,0 +1,11 @@ +Subject: Kopia has encountered an error during {{ .EventArgs.Operation }} on {{.Hostname}} + +Operation: {{ .EventArgs.OperationDetails }} +Started: {{ .EventArgs.StartTimestamp | formatTime }} +Finished: {{ .EventArgs.EndTimestamp | formatTime }} ({{ .EventArgs.Duration }}) + +{{ .EventArgs.ErrorDetails }} + +Generated at {{ .EventTime | formatTime }} by Kopia {{ .KopiaBuildVersion }}. + +https://kopia.io/ \ No newline at end of file diff --git a/notification/notifytemplate/notifytemplate_test.go b/notification/notifytemplate/notifytemplate_test.go new file mode 100644 index 000000000..182c98fbc --- /dev/null +++ b/notification/notifytemplate/notifytemplate_test.go @@ -0,0 +1,118 @@ +package notifytemplate_test + +import ( + "bytes" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/notification" + "github.com/kopia/kopia/notification/notifydata" + "github.com/kopia/kopia/notification/notifytemplate" + "github.com/kopia/kopia/snapshot" +) + +var defaultTestOptions = notifytemplate.Options{ + Timezone: time.UTC, +} + +var altTestOptions = notifytemplate.Options{ + Timezone: time.FixedZone("PST", -8*60*60), + TimeFormat: time.RFC1123, +} + +func TestNotifyTemplate_generic_error(t *testing.T) { + args := notification.MakeTemplateArgs(¬ifydata.ErrorInfo{ + Operation: "Some Operation", + OperationDetails: "Some Operation Details", + ErrorMessage: "error message", + ErrorDetails: "error details", + StartTime: time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC), + EndTime: time.Date(2020, 1, 2, 3, 4, 6, 7, time.UTC), + }) + + args.EventTime = time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC) + args.Hostname = "some-host" + + verifyTemplate(t, "generic-error.txt", ".default", args, defaultTestOptions) + verifyTemplate(t, "generic-error.html", ".default", args, defaultTestOptions) + verifyTemplate(t, "generic-error.txt", ".alt", args, altTestOptions) + verifyTemplate(t, "generic-error.html", ".alt", args, altTestOptions) +} + +func TestNotifyTemplate_snapshot_report(t *testing.T) { + args := notification.MakeTemplateArgs(¬ifydata.MultiSnapshotStatus{ + Snapshots: []*notifydata.ManifestWithError{ + { + Manifest: snapshot.Manifest{ + Source: snapshot.SourceInfo{Host: "some-host", UserName: "some-user", Path: "/some/path"}, + StartTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC).UnixNano()), + EndTime: fs.UTCTimestamp(time.Date(2020, 1, 2, 3, 4, 6, 7, time.UTC).UnixNano()), + RootEntry: &snapshot.DirEntry{ + DirSummary: &fs.DirectorySummary{ + TotalFileCount: 123, + TotalFileSize: 456, + TotalDirCount: 33, + FailedEntries: []*fs.EntryWithError{ + { + EntryPath: "/some/path", + Error: "some error", + }, + { + EntryPath: "/some/path2", + Error: "some error", + }, + }, + }, + }, + }, + }, + { + Error: "some top-level error", + Manifest: snapshot.Manifest{ + Source: snapshot.SourceInfo{Host: "some-host", UserName: "some-user", Path: "/some/other/path"}, + }, + }, + }, + }) + + args.EventTime = time.Date(2020, 1, 2, 3, 4, 5, 6, time.UTC) + args.Hostname = "some-host" + + verifyTemplate(t, "snapshot-report.txt", ".default", args, defaultTestOptions) + verifyTemplate(t, "snapshot-report.html", ".default", args, defaultTestOptions) + verifyTemplate(t, "snapshot-report.txt", ".alt", args, altTestOptions) + verifyTemplate(t, "snapshot-report.html", ".alt", args, altTestOptions) +} + +func verifyTemplate(t *testing.T, embeddedTemplateName, expectedSuffix string, args interface{}, opt notifytemplate.Options) { + t.Helper() + + tmpl, err := notifytemplate.GetEmbeddedTemplate(embeddedTemplateName) + require.NoError(t, err) + + tt, err := notifytemplate.ParseTemplate(tmpl, opt) + require.NoError(t, err) + + var buf bytes.Buffer + + require.NoError(t, tt.Execute(&buf, args)) + + actualFileName := filepath.Join("testdata", embeddedTemplateName+expectedSuffix+".actual") + require.NoError(t, os.WriteFile(actualFileName, buf.Bytes(), 0o644)) + + expectedFileName := filepath.Join("testdata", embeddedTemplateName+expectedSuffix+".expected") + + wantBytes, err := os.ReadFile(expectedFileName) + require.NoError(t, err) + + want := string(wantBytes) + + require.Equal(t, want, buf.String()) + + require.NoError(t, os.Remove(actualFileName)) +} diff --git a/notification/notifytemplate/snapshot-report.html b/notification/notifytemplate/snapshot-report.html new file mode 100644 index 000000000..272d43301 --- /dev/null +++ b/notification/notifytemplate/snapshot-report.html @@ -0,0 +1,103 @@ +Subject: Kopia created {{ len .EventArgs.Snapshots }} snapshot{{ if gt (len .EventArgs.Snapshots) 1 }}s{{end}} on {{.Hostname}} + + + + + + + + + + + + + + + + + + +{{ range .EventArgs.Snapshots | sortSnapshotManifestsByName}} + + + + + + + + + +{{ if .Error }} + + + +{{ end }} + +{{ if .RootEntry }} +{{ if .RootEntry.DirSummary }} +{{ if .RootEntry.DirSummary.FailedEntries }} + + + +{{ end }} +{{ end }} +{{ end }} + +{{ end }} +
SourceStartedDurationTotal SizeTotal FilesTotal Directories
{{ .Source.Path }}{{ .StartTimestamp | formatTime }}{{ .Duration }}{{ .TotalSize | bytes }}{{ .TotalFiles }}{{ .TotalDirs }}
+ Error: {{ .Error }} +
+ Failed Entries: +
    + {{ range .RootEntry.DirSummary.FailedEntries }} +
  • {{.EntryPath}}: {{.Error}}
    • + {{ end }} +
+
+ +

Generated at {{ .EventTime | formatTime }} by Kopia {{ .KopiaBuildVersion }}.

+ + + diff --git a/notification/notifytemplate/snapshot-report.txt b/notification/notifytemplate/snapshot-report.txt new file mode 100644 index 000000000..212c85d0f --- /dev/null +++ b/notification/notifytemplate/snapshot-report.txt @@ -0,0 +1,16 @@ +Subject: Kopia created {{ len .EventArgs.Snapshots }} snapshot{{ if gt (len .EventArgs.Snapshots) 1 }}s{{end}} on {{.Hostname}} + +{{ range .EventArgs.Snapshots | sortSnapshotManifestsByName}}Path: {{ .Source.Path }} + + Status: {{ .StatusCode }} + Start: {{ .StartTimestamp | formatTime }} Duration: {{ .Duration }} + Size: {{ .TotalSize | bytes }}, {{ .TotalFiles }} files, {{ .TotalDirs }} directories. +{{ if .Error }} Error: {{ .Error }} +{{ end }}{{ if .RootEntry }}{{ if .RootEntry.DirSummary }}{{ if .RootEntry.DirSummary.FailedEntries }} + Failed Entries: +{{ range .RootEntry.DirSummary.FailedEntries }} + - {{.EntryPath}}: {{.Error}}{{ end }}{{ end }}{{ end }} +{{ end }} +{{ end }}Generated at {{ .EventTime | formatTime }} by Kopia {{ .KopiaBuildVersion }}. + +https://kopia.io/ \ No newline at end of file diff --git a/notification/notifytemplate/test-notification.html b/notification/notifytemplate/test-notification.html index d8823c651..a9aaf6ea3 100644 --- a/notification/notifytemplate/test-notification.html +++ b/notification/notifytemplate/test-notification.html @@ -1,4 +1,4 @@ -Subject: Test notification from Kopia at {{ .EventTime }} +Subject: Test notification from Kopia at {{ .EventTime | formatTime }}

This is a test notification from Kopia.

diff --git a/notification/notifytemplate/test-notification.txt b/notification/notifytemplate/test-notification.txt index e364e2691..6624b03e7 100644 --- a/notification/notifytemplate/test-notification.txt +++ b/notification/notifytemplate/test-notification.txt @@ -1,4 +1,4 @@ -Subject: Test notification from Kopia at {{ .EventTime }} +Subject: Test notification from Kopia at {{ .EventTime | formatTime }} This is a test notification from Kopia. diff --git a/notification/notifytemplate/testdata/.gitignore b/notification/notifytemplate/testdata/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/notification/notifytemplate/testdata/generic-error.html.alt.expected b/notification/notifytemplate/testdata/generic-error.html.alt.expected new file mode 100644 index 000000000..0d8f0013f --- /dev/null +++ b/notification/notifytemplate/testdata/generic-error.html.alt.expected @@ -0,0 +1,20 @@ +Subject: Kopia has encountered an error during Some Operation on some-host + + + + + + + +

Operation: Some Operation Details

+

Started: Wed, 01 Jan 2020 19:04:05 PST

+

Finished: Wed, 01 Jan 2020 19:04:06 PST (1s)

+ +

Message: error message

+ +
error details
+ +

Generated at Wed, 01 Jan 2020 19:04:05 PST by Kopia v0-unofficial.

+ + + diff --git a/notification/notifytemplate/testdata/generic-error.html.default.expected b/notification/notifytemplate/testdata/generic-error.html.default.expected new file mode 100644 index 000000000..9faf8d9da --- /dev/null +++ b/notification/notifytemplate/testdata/generic-error.html.default.expected @@ -0,0 +1,20 @@ +Subject: Kopia has encountered an error during Some Operation on some-host + + + + + + + +

Operation: Some Operation Details

+

Started: Thu, 02 Jan 2020 03:04:05 +0000

+

Finished: Thu, 02 Jan 2020 03:04:06 +0000 (1s)

+ +

Message: error message

+ +
error details
+ +

Generated at Thu, 02 Jan 2020 03:04:05 +0000 by Kopia v0-unofficial.

+ + + diff --git a/notification/notifytemplate/testdata/generic-error.txt.alt.expected b/notification/notifytemplate/testdata/generic-error.txt.alt.expected new file mode 100644 index 000000000..170cdcdc3 --- /dev/null +++ b/notification/notifytemplate/testdata/generic-error.txt.alt.expected @@ -0,0 +1,11 @@ +Subject: Kopia has encountered an error during Some Operation on some-host + +Operation: Some Operation Details +Started: Wed, 01 Jan 2020 19:04:05 PST +Finished: Wed, 01 Jan 2020 19:04:06 PST (1s) + +error details + +Generated at Wed, 01 Jan 2020 19:04:05 PST by Kopia v0-unofficial. + +https://kopia.io/ \ No newline at end of file diff --git a/notification/notifytemplate/testdata/generic-error.txt.default.expected b/notification/notifytemplate/testdata/generic-error.txt.default.expected new file mode 100644 index 000000000..4f03e75b1 --- /dev/null +++ b/notification/notifytemplate/testdata/generic-error.txt.default.expected @@ -0,0 +1,11 @@ +Subject: Kopia has encountered an error during Some Operation on some-host + +Operation: Some Operation Details +Started: Thu, 02 Jan 2020 03:04:05 +0000 +Finished: Thu, 02 Jan 2020 03:04:06 +0000 (1s) + +error details + +Generated at Thu, 02 Jan 2020 03:04:05 +0000 by Kopia v0-unofficial. + +https://kopia.io/ \ No newline at end of file diff --git a/notification/notifytemplate/testdata/snapshot-report.html.alt.expected b/notification/notifytemplate/testdata/snapshot-report.html.alt.expected new file mode 100644 index 000000000..11647e19f --- /dev/null +++ b/notification/notifytemplate/testdata/snapshot-report.html.alt.expected @@ -0,0 +1,119 @@ +Subject: Kopia created 2 snapshots on some-host + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
SourceStartedDurationTotal SizeTotal FilesTotal Directories
/some/other/pathWed, 31 Dec 1969 16:00:00 PST0s0 B00
+ Error: some top-level error +
/some/pathWed, 01 Jan 2020 19:04:05 PST1.000000001s456 B12333
+ Failed Entries: +
    + +
  • /some/path: some error
    • + +
  • /some/path2: some error
    • + +
+
+ +

Generated at Wed, 01 Jan 2020 19:04:05 PST by Kopia v0-unofficial.

+ + + diff --git a/notification/notifytemplate/testdata/snapshot-report.html.default.expected b/notification/notifytemplate/testdata/snapshot-report.html.default.expected new file mode 100644 index 000000000..5cf75fb2d --- /dev/null +++ b/notification/notifytemplate/testdata/snapshot-report.html.default.expected @@ -0,0 +1,119 @@ +Subject: Kopia created 2 snapshots on some-host + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
SourceStartedDurationTotal SizeTotal FilesTotal Directories
/some/other/pathThu, 01 Jan 1970 00:00:00 +00000s0 B00
+ Error: some top-level error +
/some/pathThu, 02 Jan 2020 03:04:05 +00001.000000001s456 B12333
+ Failed Entries: +
    + +
  • /some/path: some error
    • + +
  • /some/path2: some error
    • + +
+
+ +

Generated at Thu, 02 Jan 2020 03:04:05 +0000 by Kopia v0-unofficial.

+ + + diff --git a/notification/notifytemplate/testdata/snapshot-report.txt.alt.expected b/notification/notifytemplate/testdata/snapshot-report.txt.alt.expected new file mode 100644 index 000000000..46de1240c --- /dev/null +++ b/notification/notifytemplate/testdata/snapshot-report.txt.alt.expected @@ -0,0 +1,23 @@ +Subject: Kopia created 2 snapshots on some-host + +Path: /some/other/path + + Status: fatal + Start: Wed, 31 Dec 1969 16:00:00 PST Duration: 0s + Size: 0 B, 0 files, 0 directories. + Error: some top-level error + +Path: /some/path + + Status: ok + Start: Wed, 01 Jan 2020 19:04:05 PST Duration: 1.000000001s + Size: 456 B, 123 files, 33 directories. + + Failed Entries: + + - /some/path: some error + - /some/path2: some error + +Generated at Wed, 01 Jan 2020 19:04:05 PST by Kopia v0-unofficial. + +https://kopia.io/ \ No newline at end of file diff --git a/notification/notifytemplate/testdata/snapshot-report.txt.default.expected b/notification/notifytemplate/testdata/snapshot-report.txt.default.expected new file mode 100644 index 000000000..b11a4e06d --- /dev/null +++ b/notification/notifytemplate/testdata/snapshot-report.txt.default.expected @@ -0,0 +1,23 @@ +Subject: Kopia created 2 snapshots on some-host + +Path: /some/other/path + + Status: fatal + Start: Thu, 01 Jan 1970 00:00:00 +0000 Duration: 0s + Size: 0 B, 0 files, 0 directories. + Error: some top-level error + +Path: /some/path + + Status: ok + Start: Thu, 02 Jan 2020 03:04:05 +0000 Duration: 1.000000001s + Size: 456 B, 123 files, 33 directories. + + Failed Entries: + + - /some/path: some error + - /some/path2: some error + +Generated at Thu, 02 Jan 2020 03:04:05 +0000 by Kopia v0-unofficial. + +https://kopia.io/ \ No newline at end of file