From de856dab99ef8816392972347678fcb49ae57e50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Rod=C3=A1k?= Date: Tue, 10 Sep 2024 15:53:30 +0200 Subject: [PATCH] Add --health-max-log-count, --health-max-log-size, --health-log-destination flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These flags can affect the output of the HealtCheck log. Currently, when a container is configured with HealthCheck, the output from the HealthCheck command is only logged to the container status file, which is accessible via `podman inspect`. It is also limited to the last five executions and the first 500 characters per execution. This makes debugging past problems very difficult, since the only information available about the failure of the HealthCheck command is the generic `healthcheck service failed` record. - The `--health-log-destination` flag sets the destination of the HealthCheck log. - `none`: (default behavior) `HealthCheckResults` are stored in overlay containers. (For example: `$runroot/healthcheck.log`) - `directory`: creates a log file named `-healthcheck.log` with JSON `HealthCheckResults` in the specified directory. - `events_logger`: The log will be written with logging mechanism set by events_loggeri. It also saves the log to a default directory, for performance on a system with a large number of logs. - The `--health-max-log-count` flag sets the maximum number of attempts in the HealthCheck log file. - A value of `0` indicates an infinite number of attempts in the log file. - The default value is `5` attempts in the log file. - The `--health-max-log-size` flag sets the maximum length of the log stored. - A value of `0` indicates an infinite log length. - The default value is `500` log characters. Add --health-max-log-count flag Signed-off-by: Jan Rodák Add --health-max-log-size flag Signed-off-by: Jan Rodák Add --health-log-destination flag Signed-off-by: Jan Rodák --- cmd/podman/common/create.go | 24 ++ cmd/podman/common/create_opts.go | 3 + .../options/health-log-destination.md | 11 + .../markdown/options/health-max-log-count.md | 7 + .../markdown/options/health-max-log-size.md | 7 + docs/source/markdown/podman-create.1.md.in | 6 + docs/source/markdown/podman-run.1.md.in | 6 + docs/source/markdown/podman-systemd.unit.5.md | 25 ++ libpod/container_config.go | 8 + libpod/container_inspect.go | 8 +- libpod/container_internal.go | 7 +- libpod/define/container_inspect.go | 8 + libpod/define/healthchecks.go | 10 + libpod/events.go | 21 +- libpod/events/config.go | 4 + libpod/events/events.go | 4 +- libpod/events/journal_linux.go | 18 +- libpod/healthcheck.go | 85 +++--- libpod/options.go | 53 ++++ libpod/runtime_ctr.go | 2 +- pkg/api/handlers/compat/containers_create.go | 111 ++++---- pkg/api/handlers/libpod/containers_create.go | 4 + pkg/domain/entities/pods.go | 261 +++++++++--------- pkg/domain/infra/abi/containers.go | 4 + pkg/domain/infra/abi/play.go | 7 +- pkg/specgen/generate/container.go | 4 + pkg/specgen/generate/container_create.go | 4 + pkg/specgen/generate/kube/kube.go | 3 + pkg/specgen/generate/pod_create.go | 6 + pkg/specgen/specgen.go | 22 +- pkg/specgenutil/specgen.go | 6 + pkg/systemd/quadlet/quadlet.go | 9 + test/system/220-healthcheck.bats | 181 ++++++++++++ 33 files changed, 701 insertions(+), 238 deletions(-) create mode 100644 docs/source/markdown/options/health-log-destination.md create mode 100644 docs/source/markdown/options/health-max-log-count.md create mode 100644 docs/source/markdown/options/health-max-log-size.md diff --git a/cmd/podman/common/create.go b/cmd/podman/common/create.go index 6de40987ee..54a2e781f5 100644 --- a/cmd/podman/common/create.go +++ b/cmd/podman/common/create.go @@ -184,6 +184,30 @@ func DefineCreateFlags(cmd *cobra.Command, cf *entities.ContainerCreateOptions, ) _ = cmd.RegisterFlagCompletionFunc(healthIntervalFlagName, completion.AutocompleteNone) + healthLogDestinationFlagName := "health-log-destination" + createFlags.StringVar( + &cf.HealthLogDestination, + healthLogDestinationFlagName, define.DefaultHealthCheckLocalDestination, + "set the destination of the HealthCheck log. Directory path, local or events_logger (local use container state file)", + ) + _ = cmd.RegisterFlagCompletionFunc(healthLogDestinationFlagName, completion.AutocompleteNone) + + healthMaxLogCountFlagName := "health-max-log-count" + createFlags.UintVar( + &cf.HealthMaxLogCount, + healthMaxLogCountFlagName, define.DefaultHealthMaxLogCount, + "set maximum number of attempts in the HealthCheck log file. ('0' value means an infinite number of attempts in the log file)", + ) + _ = cmd.RegisterFlagCompletionFunc(healthMaxLogCountFlagName, completion.AutocompleteNone) + + healthMaxLogSizeFlagName := "health-max-log-size" + createFlags.UintVar( + &cf.HealthMaxLogSize, + healthMaxLogSizeFlagName, define.DefaultHealthMaxLogSize, + "set maximum length in characters of stored HealthCheck log. ('0' value means an infinite log length)", + ) + _ = cmd.RegisterFlagCompletionFunc(healthMaxLogSizeFlagName, completion.AutocompleteNone) + healthRetriesFlagName := "health-retries" createFlags.UintVar( &cf.HealthRetries, diff --git a/cmd/podman/common/create_opts.go b/cmd/podman/common/create_opts.go index cdcd86514d..fb297c044c 100644 --- a/cmd/podman/common/create_opts.go +++ b/cmd/podman/common/create_opts.go @@ -85,4 +85,7 @@ func DefineCreateDefaults(opts *entities.ContainerCreateOptions) { opts.Ulimit = ulimits() opts.SeccompPolicy = "default" opts.Volume = volumes() + opts.HealthLogDestination = define.DefaultHealthCheckLocalDestination + opts.HealthMaxLogCount = define.DefaultHealthMaxLogCount + opts.HealthMaxLogSize = define.DefaultHealthMaxLogSize } diff --git a/docs/source/markdown/options/health-log-destination.md b/docs/source/markdown/options/health-log-destination.md new file mode 100644 index 0000000000..16b99ecc4c --- /dev/null +++ b/docs/source/markdown/options/health-log-destination.md @@ -0,0 +1,11 @@ +####> This option file is used in: +####> podman create, run +####> If file is edited, make sure the changes +####> are applicable to all of those. +#### **--health-log-destination**=*directory_path* + +Set the destination of the HealthCheck log. Directory path, local or events_logger (local use container state file) (Default: local) + +* `local`: (default) HealthCheck logs are stored in overlay containers. (For example: `$runroot/healthcheck.log`) +* `directory`: creates a log file named `-healthcheck.log` with HealthCheck logs in the specified directory. +* `events_logger`: The log will be written with logging mechanism set by events_logger. It also saves the log to a default directory, for performance on a system with a large number of logs. diff --git a/docs/source/markdown/options/health-max-log-count.md b/docs/source/markdown/options/health-max-log-count.md new file mode 100644 index 0000000000..96a7d60861 --- /dev/null +++ b/docs/source/markdown/options/health-max-log-count.md @@ -0,0 +1,7 @@ +####> This option file is used in: +####> podman create, run +####> If file is edited, make sure the changes +####> are applicable to all of those. +#### **--health-max-log-count**=*number of stored logs* + +Set maximum number of attempts in the HealthCheck log file. ('0' value means an infinite number of attempts in the log file) (Default: 5 attempts) diff --git a/docs/source/markdown/options/health-max-log-size.md b/docs/source/markdown/options/health-max-log-size.md new file mode 100644 index 0000000000..96cc399e4a --- /dev/null +++ b/docs/source/markdown/options/health-max-log-size.md @@ -0,0 +1,7 @@ +####> This option file is used in: +####> podman create, run +####> If file is edited, make sure the changes +####> are applicable to all of those. +#### **--health-max-log-size**=*size of stored logs* + +Set maximum length in characters of stored HealthCheck log. ("0" value means an infinite log length) (Default: 500 characters) diff --git a/docs/source/markdown/podman-create.1.md.in b/docs/source/markdown/podman-create.1.md.in index d0a939e873..ed5852f8f8 100644 --- a/docs/source/markdown/podman-create.1.md.in +++ b/docs/source/markdown/podman-create.1.md.in @@ -169,6 +169,12 @@ See [**Environment**](#environment) note below for precedence and examples. @@option health-interval +@@option health-log-destination + +@@option health-max-log-count + +@@option health-max-log-size + @@option health-on-failure @@option health-retries diff --git a/docs/source/markdown/podman-run.1.md.in b/docs/source/markdown/podman-run.1.md.in index da546a3354..6c131b3684 100644 --- a/docs/source/markdown/podman-run.1.md.in +++ b/docs/source/markdown/podman-run.1.md.in @@ -203,6 +203,12 @@ See [**Environment**](#environment) note below for precedence and examples. @@option health-interval +@@option health-log-destination + +@@option health-max-log-count + +@@option health-max-log-size + @@option health-on-failure @@option health-retries diff --git a/docs/source/markdown/podman-systemd.unit.5.md b/docs/source/markdown/podman-systemd.unit.5.md index 846fdb97ba..976f062dca 100644 --- a/docs/source/markdown/podman-systemd.unit.5.md +++ b/docs/source/markdown/podman-systemd.unit.5.md @@ -279,6 +279,9 @@ Valid options for `[Container]` are listed below: | GroupAdd=keep-groups | --group-add=keep-groups | | HealthCmd=/usr/bin/command | --health-cmd=/usr/bin/command | | HealthInterval=2m | --health-interval=2m | +| HealthLogDestination=/foo/log | --health-log-destination=/foo/log | +| HealthMaxLogCount=5 | --health-max-log-count=5 | +| HealthMaxLogSize=500 | --health-max-log-size=500 | | HealthOnFailure=kill | --health-on-failure=kill | | HealthRetries=5 | --health-retries=5 | | HealthStartPeriod=1m | --health-start-period=period=1m | @@ -515,6 +518,28 @@ Equivalent to the Podman `--health-cmd` option. Set an interval for the healthchecks. An interval of disable results in no automatic timer setup. Equivalent to the Podman `--health-interval` option. +### `HealthLogDestination=` + +Set the destination of the HealthCheck log. Directory path, local or events_logger (local use container state file) +(Default: local) +Equivalent to the Podman `--health-log-destination` option. + +* `local`: (default) HealthCheck logs are stored in overlay containers. (For example: `$runroot/healthcheck.log`) +* `directory`: creates a log file named `-healthcheck.log` with HealthCheck logs in the specified directory. +* `events_logger`: The log will be written with logging mechanism set by events_logger. It also saves the log to a default directory, for performance on a system with a large number of logs. + +### `HealthMaxLogCount=` + +Set maximum number of attempts in the HealthCheck log file. ('0' value means an infinite number of attempts in the log file) +(Default: 5 attempts) +Equivalent to the Podman `--Health-max-log-count` option. + +### `HealthMaxLogSize=` + +Set maximum length in characters of stored HealthCheck log. ("0" value means an infinite log length) +(Default: 500 characters) +Equivalent to the Podman `--Health-max-log-size` option. + ### `HealthOnFailure=` Action to take once the container transitions to an unhealthy state. diff --git a/libpod/container_config.go b/libpod/container_config.go index 8c4e0176c5..5ed80382a0 100644 --- a/libpod/container_config.go +++ b/libpod/container_config.go @@ -413,6 +413,14 @@ type ContainerMiscConfig struct { HealthCheckConfig *manifest.Schema2HealthConfig `json:"healthcheck"` // HealthCheckOnFailureAction defines an action to take once the container turns unhealthy. HealthCheckOnFailureAction define.HealthCheckOnFailureAction `json:"healthcheck_on_failure_action"` + // HealthLogDestination defines the destination where the log is stored + HealthLogDestination string `json:"healthLogDestination,omitempty"` + // HealthMaxLogCount is maximum number of attempts in the HealthCheck log file. + // ('0' value means an infinite number of attempts in the log file) + HealthMaxLogCount uint `json:"healthMaxLogCount,omitempty"` + // HealthMaxLogSize is the maximum length in characters of stored HealthCheck log + // ("0" value means an infinite log length) + HealthMaxLogSize uint `json:"healthMaxLogSize,omitempty"` // StartupHealthCheckConfig is the configuration of the startup // healthcheck for the container. This will run before the regular HC // runs, and when it passes the regular HC will be activated. diff --git a/libpod/container_inspect.go b/libpod/container_inspect.go index 33bd465da0..55abe11840 100644 --- a/libpod/container_inspect.go +++ b/libpod/container_inspect.go @@ -195,7 +195,7 @@ func (c *Container) getContainerInspectData(size bool, driverData *define.Driver // inspect status should be set to nil. if c.config.HealthCheckConfig != nil && !(len(c.config.HealthCheckConfig.Test) == 1 && c.config.HealthCheckConfig.Test[0] == "NONE") { // This container has a healthcheck defined in it; we need to add its state - healthCheckState, err := c.getHealthCheckLog() + healthCheckState, err := c.readHealthCheckLog() if err != nil { // An error here is not considered fatal; no health state will be displayed logrus.Error(err) @@ -426,6 +426,12 @@ func (c *Container) generateInspectContainerConfig(spec *spec.Spec) *define.Insp ctrConfig.HealthcheckOnFailureAction = c.config.HealthCheckOnFailureAction.String() + ctrConfig.HealthLogDestination = c.config.HealthLogDestination + + ctrConfig.HealthMaxLogCount = c.config.HealthMaxLogCount + + ctrConfig.HealthMaxLogSize = c.config.HealthMaxLogSize + ctrConfig.CreateCommand = c.config.CreateCommand ctrConfig.Timezone = c.config.Timezone diff --git a/libpod/container_internal.go b/libpod/container_internal.go index 2839e02433..c7efd18e4b 100644 --- a/libpod/container_internal.go +++ b/libpod/container_internal.go @@ -1123,10 +1123,9 @@ func (c *Container) init(ctx context.Context, retainRetries bool) error { // bugzilla.redhat.com/show_bug.cgi?id=2144754: // In case of a restart, make sure to remove the healthcheck log to // have a clean state. - if path := c.healthCheckLogPath(); path != "" { - if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { - logrus.Error(err) - } + err = c.writeHealthCheckLog(define.HealthCheckResults{Status: define.HealthCheckReset}) + if err != nil { + return err } if err := c.save(); err != nil { diff --git a/libpod/define/container_inspect.go b/libpod/define/container_inspect.go index 89b70e59e6..4be6fd913c 100644 --- a/libpod/define/container_inspect.go +++ b/libpod/define/container_inspect.go @@ -61,6 +61,14 @@ type InspectContainerConfig struct { Healthcheck *manifest.Schema2HealthConfig `json:"Healthcheck,omitempty"` // HealthcheckOnFailureAction defines an action to take once the container turns unhealthy. HealthcheckOnFailureAction string `json:"HealthcheckOnFailureAction,omitempty"` + // HealthLogDestination defines the destination where the log is stored + HealthLogDestination string `json:"HealthLogDestination,omitempty"` + // HealthMaxLogCount is maximum number of attempts in the HealthCheck log file. + // ('0' value means an infinite number of attempts in the log file) + HealthMaxLogCount uint `json:"HealthcheckMaxLogCount,omitempty"` + // HealthMaxLogSize is the maximum length in characters of stored HealthCheck log + // ("0" value means an infinite log length) + HealthMaxLogSize uint `json:"HealthcheckMaxLogSize,omitempty"` // CreateCommand is the full command plus arguments of the process the // container has been created with. CreateCommand []string `json:"CreateCommand,omitempty"` diff --git a/libpod/define/healthchecks.go b/libpod/define/healthchecks.go index 15ea79fc20..4fec277555 100644 --- a/libpod/define/healthchecks.go +++ b/libpod/define/healthchecks.go @@ -16,6 +16,8 @@ const ( // and the start-period (time allowed for the container to start and application // to be running) expires. HealthCheckStarting string = "starting" + // HealthCheckReset describes reset of HealthCheck logs + HealthCheckReset string = "reset" ) // HealthCheckStatus represents the current state of a container @@ -56,8 +58,16 @@ const ( DefaultHealthCheckStartPeriod = "0s" // DefaultHealthCheckTimeout default value DefaultHealthCheckTimeout = "30s" + // DefaultHealthMaxLogCount default value + DefaultHealthMaxLogCount uint = 5 + // DefaultHealthMaxLogSize default value + DefaultHealthMaxLogSize uint = 500 + // DefaultHealthCheckLocalDestination default value + DefaultHealthCheckLocalDestination string = "local" ) +const HealthCheckEventsLoggerDestination string = "events_logger" + // HealthConfig.Test options const ( // HealthConfigTestNone disables healthcheck diff --git a/libpod/events.go b/libpod/events.go index 92af63632c..6189ee895a 100644 --- a/libpod/events.go +++ b/libpod/events.go @@ -8,6 +8,7 @@ import ( "path/filepath" "sync" + "github.com/containers/podman/v5/libpod/define" "github.com/containers/podman/v5/libpod/events" "github.com/sirupsen/logrus" ) @@ -28,27 +29,37 @@ func (r *Runtime) newEventer() (events.Eventer, error) { // newContainerEvent creates a new event based on a container func (c *Container) newContainerEvent(status events.Status) { - if err := c.newContainerEventWithInspectData(status, "", false); err != nil { + if err := c.newContainerEventWithInspectData(status, define.HealthCheckResults{}, false); err != nil { logrus.Errorf("Unable to write container event: %v", err) } } // newContainerHealthCheckEvent creates a new healthcheck event with the given status -func (c *Container) newContainerHealthCheckEvent(healthStatus string) { - if err := c.newContainerEventWithInspectData(events.HealthStatus, healthStatus, false); err != nil { +func (c *Container) newContainerHealthCheckEvent(healthCheckResult define.HealthCheckResults) { + if err := c.newContainerEventWithInspectData(events.HealthStatus, healthCheckResult, false); err != nil { logrus.Errorf("Unable to write container event: %v", err) } } // newContainerEventWithInspectData creates a new event and sets the // ContainerInspectData field if inspectData is set. -func (c *Container) newContainerEventWithInspectData(status events.Status, healthStatus string, inspectData bool) error { +func (c *Container) newContainerEventWithInspectData(status events.Status, healthCheckResult define.HealthCheckResults, inspectData bool) error { e := events.NewEvent(status) e.ID = c.ID() e.Name = c.Name() e.Image = c.config.RootfsImageName e.Type = events.Container - e.HealthStatus = healthStatus + e.HealthStatus = healthCheckResult.Status + if c.config.HealthLogDestination == define.HealthCheckEventsLoggerDestination { + if len(healthCheckResult.Log) > 0 { + logData, err := json.Marshal(healthCheckResult.Log[len(healthCheckResult.Log)-1]) + if err != nil { + return fmt.Errorf("unable to marshall healthcheck log for writing: %w", err) + } + e.HealthLog = string(logData) + } + } + e.HealthFailingStreak = healthCheckResult.FailingStreak e.Details = events.Details{ PodID: c.PodID(), diff --git a/libpod/events/config.go b/libpod/events/config.go index d0ab5d45f0..1927c5a6d3 100644 --- a/libpod/events/config.go +++ b/libpod/events/config.go @@ -41,6 +41,10 @@ type Event struct { Type Type // Health status of the current container HealthStatus string `json:"health_status,omitempty"` + // Healthcheck log of the current container + HealthLog string `json:"health_log,omitempty"` + // HealthFailingStreak log of the current container + HealthFailingStreak int `json:"health_failing_streak,omitempty"` // Error code for certain events involving errors. Error string `json:"error,omitempty"` diff --git a/libpod/events/events.go b/libpod/events/events.go index 5eda0033cc..084be84cab 100644 --- a/libpod/events/events.go +++ b/libpod/events/events.go @@ -76,8 +76,10 @@ func (e *Event) ToHumanReadable(truncate bool) string { if e.PodID != "" { humanFormat += fmt.Sprintf(", pod_id=%s", e.PodID) } - if e.HealthStatus != "" { + if e.Status == HealthStatus { humanFormat += fmt.Sprintf(", health_status=%s", e.HealthStatus) + humanFormat += fmt.Sprintf(", health_failing_streak=%d", e.HealthFailingStreak) + humanFormat += fmt.Sprintf(", health_log=%s", e.HealthLog) } // check if the container has labels and add it to the output if len(e.Attributes) > 0 { diff --git a/libpod/events/journal_linux.go b/libpod/events/journal_linux.go index 2ee94090f8..fe23a40379 100644 --- a/libpod/events/journal_linux.go +++ b/libpod/events/journal_linux.go @@ -65,8 +65,13 @@ func (e EventJournalD) Write(ee Event) error { } m["PODMAN_LABELS"] = string(b) } - m["PODMAN_HEALTH_STATUS"] = ee.HealthStatus - + if ee.Status == HealthStatus { + m["PODMAN_HEALTH_STATUS"] = ee.HealthStatus + if ee.HealthLog != "" { + m["PODMAN_HEALTH_LOG"] = ee.HealthLog + } + m["PODMAN_HEALTH_FAILING_STREAK"] = strconv.Itoa(ee.HealthFailingStreak) + } if len(ee.Details.ContainerInspectData) > 0 { m["PODMAN_CONTAINER_INSPECT_DATA"] = ee.Details.ContainerInspectData } @@ -225,6 +230,15 @@ func newEventFromJournalEntry(entry *sdjournal.JournalEntry) (*Event, error) { } } newEvent.HealthStatus = entry.Fields["PODMAN_HEALTH_STATUS"] + if log, ok := entry.Fields["PODMAN_HEALTH_LOG"]; ok { + newEvent.HealthLog = log + } + if FailingStreak, ok := entry.Fields["PODMAN_HEALTH_FAILING_STREAK"]; ok { + FailingStreakInt, err := strconv.Atoi(FailingStreak) + if err == nil { + newEvent.HealthFailingStreak = FailingStreakInt + } + } newEvent.Details.ContainerInspectData = entry.Fields["PODMAN_CONTAINER_INSPECT_DATA"] case Network: newEvent.ID = entry.Fields["PODMAN_ID"] diff --git a/libpod/healthcheck.go b/libpod/healthcheck.go index 95f20f2fd1..96740d2c13 100644 --- a/libpod/healthcheck.go +++ b/libpod/healthcheck.go @@ -19,14 +19,6 @@ import ( "golang.org/x/sys/unix" ) -const ( - // MaxHealthCheckNumberLogs is the maximum number of attempts we keep - // in the healthcheck history file - MaxHealthCheckNumberLogs int = 5 - // MaxHealthCheckLogLength in characters - MaxHealthCheckLogLength = 500 -) - // HealthCheck verifies the state and validity of the healthcheck configuration // on the container and then executes the healthcheck func (r *Runtime) HealthCheck(ctx context.Context, name string) (define.HealthCheckStatus, error) { @@ -143,8 +135,8 @@ func (c *Container) runHealthCheck(ctx context.Context, isStartup bool) (define. } eventLog := output.String() - if len(eventLog) > MaxHealthCheckLogLength { - eventLog = eventLog[:MaxHealthCheckLogLength] + if c.config.HealthMaxLogSize != 0 && len(eventLog) > int(c.config.HealthMaxLogSize) { + eventLog = eventLog[:c.config.HealthMaxLogSize] } if timeEnd.Sub(timeStart) > c.HealthCheckConfig().Timeout { @@ -154,21 +146,22 @@ func (c *Container) runHealthCheck(ctx context.Context, isStartup bool) (define. } hcl := newHealthCheckLog(timeStart, timeEnd, returnCode, eventLog) - logStatus, err := c.updateHealthCheckLog(hcl, inStartPeriod, isStartup) + + healthCheckResult, err := c.updateHealthCheckLog(hcl, inStartPeriod, isStartup) if err != nil { - return hcResult, "", fmt.Errorf("unable to update health check log %s for %s: %w", c.healthCheckLogPath(), c.ID(), err) + return hcResult, "", fmt.Errorf("unable to update health check log %s for %s: %w", c.config.HealthLogDestination, c.ID(), err) } // Write HC event with appropriate status as the last thing before we // return. if hcResult == define.HealthCheckNotDefined || hcResult == define.HealthCheckInternalError { - return hcResult, logStatus, hcErr + return hcResult, healthCheckResult.Status, hcErr } if c.runtime.config.Engine.HealthcheckEvents { - c.newContainerHealthCheckEvent(logStatus) + c.newContainerHealthCheckEvent(healthCheckResult) } - return hcResult, logStatus, hcErr + return hcResult, healthCheckResult.Status, hcErr } func (c *Container) processHealthCheckStatus(status string) error { @@ -340,16 +333,12 @@ func newHealthCheckLog(start, end time.Time, exitCode int, log string) define.He // updateHealthStatus updates the health status of the container // in the healthcheck log func (c *Container) updateHealthStatus(status string) error { - healthCheck, err := c.getHealthCheckLog() + healthCheck, err := c.readHealthCheckLog() if err != nil { return err } healthCheck.Status = status - newResults, err := json.Marshal(healthCheck) - if err != nil { - return fmt.Errorf("unable to marshall healthchecks for writing status: %w", err) - } - return os.WriteFile(c.healthCheckLogPath(), newResults, 0700) + return c.writeHealthCheckLog(healthCheck) } // isUnhealthy returns true if the current health check status is unhealthy. @@ -357,7 +346,7 @@ func (c *Container) isUnhealthy() (bool, error) { if !c.HasHealthCheck() { return false, nil } - healthCheck, err := c.getHealthCheckLog() + healthCheck, err := c.readHealthCheckLog() if err != nil { return false, err } @@ -365,7 +354,7 @@ func (c *Container) isUnhealthy() (bool, error) { } // UpdateHealthCheckLog parses the health check results and writes the log -func (c *Container) updateHealthCheckLog(hcl define.HealthCheckLog, inStartPeriod, isStartup bool) (string, error) { +func (c *Container) updateHealthCheckLog(hcl define.HealthCheckLog, inStartPeriod, isStartup bool) (define.HealthCheckResults, error) { c.lock.Lock() defer c.lock.Unlock() @@ -373,12 +362,12 @@ func (c *Container) updateHealthCheckLog(hcl define.HealthCheckLog, inStartPerio // both failing and succeeding cases to match kube behavior. // So don't update the health check log till the start period is over if _, ok := c.config.Spec.Annotations[define.KubeHealthCheckAnnotation]; ok && inStartPeriod && !isStartup { - return "", nil + return define.HealthCheckResults{}, nil } - healthCheck, err := c.getHealthCheckLog() + healthCheck, err := c.readHealthCheckLog() if err != nil { - return "", err + return define.HealthCheckResults{}, err } if hcl.ExitCode == 0 { // set status to healthy, reset failing state to 0 @@ -398,28 +387,48 @@ func (c *Container) updateHealthCheckLog(hcl define.HealthCheckLog, inStartPerio } } healthCheck.Log = append(healthCheck.Log, hcl) - if len(healthCheck.Log) > MaxHealthCheckNumberLogs { + if c.config.HealthMaxLogCount != 0 && len(healthCheck.Log) > int(c.config.HealthMaxLogCount) { healthCheck.Log = healthCheck.Log[1:] } - newResults, err := json.Marshal(healthCheck) + return healthCheck, c.writeHealthCheckLog(healthCheck) +} + +func (c *Container) witeToFileHealthCheckResults(path string, result define.HealthCheckResults) error { + newResults, err := json.Marshal(result) if err != nil { - return "", fmt.Errorf("unable to marshall healthchecks for writing: %w", err) + return fmt.Errorf("unable to marshall healthchecks for writing: %w", err) } - return healthCheck.Status, os.WriteFile(c.healthCheckLogPath(), newResults, 0700) + return os.WriteFile(path, newResults, 0700) } -// HealthCheckLogPath returns the path for where the health check log is -func (c *Container) healthCheckLogPath() string { - return filepath.Join(filepath.Dir(c.state.RunDir), "healthcheck.log") +func (c *Container) getHealthCheckLogDestination() string { + var destination string + switch c.config.HealthLogDestination { + case define.DefaultHealthCheckLocalDestination, define.HealthCheckEventsLoggerDestination, "": + destination = filepath.Join(filepath.Dir(c.state.RunDir), "healthcheck.log") + default: + destination = filepath.Join(c.config.HealthLogDestination, c.ID()+"-healthcheck.log") + } + return destination } -// getHealthCheckLog returns HealthCheck results by reading the container's +func (c *Container) writeHealthCheckLog(result define.HealthCheckResults) error { + return c.witeToFileHealthCheckResults(c.getHealthCheckLogDestination(), result) +} + +// readHealthCheckLog read HealthCheck logs from the path or events_logger +// The caller should lock the container before this function is called. +func (c *Container) readHealthCheckLog() (define.HealthCheckResults, error) { + return c.readFromFileHealthCheckLog(c.getHealthCheckLogDestination()) +} + +// readFromFileHealthCheckLog returns HealthCheck results by reading the container's // health check log file. If the health check log file does not exist, then // an empty healthcheck struct is returned // The caller should lock the container before this function is called. -func (c *Container) getHealthCheckLog() (define.HealthCheckResults, error) { +func (c *Container) readFromFileHealthCheckLog(path string) (define.HealthCheckResults, error) { var healthCheck define.HealthCheckResults - b, err := os.ReadFile(c.healthCheckLogPath()) + b, err := os.ReadFile(path) if err != nil { if errors.Is(err, fs.ErrNotExist) { // If the file does not exists just return empty healthcheck and no error. @@ -428,7 +437,7 @@ func (c *Container) getHealthCheckLog() (define.HealthCheckResults, error) { return healthCheck, fmt.Errorf("failed to read health check log file: %w", err) } if err := json.Unmarshal(b, &healthCheck); err != nil { - return healthCheck, fmt.Errorf("failed to unmarshal existing healthcheck results in %s: %w", c.healthCheckLogPath(), err) + return healthCheck, fmt.Errorf("failed to unmarshal existing healthcheck results in %s: %w", path, err) } return healthCheck, nil } @@ -454,7 +463,7 @@ func (c *Container) healthCheckStatus() (string, error) { return "", err } - results, err := c.getHealthCheckLog() + results, err := c.readHealthCheckLog() if err != nil { return "", fmt.Errorf("unable to get healthcheck log for %s: %w", c.ID(), err) } diff --git a/libpod/options.go b/libpod/options.go index 9f30f2f325..ca9f999810 100644 --- a/libpod/options.go +++ b/libpod/options.go @@ -6,6 +6,8 @@ import ( "errors" "fmt" "net" + "os" + "path/filepath" "strings" "syscall" "time" @@ -1500,6 +1502,57 @@ func WithHealthCheck(healthCheck *manifest.Schema2HealthConfig) CtrCreateOption } } +// WithHealthCheckLogDestination adds the healthLogDestination to the container config +func WithHealthCheckLogDestination(destination string) CtrCreateOption { + return func(ctr *Container) error { + if ctr.valid { + return define.ErrCtrFinalized + } + switch destination { + case define.HealthCheckEventsLoggerDestination, define.DefaultHealthCheckLocalDestination: + ctr.config.HealthLogDestination = destination + default: + fileInfo, err := os.Stat(destination) + if err != nil { + return fmt.Errorf("HealthCheck Log '%s' destination error: %w", destination, err) + } + mode := fileInfo.Mode() + if !mode.IsDir() { + return fmt.Errorf("HealthCheck Log '%s' destination must be directory", destination) + } + + absPath, err := filepath.Abs(destination) + if err != nil { + return err + } + ctr.config.HealthLogDestination = absPath + } + return nil + } +} + +// WithHealthCheckMaxLogCount adds the healthMaxLogCount to the container config +func WithHealthCheckMaxLogCount(maxLogCount uint) CtrCreateOption { + return func(ctr *Container) error { + if ctr.valid { + return define.ErrCtrFinalized + } + ctr.config.HealthMaxLogCount = maxLogCount + return nil + } +} + +// WithHealthCheckMaxLogSize adds the healthMaxLogSize to the container config +func WithHealthCheckMaxLogSize(maxLogSize uint) CtrCreateOption { + return func(ctr *Container) error { + if ctr.valid { + return define.ErrCtrFinalized + } + ctr.config.HealthMaxLogSize = maxLogSize + return nil + } +} + // WithHealthCheckOnFailureAction adds an on-failure action to health-check config func WithHealthCheckOnFailureAction(action define.HealthCheckOnFailureAction) CtrCreateOption { return func(ctr *Container) error { diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index a3f26cc527..3551613a1f 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -576,7 +576,7 @@ func (r *Runtime) setupContainer(ctx context.Context, ctr *Container) (_ *Contai } if ctr.runtime.config.Engine.EventsContainerCreateInspectData { - if err := ctr.newContainerEventWithInspectData(events.Create, "", true); err != nil { + if err := ctr.newContainerEventWithInspectData(events.Create, define.HealthCheckResults{}, true); err != nil { return nil, err } } else { diff --git a/pkg/api/handlers/compat/containers_create.go b/pkg/api/handlers/compat/containers_create.go index 404c117bb6..14b37804f3 100644 --- a/pkg/api/handlers/compat/containers_create.go +++ b/pkg/api/handlers/compat/containers_create.go @@ -423,60 +423,63 @@ func cliOpts(cc handlers.CreateContainerConfig, rtc *config.Config) (*entities.C CPUSetMems: cc.HostConfig.CpusetMems, // Detach: false, // don't need // DetachKeys: "", // don't need - Devices: devices, - DeviceCgroupRule: cc.HostConfig.DeviceCgroupRules, - DeviceReadBPs: readBps, - DeviceReadIOPs: readIops, - DeviceWriteBPs: writeBps, - DeviceWriteIOPs: writeIops, - Entrypoint: entrypoint, - Env: cc.Config.Env, - Expose: expose, - GroupAdd: cc.HostConfig.GroupAdd, - Hostname: cc.Config.Hostname, - ImageVolume: "anonymous", - Init: init, - Interactive: cc.Config.OpenStdin, - IPC: string(cc.HostConfig.IpcMode), - Label: stringMaptoArray(cc.Config.Labels), - LogDriver: cc.HostConfig.LogConfig.Type, - LogOptions: stringMaptoArray(cc.HostConfig.LogConfig.Config), - Name: cc.Name, - OOMScoreAdj: &cc.HostConfig.OomScoreAdj, - Arch: "", - OS: "", - Variant: "", - PID: string(cc.HostConfig.PidMode), - PIDsLimit: cc.HostConfig.PidsLimit, - Privileged: cc.HostConfig.Privileged, - PublishAll: cc.HostConfig.PublishAllPorts, - Quiet: false, - ReadOnly: cc.HostConfig.ReadonlyRootfs, - ReadWriteTmpFS: true, // podman default - Rm: cc.HostConfig.AutoRemove, - Annotation: stringMaptoArray(cc.HostConfig.Annotations), - SecurityOpt: cc.HostConfig.SecurityOpt, - StopSignal: cc.Config.StopSignal, - StopTimeout: rtc.Engine.StopTimeout, // podman default - StorageOpts: stringMaptoArray(cc.HostConfig.StorageOpt), - Sysctl: stringMaptoArray(cc.HostConfig.Sysctls), - Systemd: "true", // podman default - TmpFS: parsedTmp, - TTY: cc.Config.Tty, - EnvMerge: cc.EnvMerge, - UnsetEnv: cc.UnsetEnv, - UnsetEnvAll: cc.UnsetEnvAll, - User: cc.Config.User, - UserNS: string(cc.HostConfig.UsernsMode), - UTS: string(cc.HostConfig.UTSMode), - Mount: mounts, - VolumesFrom: cc.HostConfig.VolumesFrom, - Workdir: cc.Config.WorkingDir, - Net: &netInfo, - HealthInterval: define.DefaultHealthCheckInterval, - HealthRetries: define.DefaultHealthCheckRetries, - HealthTimeout: define.DefaultHealthCheckTimeout, - HealthStartPeriod: define.DefaultHealthCheckStartPeriod, + Devices: devices, + DeviceCgroupRule: cc.HostConfig.DeviceCgroupRules, + DeviceReadBPs: readBps, + DeviceReadIOPs: readIops, + DeviceWriteBPs: writeBps, + DeviceWriteIOPs: writeIops, + Entrypoint: entrypoint, + Env: cc.Config.Env, + Expose: expose, + GroupAdd: cc.HostConfig.GroupAdd, + Hostname: cc.Config.Hostname, + ImageVolume: "anonymous", + Init: init, + Interactive: cc.Config.OpenStdin, + IPC: string(cc.HostConfig.IpcMode), + Label: stringMaptoArray(cc.Config.Labels), + LogDriver: cc.HostConfig.LogConfig.Type, + LogOptions: stringMaptoArray(cc.HostConfig.LogConfig.Config), + Name: cc.Name, + OOMScoreAdj: &cc.HostConfig.OomScoreAdj, + Arch: "", + OS: "", + Variant: "", + PID: string(cc.HostConfig.PidMode), + PIDsLimit: cc.HostConfig.PidsLimit, + Privileged: cc.HostConfig.Privileged, + PublishAll: cc.HostConfig.PublishAllPorts, + Quiet: false, + ReadOnly: cc.HostConfig.ReadonlyRootfs, + ReadWriteTmpFS: true, // podman default + Rm: cc.HostConfig.AutoRemove, + Annotation: stringMaptoArray(cc.HostConfig.Annotations), + SecurityOpt: cc.HostConfig.SecurityOpt, + StopSignal: cc.Config.StopSignal, + StopTimeout: rtc.Engine.StopTimeout, // podman default + StorageOpts: stringMaptoArray(cc.HostConfig.StorageOpt), + Sysctl: stringMaptoArray(cc.HostConfig.Sysctls), + Systemd: "true", // podman default + TmpFS: parsedTmp, + TTY: cc.Config.Tty, + EnvMerge: cc.EnvMerge, + UnsetEnv: cc.UnsetEnv, + UnsetEnvAll: cc.UnsetEnvAll, + User: cc.Config.User, + UserNS: string(cc.HostConfig.UsernsMode), + UTS: string(cc.HostConfig.UTSMode), + Mount: mounts, + VolumesFrom: cc.HostConfig.VolumesFrom, + Workdir: cc.Config.WorkingDir, + Net: &netInfo, + HealthInterval: define.DefaultHealthCheckInterval, + HealthRetries: define.DefaultHealthCheckRetries, + HealthTimeout: define.DefaultHealthCheckTimeout, + HealthStartPeriod: define.DefaultHealthCheckStartPeriod, + HealthLogDestination: define.DefaultHealthCheckLocalDestination, + HealthMaxLogCount: define.DefaultHealthMaxLogCount, + HealthMaxLogSize: define.DefaultHealthMaxLogSize, } if !rootless.IsRootless() { var ulimits []string diff --git a/pkg/api/handlers/libpod/containers_create.go b/pkg/api/handlers/libpod/containers_create.go index 1a12942226..1fcf1c413d 100644 --- a/pkg/api/handlers/libpod/containers_create.go +++ b/pkg/api/handlers/libpod/containers_create.go @@ -10,6 +10,7 @@ import ( "strconv" "github.com/containers/podman/v5/libpod" + "github.com/containers/podman/v5/libpod/define" "github.com/containers/podman/v5/pkg/api/handlers/utils" api "github.com/containers/podman/v5/pkg/api/types" "github.com/containers/podman/v5/pkg/domain/entities" @@ -42,6 +43,9 @@ func CreateContainer(w http.ResponseWriter, r *http.Request) { Umask: conf.Containers.Umask, Privileged: &privileged, }, + ContainerHealthCheckConfig: specgen.ContainerHealthCheckConfig{ + HealthLogDestination: define.DefaultHealthCheckLocalDestination, + }, } if err := json.NewDecoder(r.Body).Decode(&sg); err != nil { diff --git a/pkg/domain/entities/pods.go b/pkg/domain/entities/pods.go index 63b88e3355..96c4a3bf66 100644 --- a/pkg/domain/entities/pods.go +++ b/pkg/domain/entities/pods.go @@ -134,135 +134,138 @@ const ( ) type ContainerCreateOptions struct { - Annotation []string - Attach []string - Authfile string - BlkIOWeight string - BlkIOWeightDevice []string - CapAdd []string - CapDrop []string - CgroupNS string - CgroupsMode string - CgroupParent string `json:"cgroup_parent,omitempty"` - CIDFile string - ConmonPIDFile string `json:"container_conmon_pidfile,omitempty"` - CPUPeriod uint64 - CPUQuota int64 - CPURTPeriod uint64 - CPURTRuntime int64 - CPUShares uint64 - CPUS float64 `json:"cpus,omitempty"` - CPUSetCPUs string `json:"cpuset_cpus,omitempty"` - CPUSetMems string - Devices []string `json:"devices,omitempty"` - DeviceCgroupRule []string - DeviceReadBPs []string `json:"device_read_bps,omitempty"` - DeviceReadIOPs []string - DeviceWriteBPs []string - DeviceWriteIOPs []string - Entrypoint *string `json:"container_command,omitempty"` - Env []string - EnvHost bool - EnvFile []string - Expose []string - GIDMap []string - GPUs []string - GroupAdd []string - HealthCmd string - HealthInterval string - HealthRetries uint - HealthStartPeriod string - HealthTimeout string - HealthOnFailure string - Hostname string `json:"hostname,omitempty"` - HTTPProxy bool - HostUsers []string - ImageVolume string - Init bool - InitContainerType string - InitPath string - IntelRdtClosID string - Interactive bool - IPC string - Label []string - LabelFile []string - LogDriver string - LogOptions []string - Memory string - MemoryReservation string - MemorySwap string - MemorySwappiness int64 - Name string `json:"container_name"` - NoHealthCheck bool - OOMKillDisable bool - OOMScoreAdj *int - Arch string - OS string - Variant string - PID string `json:"pid,omitempty"` - PIDsLimit *int64 - Platform string - Pod string - PodIDFile string - Personality string - PreserveFDs uint - PreserveFD []uint - Privileged bool - PublishAll bool - Pull string - Quiet bool - ReadOnly bool - ReadWriteTmpFS bool - Restart string - Replace bool - Requires []string - Retry *uint `json:"retry,omitempty"` - RetryDelay string `json:"retry_delay,omitempty"` - Rm bool - RootFS bool - Secrets []string - SecurityOpt []string `json:"security_opt,omitempty"` - SdNotifyMode string - ShmSize string - ShmSizeSystemd string - SignaturePolicy string - StartupHCCmd string - StartupHCInterval string - StartupHCRetries uint - StartupHCSuccesses uint - StartupHCTimeout string - StopSignal string - StopTimeout uint - StorageOpts []string - SubGIDName string - SubUIDName string - Sysctl []string `json:"sysctl,omitempty"` - Systemd string - Timeout uint - TLSVerify commonFlag.OptionalBool - TmpFS []string - TTY bool - Timezone string - Umask string - EnvMerge []string - UnsetEnv []string - UnsetEnvAll bool - UIDMap []string - Ulimit []string - User string - UserNS string `json:"-"` - UTS string - Mount []string - Volume []string `json:"volume,omitempty"` - VolumesFrom []string `json:"volumes_from,omitempty"` - Workdir string - SeccompPolicy string - PidFile string - ChrootDirs []string - IsInfra bool - IsClone bool - DecryptionKeys []string - Net *NetOptions `json:"net,omitempty"` + Annotation []string + Attach []string + Authfile string + BlkIOWeight string + BlkIOWeightDevice []string + CapAdd []string + CapDrop []string + CgroupNS string + CgroupsMode string + CgroupParent string `json:"cgroup_parent,omitempty"` + CIDFile string + ConmonPIDFile string `json:"container_conmon_pidfile,omitempty"` + CPUPeriod uint64 + CPUQuota int64 + CPURTPeriod uint64 + CPURTRuntime int64 + CPUShares uint64 + CPUS float64 `json:"cpus,omitempty"` + CPUSetCPUs string `json:"cpuset_cpus,omitempty"` + CPUSetMems string + Devices []string `json:"devices,omitempty"` + DeviceCgroupRule []string + DeviceReadBPs []string `json:"device_read_bps,omitempty"` + DeviceReadIOPs []string + DeviceWriteBPs []string + DeviceWriteIOPs []string + Entrypoint *string `json:"container_command,omitempty"` + Env []string + EnvHost bool + EnvFile []string + Expose []string + GIDMap []string + GPUs []string + GroupAdd []string + HealthCmd string + HealthInterval string + HealthRetries uint + HealthLogDestination string + HealthMaxLogCount uint + HealthMaxLogSize uint + HealthStartPeriod string + HealthTimeout string + HealthOnFailure string + Hostname string `json:"hostname,omitempty"` + HTTPProxy bool + HostUsers []string + ImageVolume string + Init bool + InitContainerType string + InitPath string + IntelRdtClosID string + Interactive bool + IPC string + Label []string + LabelFile []string + LogDriver string + LogOptions []string + Memory string + MemoryReservation string + MemorySwap string + MemorySwappiness int64 + Name string `json:"container_name"` + NoHealthCheck bool + OOMKillDisable bool + OOMScoreAdj *int + Arch string + OS string + Variant string + PID string `json:"pid,omitempty"` + PIDsLimit *int64 + Platform string + Pod string + PodIDFile string + Personality string + PreserveFDs uint + PreserveFD []uint + Privileged bool + PublishAll bool + Pull string + Quiet bool + ReadOnly bool + ReadWriteTmpFS bool + Restart string + Replace bool + Requires []string + Retry *uint `json:"retry,omitempty"` + RetryDelay string `json:"retry_delay,omitempty"` + Rm bool + RootFS bool + Secrets []string + SecurityOpt []string `json:"security_opt,omitempty"` + SdNotifyMode string + ShmSize string + ShmSizeSystemd string + SignaturePolicy string + StartupHCCmd string + StartupHCInterval string + StartupHCRetries uint + StartupHCSuccesses uint + StartupHCTimeout string + StopSignal string + StopTimeout uint + StorageOpts []string + SubGIDName string + SubUIDName string + Sysctl []string `json:"sysctl,omitempty"` + Systemd string + Timeout uint + TLSVerify commonFlag.OptionalBool + TmpFS []string + TTY bool + Timezone string + Umask string + EnvMerge []string + UnsetEnv []string + UnsetEnvAll bool + UIDMap []string + Ulimit []string + User string + UserNS string `json:"-"` + UTS string + Mount []string + Volume []string `json:"volume,omitempty"` + VolumesFrom []string `json:"volumes_from,omitempty"` + Workdir string + SeccompPolicy string + PidFile string + ChrootDirs []string + IsInfra bool + IsClone bool + DecryptionKeys []string + Net *NetOptions `json:"net,omitempty"` CgroupConf []string diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go index f20c412858..6c9a820996 100644 --- a/pkg/domain/infra/abi/containers.go +++ b/pkg/domain/infra/abi/containers.go @@ -1758,6 +1758,10 @@ func (ic *ContainerEngine) ContainerClone(ctx context.Context, ctrCloneOpts enti spec.Name = generate.CheckName(ic.Libpod, n, true) } + spec.HealthLogDestination = define.DefaultHealthCheckLocalDestination + spec.HealthMaxLogCount = define.DefaultHealthMaxLogCount + spec.HealthMaxLogSize = define.DefaultHealthMaxLogSize + rtSpec, spec, opts, err := generate.MakeContainer(context.Background(), ic.Libpod, spec, true, c) if err != nil { return nil, err diff --git a/pkg/domain/infra/abi/play.go b/pkg/domain/infra/abi/play.go index 609ec03d85..8319fd9ed1 100644 --- a/pkg/domain/infra/abi/play.go +++ b/pkg/domain/infra/abi/play.go @@ -84,8 +84,11 @@ func (ic *ContainerEngine) createServiceContainer(ctx context.Context, name stri ReadOnly: true, ReadWriteTmpFS: false, // No need to spin up slirp etc. - Net: &entities.NetOptions{Network: specgen.Namespace{NSMode: specgen.NoNetwork}}, - StopTimeout: rtc.Engine.StopTimeout, + Net: &entities.NetOptions{Network: specgen.Namespace{NSMode: specgen.NoNetwork}}, + StopTimeout: rtc.Engine.StopTimeout, + HealthLogDestination: define.DefaultHealthCheckLocalDestination, + HealthMaxLogCount: define.DefaultHealthMaxLogCount, + HealthMaxLogSize: define.DefaultHealthMaxLogSize, } // Create and fill out the runtime spec. diff --git a/pkg/specgen/generate/container.go b/pkg/specgen/generate/container.go index f224453da7..ef9aed24a8 100644 --- a/pkg/specgen/generate/container.go +++ b/pkg/specgen/generate/container.go @@ -444,6 +444,10 @@ func ConfigToSpec(rt *libpod.Runtime, specg *specgen.SpecGenerator, containerID } } + specg.HealthLogDestination = conf.HealthLogDestination + specg.HealthMaxLogCount = conf.HealthMaxLogCount + specg.HealthMaxLogSize = conf.HealthMaxLogSize + specg.IDMappings = &conf.IDMappings specg.ContainerCreateCommand = conf.CreateCommand if len(specg.Rootfs) == 0 { diff --git a/pkg/specgen/generate/container_create.go b/pkg/specgen/generate/container_create.go index 8d4029114b..ca77218945 100644 --- a/pkg/specgen/generate/container_create.go +++ b/pkg/specgen/generate/container_create.go @@ -642,6 +642,10 @@ func createContainerOptions(rt *libpod.Runtime, s *specgen.SpecGenerator, pod *l options = append(options, libpod.WithHealthCheckOnFailureAction(s.ContainerHealthCheckConfig.HealthCheckOnFailureAction)) } + options = append(options, libpod.WithHealthCheckLogDestination(s.ContainerHealthCheckConfig.HealthLogDestination)) + options = append(options, libpod.WithHealthCheckMaxLogCount(s.ContainerHealthCheckConfig.HealthMaxLogCount)) + options = append(options, libpod.WithHealthCheckMaxLogSize(s.ContainerHealthCheckConfig.HealthMaxLogSize)) + if s.SdNotifyMode == define.SdNotifyModeHealthy && !healthCheckSet { return nil, fmt.Errorf("%w: sdnotify policy %q requires a healthcheck to be set", define.ErrInvalidArg, s.SdNotifyMode) } diff --git a/pkg/specgen/generate/kube/kube.go b/pkg/specgen/generate/kube/kube.go index c44a3c8344..9f532d3ec1 100644 --- a/pkg/specgen/generate/kube/kube.go +++ b/pkg/specgen/generate/kube/kube.go @@ -438,6 +438,9 @@ func ToSpecGen(ctx context.Context, opts *CtrSpecGenOptions) (*specgen.SpecGener } s.Annotations[define.KubeHealthCheckAnnotation] = "true" + s.HealthLogDestination = define.DefaultHealthCheckLocalDestination + s.HealthMaxLogCount = define.DefaultHealthMaxLogCount + s.HealthMaxLogSize = define.DefaultHealthMaxLogSize // Environment Variables envs := map[string]string{} diff --git a/pkg/specgen/generate/pod_create.go b/pkg/specgen/generate/pod_create.go index c4c8dc4511..9b01a747e0 100644 --- a/pkg/specgen/generate/pod_create.go +++ b/pkg/specgen/generate/pod_create.go @@ -85,6 +85,12 @@ func MakePod(p *entities.PodSpec, rt *libpod.Runtime) (_ *libpod.Pod, finalErr e // make sure of that here. p.PodSpecGen.InfraContainerSpec.ResourceLimits = nil p.PodSpecGen.InfraContainerSpec.WeightDevice = nil + + // Set default for HealthCheck + p.PodSpecGen.InfraContainerSpec.HealthLogDestination = define.DefaultHealthCheckLocalDestination + p.PodSpecGen.InfraContainerSpec.HealthMaxLogCount = define.DefaultHealthMaxLogCount + p.PodSpecGen.InfraContainerSpec.HealthMaxLogSize = define.DefaultHealthMaxLogSize + rtSpec, spec, opts, err := MakeContainer(context.Background(), rt, p.PodSpecGen.InfraContainerSpec, false, nil) if err != nil { return nil, err diff --git a/pkg/specgen/specgen.go b/pkg/specgen/specgen.go index ab4aeb7aec..eca50d6cf7 100644 --- a/pkg/specgen/specgen.go +++ b/pkg/specgen/specgen.go @@ -599,6 +599,14 @@ type ContainerHealthCheckConfig struct { // Requires that HealthConfig be set. // Optional. StartupHealthConfig *define.StartupHealthCheck `json:"startupHealthConfig,omitempty"` + // HealthLogDestination defines the destination where the log is stored + HealthLogDestination string `json:"healthLogDestination,omitempty"` + // HealthMaxLogCount is maximum number of attempts in the HealthCheck log file. + // ('0' value means an infinite number of attempts in the log file) + HealthMaxLogCount uint `json:"healthMaxLogCount,omitempty"` + // HealthMaxLogSize is the maximum length in characters of stored HealthCheck log + // ("0" value means an infinite log length) + HealthMaxLogSize uint `json:"healthMaxLogSize,omitempty"` } // SpecGenerator creates an OCI spec and Libpod configuration options to create @@ -671,13 +679,25 @@ func NewSpecGenerator(arg string, rootfs bool) *SpecGenerator { } return &SpecGenerator{ ContainerStorageConfig: csc, + ContainerHealthCheckConfig: ContainerHealthCheckConfig{ + HealthLogDestination: define.DefaultHealthCheckLocalDestination, + HealthMaxLogCount: define.DefaultHealthMaxLogCount, + HealthMaxLogSize: define.DefaultHealthMaxLogSize, + }, } } // NewSpecGenerator returns a SpecGenerator struct given one of two mandatory inputs func NewSpecGeneratorWithRootfs(rootfs string) *SpecGenerator { csc := ContainerStorageConfig{Rootfs: rootfs} - return &SpecGenerator{ContainerStorageConfig: csc} + return &SpecGenerator{ + ContainerStorageConfig: csc, + ContainerHealthCheckConfig: ContainerHealthCheckConfig{ + HealthLogDestination: define.DefaultHealthCheckLocalDestination, + HealthMaxLogCount: define.DefaultHealthMaxLogCount, + HealthMaxLogSize: define.DefaultHealthMaxLogSize, + }, + } } func StringSlicesEqual(a, b []string) bool { diff --git a/pkg/specgenutil/specgen.go b/pkg/specgenutil/specgen.go index 6cb1f154d5..0bc9b419a7 100644 --- a/pkg/specgenutil/specgen.go +++ b/pkg/specgenutil/specgen.go @@ -370,6 +370,12 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions } s.HealthCheckOnFailureAction = onFailureAction + s.HealthLogDestination = c.HealthLogDestination + + s.HealthMaxLogCount = c.HealthMaxLogCount + + s.HealthMaxLogSize = c.HealthMaxLogSize + if c.StartupHCCmd != "" { if c.NoHealthCheck { return errors.New("cannot specify both --no-healthcheck and --health-startup-cmd") diff --git a/pkg/systemd/quadlet/quadlet.go b/pkg/systemd/quadlet/quadlet.go index 092256ec35..ca3773da70 100644 --- a/pkg/systemd/quadlet/quadlet.go +++ b/pkg/systemd/quadlet/quadlet.go @@ -93,6 +93,9 @@ const ( KeyGroupAdd = "GroupAdd" KeyHealthCmd = "HealthCmd" KeyHealthInterval = "HealthInterval" + KeyHealthLogDestination = "HealthLogDestination" + KeyHealthMaxLogCount = "HealthMaxLogCount" + KeyHealthMaxLogSize = "HealthMaxLogSize" KeyHealthOnFailure = "HealthOnFailure" KeyHealthRetries = "HealthRetries" KeyHealthStartPeriod = "HealthStartPeriod" @@ -214,6 +217,9 @@ var ( KeyHealthCmd: true, KeyHealthInterval: true, KeyHealthOnFailure: true, + KeyHealthLogDestination: true, + KeyHealthMaxLogCount: true, + KeyHealthMaxLogSize: true, KeyHealthRetries: true, KeyHealthStartPeriod: true, KeyHealthStartupCmd: true, @@ -2065,6 +2071,9 @@ func handleHealth(unitFile *parser.UnitFile, groupName string, podman *PodmanCmd {KeyHealthCmd, "cmd"}, {KeyHealthInterval, "interval"}, {KeyHealthOnFailure, "on-failure"}, + {KeyHealthLogDestination, "log-destination"}, + {KeyHealthMaxLogCount, "max-log-count"}, + {KeyHealthMaxLogSize, "max-log-size"}, {KeyHealthRetries, "retries"}, {KeyHealthStartPeriod, "start-period"}, {KeyHealthTimeout, "timeout"}, diff --git a/test/system/220-healthcheck.bats b/test/system/220-healthcheck.bats index 1cf1f9c0b3..43c8e1d012 100644 --- a/test/system/220-healthcheck.bats +++ b/test/system/220-healthcheck.bats @@ -273,4 +273,185 @@ Log[-1].Output | \"Uh-oh on stdout!\\\nUh-oh on stderr!\\\n\" done } +function _create_container_with_health_log_settings { + local ctrname="$1" + local msg="$2" + local format="$3" + local flag="$4" + local expect="$5" + local expect_msg="$6" + + run_podman run -d --name $ctrname \ + --health-cmd "echo $msg" \ + $flag \ + $IMAGE /home/podman/pause + cid="$output" + + run_podman inspect $ctrname --format $format + is "$output" "$expect" "$expect_msg" + + output=$cid +} + +function _check_health_log { + local ctrname="$1" + local expect_msg="$2" + local comparison=$3 + local expect_count="$4" + + run_podman inspect $ctrname --format "{{.State.Health.Log}}" + count=$(grep -co "$expect_msg" <<< "$output") + assert "$count" $comparison $expect_count "Number of matching health log messages" +} + +@test "podman healthcheck --health-max-log-count default value (5)" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + _create_container_with_health_log_settings $ctrname $msg "{{.Config.HealthMaxLogCount}}" "" "5" "HealthMaxLogCount is the expected default" + + for i in $(seq 1 10); + do + run_podman healthcheck run $ctrname + is "$output" "" "unexpected output from podman healthcheck run (pass $i)" + done + + _check_health_log $ctrname $msg -eq 5 + + run_podman rm -t 0 -f $ctrname +} + +@test "podman healthcheck --health-max-log-count infinite value (0)" { + local repeat_count=10 + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + _create_container_with_health_log_settings $ctrname $msg "{{.Config.HealthMaxLogCount}}" "--health-max-log-count 0" "0" "HealthMaxLogCount" + + # This is run one more time than repeat_count to check that the cap is working. + for i in $(seq 1 $(($repeat_count + 1))); + do + run_podman healthcheck run $ctrname + is "$output" "" "unexpected output from podman healthcheck run (pass $i)" + done + + # The healthcheck is triggered by the podman when the container is started, but its execution depends on systemd. + # And since `run_podman healthcheck run` is also run manually, it will result in two runs. + _check_health_log $ctrname $msg -ge 11 + + run_podman rm -t 0 -f $ctrname +} + + +@test "podman healthcheck --health-max-log-count 10" { + local repeat_count=10 + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + _create_container_with_health_log_settings $ctrname $msg "{{.Config.HealthMaxLogCount}}" "--health-max-log-count $repeat_count" "$repeat_count" "HealthMaxLogCount" + + # This is run one more time than repeat_count to check that the cap is working. + for i in $(seq 1 $(($repeat_count + 1))); + do + run_podman healthcheck run $ctrname + is "$output" "" "unexpected output from podman healthcheck run (pass $i)" + done + + _check_health_log $ctrname $msg -eq $repeat_count + + run_podman rm -t 0 -f $ctrname +} + +@test "podman healthcheck --health-max-log-size 10" { + local msg="healthmsg-$(random_string)" + local ctrname="c-h-$(safename)" + _create_container_with_health_log_settings $ctrname $msg "{{.Config.HealthMaxLogSize}}" "--health-max-log-size 10" "10" "HealthMaxLogSize" + + run_podman healthcheck run $ctrname + is "$output" "" "output from 'podman healthcheck run'" + + local substr=${msg:0:10} + _check_health_log $ctrname "$substr}]\$" -eq 1 + + run_podman rm -t 0 -f $ctrname +} + +@test "podman healthcheck --health-max-log-size infinite value (0)" { + local s=$(printf "healthmsg-%1000s") + local long_msg=${s// /$(random_string)} + local ctrname="c-h-$(safename)" + _create_container_with_health_log_settings $ctrname $long_msg "{{.Config.HealthMaxLogSize}}" "--health-max-log-size 0" "0" "HealthMaxLogSize" + + run_podman healthcheck run $ctrname + is "$output" "" "output from 'podman healthcheck run'" + + # The healthcheck is triggered by the podman when the container is started, but its execution depends on systemd. + # And since `run_podman healthcheck run` is also run manually, it will result in two runs. + _check_health_log $ctrname "$long_msg" -ge 1 + + run_podman rm -t 0 -f $ctrname +} + +@test "podman healthcheck --health-max-log-size default value (500)" { + local s=$(printf "healthmsg-%1000s") + local long_msg=${s// /$(random_string)} + local ctrname="c-h-$(safename)" + _create_container_with_health_log_settings $ctrname $long_msg "{{.Config.HealthMaxLogSize}}" "" "500" "HealthMaxLogSize is the expected default" + + run_podman healthcheck run $ctrname + is "$output" "" "output from 'podman healthcheck run'" + + local expect_msg="${long_msg:0:500}" + _check_health_log $ctrname "$expect_msg}]\$" -eq 1 + + run_podman rm -t 0 -f $ctrname +} + + +@test "podman healthcheck --health-log-destination file" { + local TMP_DIR_HEALTHCHECK="$PODMAN_TMPDIR/healthcheck" + mkdir $TMP_DIR_HEALTHCHECK + local ctrname="c-h-$(safename)" + local msg="healthmsg-$(random_string)" + _create_container_with_health_log_settings $ctrname $msg "{{.Config.HealthLogDestination}}" "--health-log-destination $TMP_DIR_HEALTHCHECK" "$TMP_DIR_HEALTHCHECK" "HealthLogDestination" + cid="$output" + + run_podman healthcheck run $ctrname + is "$output" "" "output from 'podman healthcheck run'" + + healthcheck_log_path="${TMP_DIR_HEALTHCHECK}/${cid}-healthcheck.log" + # The healthcheck is triggered by the podman when the container is started, but its execution depends on systemd. + # And since `run_podman healthcheck run` is also run manually, it will result in two runs. + count=$(grep -co "$msg" $healthcheck_log_path) + assert "$count" -ge 1 "Number of matching health log messages" + + run_podman rm -t 0 -f $ctrname +} + + +@test "podman healthcheck --health-log-destination journal" { + skip_if_remote "We cannot read journalctl over remote." + + # We can't use journald on RHEL as rootless, either: rhbz#1895105 + skip_if_journald_unavailable + + local ctrname="c-h-$(safename)" + local msg="healthmsg-$(random_string)" + _create_container_with_health_log_settings $ctrname $msg "{{.Config.HealthLogDestination}}" "--health-log-destination events_logger" "events_logger" "HealthLogDestination" + cid="$output" + + run_podman healthcheck run $ctrname + is "$output" "" "output from 'podman healthcheck run'" + + cmd="journalctl --output cat --output-fields=PODMAN_HEALTH_LOG PODMAN_ID=$cid" + echo "$_LOG_PROMPT $cmd" + run $cmd + echo "$output" + assert "$status" -eq 0 "exit status of journalctl" + + # The healthcheck is triggered by the podman when the container is started, but its execution depends on systemd. + # And since `run_podman healthcheck run` is also run manually, it will result in two runs. + count=$(grep -co "$msg" <<< "$output") + assert "$count" -ge 1 "Number of matching health log messages" + + run_podman rm -t 0 -f $ctrname +} + # vim: filetype=sh