zoneminder/scripts/zmwatch.pl.in

#!@PERL_EXECUTABLE@ -wT
#
# ==========================================================================
#
# ZoneMinder WatchDog Script, $Date$, $Revision$
# Copyright (C) 2001-2008 Philip Coombes
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# ==========================================================================

=head1 NAME

zmwatch.pl - ZoneMinder WatchDog Script

=head1 SYNOPSIS

zmwatch.pl

=head1 DESCRIPTION

This does some basic setup for ZoneMinder to run and then periodically
checks the fps output of the active daemons to check they haven't 
locked up. If they have then they are killed and restarted

=cut
use strict;
use bytes;

# ==========================================================================
#
# These are the elements you can edit to suit your installation
#
# ==========================================================================


# ==========================================================================
#
# Don't change anything below here
#
# ==========================================================================

@EXTRA_PERL_LIB@
use ZoneMinder;
use ZoneMinder::Monitor;
use POSIX;
use DBI;
use autouse 'Data::Dumper'=>qw(Dumper);

$| = 1;

$ENV{PATH}  = '@ZM_SCRIPT_PATH@';
$ENV{SHELL} = '/bin/sh' if exists $ENV{SHELL};
delete @ENV{qw(IFS CDPATH ENV BASH_ENV)};

logInit();
logSetSignal();
my $zm_terminate = 0;
sub TermHandler {
  Debug('Received TERM, exiting');
  $zm_terminate = 1;
}
$SIG{TERM} = \&TermHandler;
$SIG{INT} = \&TermHandler;

my $dbh = zmDbConnect();

# We don't need to keep objects cached
$ZoneMinder::Object::no_cache = 1;

# Wall-clock of the last completed check pass, to detect systemic stalls (see below).
my $last_pass_time = time();

# --- Startup grace --------------------------------------------------------
# This used to be a flat sleep(30) before the first pass. When we start, zmc
# has not necessarily created its shared memory or written a heartbeat yet,
# and neither is distinguishable from a camera that has died, so the sleep was
# there to stay out of the way while the system came up.
#
# A fixed delay is the wrong shape for that. Too short and a busy host with
# many cameras still gets its whole fleet restarted the moment we wake; too
# long and a genuinely dead camera goes unwatched for no reason. It also
# punishes anyone running this by hand.
#
# Instead, give each monitor we have not yet seen healthy until
# ZM_WATCH_MAX_DELAY after we started -- the same threshold used to decide a
# heartbeat is stale -- before acting on it. Monitors we have seen healthy are
# handled immediately, so a camera that dies after we start is caught as fast
# as it ever was, and every other check runs from the first pass.
my $start_time = time();
my %seen_healthy;

while (!$zm_terminate) {
  my $db_was_down = 0;
  while (!($dbh and $dbh->ping()) and !$zm_terminate) {
    $db_was_down = 1;
    if (!($dbh = zmDbConnect())) {
      sleep($Config{ZM_WATCH_CHECK_INTERVAL});
    }
  }

  # --- Systemic-stall guard -------------------------------------------------
  # The capture daemons and this watchdog share the database. If the DB (or the
  # host/network) drops out, EVERY zmc stops updating its shared-memory
  # heartbeat at the same instant. When the DB returns, this loop would see the
  # whole fleet as simultaneously "stale" and restart -- and reboot -- every
  # camera, even though nothing is wrong with any of them. That is the cause of
  # the fleet-wide restart storms (each one coincided with a DB outage).
  #
  # Distinguish "this camera died" from "the thing we both depend on died": if
  # we just came out of a DB reconnect, or far more wall-clock than a normal
  # cycle has elapsed (host stall / clock step), skip restart+reboot actions for
  # ONE pass so the daemons can refresh their heartbeats. A genuinely dead
  # camera is still stale next pass (~check interval later) and gets handled
  # then -- reboot behaviour is preserved, just no longer fired on a false
  # fleet-wide positive.
  my $pass_now = time();
  my $since_last_pass = $pass_now - $last_pass_time;
  $last_pass_time = $pass_now;
  my $stall_grace = $db_was_down
    || ($since_last_pass > 2 * $Config{ZM_WATCH_CHECK_INTERVAL});
  if ($stall_grace) {
    Warning("zmwatch: systemic stall detected (db_was_down=$db_was_down, "
      ."${since_last_pass}s since last check) - skipping restart/reboot actions "
      ."this pass so capture daemons can refresh their heartbeats");
  }

  foreach my $monitor (ZoneMinder::Monitor->find(Deleted=>0, $Config{ZM_SERVER_ID} ? (ServerId=>$Config{ZM_SERVER_ID}) : ())) {
    next if $stall_grace; # systemic DB/host stall this pass: don't restart/reboot on false staleness
    next if $monitor->{Capturing} eq 'None';
    next if $monitor->{Type} eq 'WebSite';

    my $now = time(); 
    my $restart = 0;
    # True only while a monitor we have never seen running is still inside the
    # window it is allowed to take to come up. See the startup grace note above.
    my $starting_up = (!$seen_healthy{$monitor->{Id}})
      && (($now - $start_time) < $Config{ZM_WATCH_MAX_DELAY});

    if (!zmMemVerify($monitor)) {
      if ($starting_up) {
        Debug("Not restarting $monitor->{Id} $monitor->{Name} yet, no shared data but still inside its startup grace");
        next;
      }
      Info("Restarting capture daemon for $monitor->{Id} $monitor->{Name}, shared data not valid");
      $monitor->control('restart');
      next;
    }

    my $heartbeat_time = zmMemRead($monitor, 'shared_data:heartbeat_time');
    my $heartbeat_elapsed = $now-$heartbeat_time;
    if ($heartbeat_elapsed > $Config{ZM_WATCH_MAX_DELAY}) {
      if ($starting_up) {
        Debug("Not restarting $monitor->{Id} $monitor->{Name} yet, no heartbeat but still inside its startup grace");
        next;
      }
      Info("Restarting capture daemon for $monitor->{Id} $monitor->{Name}, $now - heartbeat time $heartbeat_time $heartbeat_elapsed > $Config{ZM_WATCH_MAX_DELAY}");
      $monitor->control('restart');
      next;
    } else {
      # It is up. From here on it is judged immediately, like any other monitor.
      $seen_healthy{$monitor->{Id}} = 1;
      Debug("Monitor $monitor->{Id} $monitor->{Name}, heartbeat time $now - $heartbeat_time $heartbeat_elapsed < $Config{ZM_WATCH_MAX_DELAY}");
    }

    next if $monitor->{Capturing} eq 'Ondemand';
    next if $monitor->{Decoding} eq 'None' or $monitor->{Decoding} eq 'Ondemand';

    # Check we have got an image recently
    my $capture_time = zmGetLastWriteTime($monitor);
    if (!defined($capture_time)) {
      # Can't read from shared data
      Warning('LastWriteTime is not defined.');
      next;
    }
    Debug("Monitor $$monitor{Id} LastWriteTime is $capture_time.");
    if (!$capture_time) {
      # We can't get the last capture time so can't be sure it's died, it might just be starting up.
      my $startup_time = zmGetStartupTime($monitor);
      my $startup_elapsed = $now - $startup_time;
      if ($startup_elapsed > $Config{ZM_WATCH_MAX_DELAY}) {
        Debug("Monitor $monitor->{Id} $monitor->{Name}, startup time $now - $startup_time $startup_elapsed <? $Config{ZM_WATCH_MAX_DELAY}");
        if ($monitor->ControlId()) {
          my $control = $monitor->Control();
          # Only try to reboot the camera if it actually answers. Otherwise
          # open() blocks until it times out on a camera that is down (the
          # common reason there is no image since startup). ping() resolves the
          # host from the monitor itself, so no need to dig the ip out here.
          if ($control and $control->CanReboot()) {
            if (!$control->ping()) {
              Debug("Not rebooting $monitor->{Id} $monitor->{Name}: camera is not reachable");
            } elsif ($control->open()) {
              $control->reboot();
            }
          }
        }
        WarningImportance($monitor->ImportanceNumber(),
          "Restarting capture daemon for $monitor->{Id} $$monitor{Name}, no image since startup. ".
          "Startup time was $startup_time - now $now > $Config{ZM_WATCH_MAX_DELAY}"
        );
        $monitor->control('restart');
      }
      next;
    }

    my $max_image_delay = (
      $monitor->{MaxFPS}
      &&($monitor->{MaxFPS}>0)
      &&($monitor->{MaxFPS}<1)
    ) ? (3/$monitor->{MaxFPS})
    : $Config{ZM_WATCH_MAX_DELAY};

    my $image_delay = $now - $capture_time;
    Debug("Monitor $monitor->{Id} last captured $image_delay seconds ago, max is $max_image_delay");
    if ($image_delay > $max_image_delay) {
      WarningImportance($monitor->ImportanceNumber(),
        'Restarting capture daemon for '.$monitor->{Name}.
        ", time since last capture $image_delay seconds ($now-$capture_time)");
      $monitor->control('restart');
      next;
    }

    if ($monitor->{Analysing} ne 'None') {
      # Now check analysis thread
# Check we have got an image recently
      my $image_time = zmGetLastReadTime($monitor);
      if (!defined($image_time)) {
        # Can't read from shared data
        Error("Error reading shared data for $$monitor{Id} $$monitor{Name}");
        $monitor->control('restart');
        next;
      } elsif (!$image_time) {
        Debug("Last analyse time for $$monitor{Id} $$monitor{Name} was zero.");
      } else {
        my $max_image_delay = ( $monitor->{MaxFPS}
            &&($monitor->{MaxFPS}>0)
            &&($monitor->{MaxFPS}<1)
            ) ? (3/$monitor->{MaxFPS})
          : $Config{ZM_WATCH_MAX_DELAY}
        ;
        my $image_delay = $now-$image_time;
        Debug("Monitor $monitor->{Id} last analysed $image_delay seconds ago, max is $max_image_delay");
        if ($image_delay > $max_image_delay) {
          WarningImportance($monitor->ImportanceNumber(),
            "daemon for $$monitor{Id} $$monitor{Name} needs restarting,"
            ." time since last analysis $image_delay seconds ($now-$image_time)");
          $monitor->control('restart');
          next;
        }
      }
    } # end if check analysis daemon
  } # end foreach monitor

  Debug("Sleeping $Config{ZM_WATCH_CHECK_INTERVAL}");
  sleep($Config{ZM_WATCH_CHECK_INTERVAL});
  Debug("Done Sleeping $Config{ZM_WATCH_CHECK_INTERVAL}");
} # end while (!$zm_terminate)

Debug('Watchdog exiting');
exit();

1;
__END__