mirror of
https://github.com/ZoneMinder/zoneminder.git
synced 2026-11-07 05:03:00 -05:00
319 lines
13 KiB
Plaintext
319 lines
13 KiB
Plaintext
#!@PERL_EXECUTABLE@ -wT
|
|
#
|
|
# ==========================================================================
|
|
#
|
|
# ZoneMinder WatchDog Script, $Date$, $Revision$
|
|
# Copyright (C) 2001-2008 Philip Coombes
|
|
#
|
|
# This program is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU General Public License
|
|
# as published by the Free Software Foundation; either version 2
|
|
# of the License, or (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program; if not, write to the Free Software
|
|
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
|
#
|
|
# ==========================================================================
|
|
|
|
=head1 NAME
|
|
|
|
zmwatch.pl - ZoneMinder WatchDog Script
|
|
|
|
=head1 SYNOPSIS
|
|
|
|
zmwatch.pl
|
|
|
|
=head1 DESCRIPTION
|
|
|
|
This does some basic setup for ZoneMinder to run and then periodically
|
|
checks the fps output of the active daemons to check they haven't
|
|
locked up. If they have then they are killed and restarted
|
|
|
|
=cut
|
|
use strict;
|
|
use bytes;
|
|
|
|
# ==========================================================================
|
|
#
|
|
# These are the elements you can edit to suit your installation
|
|
#
|
|
# ==========================================================================
|
|
|
|
|
|
# ==========================================================================
|
|
#
|
|
# Don't change anything below here
|
|
#
|
|
# ==========================================================================
|
|
|
|
@EXTRA_PERL_LIB@
|
|
use ZoneMinder;
|
|
use ZoneMinder::Monitor;
|
|
use ZoneMinder::Event;
|
|
use POSIX;
|
|
use DBI;
|
|
use autouse 'Data::Dumper'=>qw(Dumper);
|
|
|
|
$| = 1;
|
|
|
|
$ENV{PATH} = '@ZM_SCRIPT_PATH@';
|
|
$ENV{SHELL} = '/bin/sh' if exists $ENV{SHELL};
|
|
delete @ENV{qw(IFS CDPATH ENV BASH_ENV)};
|
|
|
|
logInit();
|
|
logSetSignal();
|
|
my $zm_terminate = 0;
|
|
sub TermHandler {
|
|
Debug('Received TERM, exiting');
|
|
$zm_terminate = 1;
|
|
}
|
|
$SIG{TERM} = \&TermHandler;
|
|
$SIG{INT} = \&TermHandler;
|
|
|
|
my $dbh = zmDbConnect();
|
|
|
|
# We don't need to keep objects cached
|
|
$ZoneMinder::Object::no_cache = 1;
|
|
|
|
# Wall-clock of the last completed check pass, to detect systemic stalls (see below).
|
|
my $last_pass_time = time();
|
|
|
|
# --- Startup grace --------------------------------------------------------
|
|
# This used to be a flat sleep(30) before the first pass. When we start, zmc
|
|
# has not necessarily created its shared memory or written a heartbeat yet,
|
|
# and neither is distinguishable from a camera that has died, so the sleep was
|
|
# there to stay out of the way while the system came up.
|
|
#
|
|
# A fixed delay is the wrong shape for that. Too short and a busy host with
|
|
# many cameras still gets its whole fleet restarted the moment we wake; too
|
|
# long and a genuinely dead camera goes unwatched for no reason. It also
|
|
# punishes anyone running this by hand.
|
|
#
|
|
# Instead, give each monitor we have not yet seen healthy until
|
|
# ZM_WATCH_MAX_DELAY after we started -- the same threshold used to decide a
|
|
# heartbeat is stale -- before acting on it. Monitors we have seen healthy are
|
|
# handled immediately, so a camera that dies after we start is caught as fast
|
|
# as it ever was, and every other check runs from the first pass.
|
|
#
|
|
# The same applies whenever a zmc starts later: a monitor added or restarted
|
|
# (web UI, API, zmdc, or us) has no shared data until its new zmc has loaded
|
|
# the monitor and mapped it, which takes a few seconds. So a missing or invalid
|
|
# shared data block is also given ZM_WATCH_MAX_DELAY, counted from the pass
|
|
# that first found it that way, before we restart the daemon. Restarting it
|
|
# earlier only replaces a zmc that was about to come up with another one.
|
|
my $start_time = time();
|
|
my %seen_healthy;
|
|
my %no_shared_data_since;
|
|
|
|
# --- Events left open by a zmc that died ------------------------------------
|
|
# A zmc that is killed (or crashes) mid-event never writes the event's end, and
|
|
# nothing else closes it: zmaudit only does when ZM_RUN_AUDIT is on, and only a
|
|
# day later. A running zmc closes every event it opened before it reconnects,
|
|
# and records the time it connected as startup_time in shared memory, so any
|
|
# event of its monitor that started before then and is still open belongs to a
|
|
# process that is gone. Look once per zmc start, when its heartbeat shows it is
|
|
# up. Events from zma's re-analysis are created on the monitor too, with their
|
|
# original times, and are left to zma.
|
|
my %recovered_before;
|
|
|
|
sub close_abandoned_events {
|
|
my ($monitor, $before) = @_;
|
|
foreach my $event (ZoneMinder::Event->find(custom => [
|
|
'`MonitorId`=? AND `EndDateTime` IS NULL AND `StartDateTime` < FROM_UNIXTIME(?) AND NOT (`Cause` <=> ?)',
|
|
$$monitor{Id}, $before, 'Re-analysis'])) {
|
|
my $error = $event->Close();
|
|
if ($error) {
|
|
Warning("Unable to close event $$event{Id} left open on monitor $$monitor{Id} $$monitor{Name}: $error");
|
|
} else {
|
|
Info("Closed event $$event{Id} on monitor $$monitor{Id} $$monitor{Name}, left open since $$event{StartDateTime} by a capture daemon that is gone");
|
|
}
|
|
}
|
|
}
|
|
|
|
while (!$zm_terminate) {
|
|
my $db_was_down = 0;
|
|
while (!($dbh and $dbh->ping()) and !$zm_terminate) {
|
|
$db_was_down = 1;
|
|
if (!($dbh = zmDbConnect())) {
|
|
sleep($Config{ZM_WATCH_CHECK_INTERVAL});
|
|
}
|
|
}
|
|
|
|
# --- Systemic-stall guard -------------------------------------------------
|
|
# The capture daemons and this watchdog share the database. If the DB (or the
|
|
# host/network) drops out, EVERY zmc stops updating its shared-memory
|
|
# heartbeat at the same instant. When the DB returns, this loop would see the
|
|
# whole fleet as simultaneously "stale" and restart -- and reboot -- every
|
|
# camera, even though nothing is wrong with any of them. That is the cause of
|
|
# the fleet-wide restart storms (each one coincided with a DB outage).
|
|
#
|
|
# Distinguish "this camera died" from "the thing we both depend on died": if
|
|
# we just came out of a DB reconnect, or far more wall-clock than a normal
|
|
# cycle has elapsed (host stall / clock step), skip restart+reboot actions for
|
|
# ONE pass so the daemons can refresh their heartbeats. A genuinely dead
|
|
# camera is still stale next pass (~check interval later) and gets handled
|
|
# then -- reboot behaviour is preserved, just no longer fired on a false
|
|
# fleet-wide positive.
|
|
my $pass_now = time();
|
|
my $since_last_pass = $pass_now - $last_pass_time;
|
|
$last_pass_time = $pass_now;
|
|
my $stall_grace = $db_was_down
|
|
|| ($since_last_pass > 2 * $Config{ZM_WATCH_CHECK_INTERVAL});
|
|
if ($stall_grace) {
|
|
Warning("zmwatch: systemic stall detected (db_was_down=$db_was_down, "
|
|
."${since_last_pass}s since last check) - skipping restart/reboot actions "
|
|
."this pass so capture daemons can refresh their heartbeats");
|
|
}
|
|
|
|
foreach my $monitor (ZoneMinder::Monitor->find(Deleted=>0, $Config{ZM_SERVER_ID} ? (ServerId=>$Config{ZM_SERVER_ID}) : ())) {
|
|
next if $stall_grace; # systemic DB/host stall this pass: don't restart/reboot on false staleness
|
|
next if $monitor->{Capturing} eq 'None';
|
|
next if $monitor->{Type} eq 'WebSite';
|
|
|
|
my $now = time();
|
|
my $restart = 0;
|
|
# True only while a monitor we have never seen running is still inside the
|
|
# window it is allowed to take to come up. See the startup grace note above.
|
|
my $starting_up = (!$seen_healthy{$monitor->{Id}})
|
|
&& (($now - $start_time) < $Config{ZM_WATCH_MAX_DELAY});
|
|
|
|
if (!zmMemVerify($monitor)) {
|
|
my $since = $no_shared_data_since{$monitor->{Id}} //= $now;
|
|
if ($starting_up or (($now - $since) < $Config{ZM_WATCH_MAX_DELAY})) {
|
|
Debug("Not restarting $monitor->{Id} $monitor->{Name} yet, no shared data for ".($now - $since)."s, still inside its startup grace");
|
|
next;
|
|
}
|
|
Info("Restarting capture daemon for $monitor->{Id} $monitor->{Name}, shared data not valid for ".($now - $since).'s');
|
|
delete $no_shared_data_since{$monitor->{Id}};
|
|
$monitor->control('restart');
|
|
next;
|
|
}
|
|
delete $no_shared_data_since{$monitor->{Id}};
|
|
|
|
my $heartbeat_time = zmMemRead($monitor, 'shared_data:heartbeat_time');
|
|
my $heartbeat_elapsed = $now-$heartbeat_time;
|
|
if ($heartbeat_elapsed > $Config{ZM_WATCH_MAX_DELAY}) {
|
|
if ($starting_up) {
|
|
Debug("Not restarting $monitor->{Id} $monitor->{Name} yet, no heartbeat but still inside its startup grace");
|
|
next;
|
|
}
|
|
Info("Restarting capture daemon for $monitor->{Id} $monitor->{Name}, $now - heartbeat time $heartbeat_time $heartbeat_elapsed > $Config{ZM_WATCH_MAX_DELAY}");
|
|
$monitor->control('restart');
|
|
next;
|
|
} else {
|
|
# It is up. From here on it is judged immediately, like any other monitor.
|
|
$seen_healthy{$monitor->{Id}} = 1;
|
|
Debug("Monitor $monitor->{Id} $monitor->{Name}, heartbeat time $now - $heartbeat_time $heartbeat_elapsed < $Config{ZM_WATCH_MAX_DELAY}");
|
|
}
|
|
|
|
my $zmc_startup_time = zmGetStartupTime($monitor);
|
|
if ($zmc_startup_time and (($recovered_before{$monitor->{Id}} // 0) != $zmc_startup_time)) {
|
|
close_abandoned_events($monitor, $zmc_startup_time);
|
|
$recovered_before{$monitor->{Id}} = $zmc_startup_time;
|
|
}
|
|
|
|
next if $monitor->{Capturing} eq 'Ondemand';
|
|
next if $monitor->{Decoding} eq 'None' or $monitor->{Decoding} eq 'Ondemand';
|
|
|
|
# Check we have got an image recently
|
|
my $capture_time = zmGetLastWriteTime($monitor);
|
|
if (!defined($capture_time)) {
|
|
# Can't read from shared data
|
|
Warning('LastWriteTime is not defined.');
|
|
next;
|
|
}
|
|
Debug("Monitor $$monitor{Id} LastWriteTime is $capture_time.");
|
|
if (!$capture_time) {
|
|
# We can't get the last capture time so can't be sure it's died, it might just be starting up.
|
|
my $startup_time = zmGetStartupTime($monitor);
|
|
my $startup_elapsed = $now - $startup_time;
|
|
if ($startup_elapsed > $Config{ZM_WATCH_MAX_DELAY}) {
|
|
Debug("Monitor $monitor->{Id} $monitor->{Name}, startup time $now - $startup_time $startup_elapsed <? $Config{ZM_WATCH_MAX_DELAY}");
|
|
if ($monitor->ControlId()) {
|
|
my $control = $monitor->Control();
|
|
# Only try to reboot the camera if it actually answers. Otherwise
|
|
# open() blocks until it times out on a camera that is down (the
|
|
# common reason there is no image since startup). ping() resolves the
|
|
# host from the monitor itself, so no need to dig the ip out here.
|
|
if ($control and $control->CanReboot()) {
|
|
if (!$control->ping()) {
|
|
Debug("Not rebooting $monitor->{Id} $monitor->{Name}: camera is not reachable");
|
|
} elsif ($control->open()) {
|
|
$control->reboot();
|
|
}
|
|
}
|
|
}
|
|
WarningImportance($monitor->ImportanceNumber(),
|
|
"Restarting capture daemon for $monitor->{Id} $$monitor{Name}, no image since startup. ".
|
|
"Startup time was $startup_time - now $now > $Config{ZM_WATCH_MAX_DELAY}"
|
|
);
|
|
$monitor->control('restart');
|
|
}
|
|
next;
|
|
}
|
|
|
|
my $max_image_delay = (
|
|
$monitor->{MaxFPS}
|
|
&&($monitor->{MaxFPS}>0)
|
|
&&($monitor->{MaxFPS}<1)
|
|
) ? (3/$monitor->{MaxFPS})
|
|
: $Config{ZM_WATCH_MAX_DELAY};
|
|
|
|
my $image_delay = $now - $capture_time;
|
|
Debug("Monitor $monitor->{Id} last captured $image_delay seconds ago, max is $max_image_delay");
|
|
if ($image_delay > $max_image_delay) {
|
|
WarningImportance($monitor->ImportanceNumber(),
|
|
'Restarting capture daemon for '.$monitor->{Name}.
|
|
", time since last capture $image_delay seconds ($now-$capture_time)");
|
|
$monitor->control('restart');
|
|
next;
|
|
}
|
|
|
|
if ($monitor->{Analysing} ne 'None') {
|
|
# Now check analysis thread
|
|
# Check we have got an image recently
|
|
my $image_time = zmGetLastReadTime($monitor);
|
|
if (!defined($image_time)) {
|
|
# Can't read from shared data
|
|
Error("Error reading shared data for $$monitor{Id} $$monitor{Name}");
|
|
$monitor->control('restart');
|
|
next;
|
|
} elsif (!$image_time) {
|
|
Debug("Last analyse time for $$monitor{Id} $$monitor{Name} was zero.");
|
|
} else {
|
|
my $max_image_delay = ( $monitor->{MaxFPS}
|
|
&&($monitor->{MaxFPS}>0)
|
|
&&($monitor->{MaxFPS}<1)
|
|
) ? (3/$monitor->{MaxFPS})
|
|
: $Config{ZM_WATCH_MAX_DELAY}
|
|
;
|
|
my $image_delay = $now-$image_time;
|
|
Debug("Monitor $monitor->{Id} last analysed $image_delay seconds ago, max is $max_image_delay");
|
|
if ($image_delay > $max_image_delay) {
|
|
WarningImportance($monitor->ImportanceNumber(),
|
|
"daemon for $$monitor{Id} $$monitor{Name} needs restarting,"
|
|
." time since last analysis $image_delay seconds ($now-$image_time)");
|
|
$monitor->control('restart');
|
|
next;
|
|
}
|
|
}
|
|
} # end if check analysis daemon
|
|
} # end foreach monitor
|
|
|
|
Debug("Sleeping $Config{ZM_WATCH_CHECK_INTERVAL}");
|
|
sleep($Config{ZM_WATCH_CHECK_INTERVAL});
|
|
Debug("Done Sleeping $Config{ZM_WATCH_CHECK_INTERVAL}");
|
|
} # end while (!$zm_terminate)
|
|
|
|
Debug('Watchdog exiting');
|
|
exit();
|
|
|
|
1;
|
|
__END__
|